From 25dbade70e61c5d28ba6c20b2ade8b2b773d5e0d Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Wed, 24 Jul 2024 18:07:02 +0530 Subject: [PATCH 01/34] avniproject#762 | A working API has been successfully built to automate the creation of questions for the tables subject types and Address --- .../dao/metabase/DatabaseRepository.java | 21 +++ .../dao/metabase/MetabaseConnector.java | 5 +- .../avni/server/service/DatabaseService.java | 151 ++++++++++++++++++ .../avni/server/service/MetabaseService.java | 33 ++-- .../org/avni/server/util/StringUtils.java | 10 ++ .../avni/server/web/MetabaseController.java | 15 +- 6 files changed, 220 insertions(+), 15 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/service/DatabaseService.java create mode 100644 avni-server-api/src/main/java/org/avni/server/util/StringUtils.java diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java index bd06541b6..ac7755e7b 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java @@ -1,5 +1,6 @@ package org.avni.server.dao.metabase; +import com.fasterxml.jackson.databind.JsonNode; import org.avni.server.domain.metabase.Database; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.stereotype.Repository; @@ -16,4 +17,24 @@ public Database save(Database database) { database.setId(response.getId()); return database; } + + public JsonNode getDatabaseDetails(int databaseId) { + String url = metabaseApiUrl + "/database/" + databaseId + "?include=tables"; + return getForObject(url, JsonNode.class); + } + + public JsonNode getFields(int databaseId) { + String url = metabaseApiUrl + "/database/" + databaseId + "/fields"; + return getForObject(url, JsonNode.class); + } + + public JsonNode getInitialSyncStatus(int databaseId) { + String url = metabaseApiUrl + "/database/" + databaseId; + return getForObject(url, JsonNode.class); + } + + public JsonNode getDataset(String requestBody) { + String url = metabaseApiUrl + "/dataset"; + return postForObject(url, requestBody, JsonNode.class); + } } diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/MetabaseConnector.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/MetabaseConnector.java index d7f227e24..a4d8bea48 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/MetabaseConnector.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/MetabaseConnector.java @@ -19,7 +19,7 @@ public class MetabaseConnector { protected String metabaseApiUrl; @Value("${metabase.api.key}") - private String apiKey; + protected String apiKey; public MetabaseConnector(RestTemplateBuilder restTemplateBuilder) { this.restTemplate = restTemplateBuilder.build(); @@ -42,7 +42,7 @@ protected void sendPutRequest(String url, Map requestBody) { restTemplate.exchange(url, HttpMethod.PUT, entity, Map.class); } - protected T postForObject(String url, Object request, Class responseType) { + public T postForObject(String url, Object request, Class responseType) { HttpEntity entity = createHttpEntity(request); return restTemplate.postForObject(url, entity, responseType); } @@ -57,4 +57,5 @@ protected HttpEntity> createJsonEntity(GroupPermissionsBody HttpHeaders headers = getHeaders(); return new HttpEntity<>(body.getBody(), headers); } + } diff --git a/avni-server-api/src/main/java/org/avni/server/service/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/DatabaseService.java new file mode 100644 index 000000000..1817d8798 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/service/DatabaseService.java @@ -0,0 +1,151 @@ +package org.avni.server.service; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.avni.server.dao.metabase.DatabaseRepository; +import org.avni.server.util.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +@Service +public class DatabaseService { + + private final DatabaseRepository databaseRepository; + private final ObjectMapper objectMapper; + private final MetabaseService metabaseService; + + @Value("${metabase.api.url}") + private String metabaseApiUrl; + + @Value("${metabase.api.key}") + private String apiKey; + + @Autowired + public DatabaseService(DatabaseRepository databaseRepository, ObjectMapper objectMapper, MetabaseService metabaseService) { + this.databaseRepository = databaseRepository; + this.objectMapper = objectMapper; + this.metabaseService = metabaseService; + } + + public int getTableIdByName(int databaseId, String tableName) { + JsonNode rootNode = databaseRepository.getDatabaseDetails(databaseId); + JsonNode tablesArray = rootNode.path("tables"); + for (JsonNode tableNode : tablesArray) { + if (tableName.equals(tableNode.path("display_name").asText())) { + return tableNode.path("id").asInt(); + } + } + return -1; + } + + public int getFieldIdByTableNameAndFieldName(int databaseId, String tableName, String fieldName) { + JsonNode fieldsArray = databaseRepository.getFields(databaseId); + String snakeCaseTableName = StringUtils.toSnakeCase(tableName); + for (JsonNode fieldNode : fieldsArray) { + if (snakeCaseTableName.equals(fieldNode.path("table_name").asText()) && fieldName.equals(fieldNode.path("name").asText())) { + return fieldNode.path("id").asInt(); + } + } + return -1; + } + + public void waitForSyncCompletion(int databaseId) { + while (true) { + String syncStatus = getInitialSyncStatus(databaseId); + if ("complete".equals(syncStatus)) { + return; + } + try { + TimeUnit.SECONDS.sleep(5); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Thread was interrupted while waiting for sync completion", e); + } + } + } + + public String getInitialSyncStatus(int databaseId) { + JsonNode responseBody = databaseRepository.getInitialSyncStatus(databaseId); + return responseBody.path("initial_sync_status").asText(); + } + + public List getSubjectTypeNames(int databaseId) { + int tableId = getTableIdByName(databaseId, "Subject Type"); + String requestBody = "{\"database\":" + databaseId + ",\"query\":{\"source-table\":" + tableId + "},\"type\":\"query\",\"parameters\":[]}"; + + JsonNode response = databaseRepository.getDataset(requestBody); + + JsonNode dataNode = response.path("data"); + JsonNode rows = dataNode.path("rows"); + + List subjectTypeNames = new ArrayList<>(); + for (JsonNode row : rows) { + String name = row.get(4).asText(); + boolean isVoided = row.get(6).asBoolean(); + if (!isVoided) { + subjectTypeNames.add(name); + } + } + return subjectTypeNames; + } + + public void createQuestionsForSubjectTypes() { + int databaseId = metabaseService.getGlobalDatabaseId(); + int collectionId = metabaseService.getGlobalCollectionId(); + + waitForSyncCompletion(databaseId); + + List subjectTypeNames = getSubjectTypeNames(databaseId); + + int addressTableId = getTableIdByName(databaseId, "Address"); + int joinFieldId1 = getFieldIdByTableNameAndFieldName(databaseId, "Address", "id"); + + for (String subjectTypeName : subjectTypeNames) { + int subjectTableId = getTableIdByName(databaseId, subjectTypeName); + int joinFieldId2 = getFieldIdByTableNameAndFieldName(databaseId, subjectTypeName, "address_id"); + + ObjectNode datasetQuery = objectMapper.createObjectNode(); + datasetQuery.put("database", databaseId); + datasetQuery.put("type", "query"); + + ObjectNode query = objectMapper.createObjectNode(); + query.put("source-table", addressTableId); + + ArrayNode joins = objectMapper.createArrayNode(); + ObjectNode join = objectMapper.createObjectNode(); + join.put("fields", "all"); + join.put("alias", subjectTypeName); + + ArrayNode condition = objectMapper.createArrayNode(); + condition.add("="); + condition.add(objectMapper.createArrayNode().add("field").add(joinFieldId1).add(objectMapper.createObjectNode().put("base-type", "type/Integer"))); + condition.add(objectMapper.createArrayNode().add("field").add(joinFieldId2).add(objectMapper.createObjectNode().put("base-type", "type/Integer").put("join-alias", subjectTypeName))); + + join.set("condition", condition); + join.put("source-table", subjectTableId); + joins.add(join); + + query.set("joins", joins); + datasetQuery.set("query", query); + + ObjectNode body = objectMapper.createObjectNode(); + body.put("name", "Address + " + subjectTypeName); + body.set("dataset_query", datasetQuery); + body.put("display", "table"); + body.putNull("description"); + body.set("visualization_settings", objectMapper.createObjectNode()); + body.put("collection_id", collectionId); + body.putNull("collection_position"); + body.putNull("result_metadata"); + + databaseRepository.postForObject(metabaseApiUrl + "/card", body, JsonNode.class); + } + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/service/MetabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/MetabaseService.java index b67316a55..6d8e75997 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/MetabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/MetabaseService.java @@ -5,38 +5,37 @@ import org.avni.server.dao.metabase.DatabaseRepository; import org.avni.server.dao.metabase.GroupPermissionsRepository; import org.avni.server.domain.Organisation; -import org.avni.server.domain.metabase.AvniDatabase; -import org.avni.server.domain.metabase.Collection; -import org.avni.server.domain.metabase.CollectionPermissionsService; -import org.avni.server.domain.metabase.CollectionResponse; -import org.avni.server.domain.metabase.Database; -import org.avni.server.domain.metabase.DatabaseDetails; -import org.avni.server.domain.metabase.Group; -import org.avni.server.domain.metabase.GroupPermissionsService; +import org.avni.server.domain.metabase.*; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; + @Service public class MetabaseService { private final OrganisationService organisationService; private final AvniDatabase avniDatabase; private final DatabaseRepository databaseRepository; + private final DatabaseService databaseService; private final GroupPermissionsRepository groupPermissionsRepository; private final CollectionPermissionsRepository collectionPermissionsRepository; private final CollectionRepository collectionRepository; + private Database globalDatabase; + private CollectionResponse globalCollection; @Autowired public MetabaseService(OrganisationService organisationService, AvniDatabase avniDatabase, DatabaseRepository databaseRepository, + @Lazy DatabaseService databaseService, GroupPermissionsRepository groupPermissionsRepository, - GroupPermissionsService permissions, CollectionPermissionsRepository collectionPermissionsRepository, CollectionRepository collectionRepository) { this.organisationService = organisationService; this.avniDatabase = avniDatabase; this.databaseRepository = databaseRepository; + this.databaseService = databaseService; this.groupPermissionsRepository = groupPermissionsRepository; this.collectionPermissionsRepository = collectionPermissionsRepository; this.collectionRepository = collectionRepository; @@ -48,8 +47,10 @@ public void setupMetabase() { String dbUser = currentOrganisation.getDbUser(); Database database = databaseRepository.save(new Database(name, "postgres", new DatabaseDetails(avniDatabase, dbUser))); - + this.globalDatabase = database; + CollectionResponse metabaseCollection = collectionRepository.save(new Collection(name, name + " collection")); + this.globalCollection = metabaseCollection; Group metabaseGroup = groupPermissionsRepository.save(new Group(name)); @@ -61,4 +62,16 @@ public void setupMetabase() { collectionPermissions.updatePermissions(metabaseGroup.getId(), metabaseCollection.getId()); collectionPermissionsRepository.updateCollectionPermissions(collectionPermissions, metabaseGroup.getId(), metabaseCollection.getId()); } + + public void createQuestionsForSubjectTypes() { + databaseService.createQuestionsForSubjectTypes(); + } + + public int getGlobalDatabaseId() { + return globalDatabase.getId(); + } + + public int getGlobalCollectionId() { + return globalCollection.getId(); + } } diff --git a/avni-server-api/src/main/java/org/avni/server/util/StringUtils.java b/avni-server-api/src/main/java/org/avni/server/util/StringUtils.java new file mode 100644 index 000000000..98e592c21 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/util/StringUtils.java @@ -0,0 +1,10 @@ +package org.avni.server.util; + +public class StringUtils { + public static String toSnakeCase(String input) { + if (input == null) { + return null; + } + return input.trim().replaceAll(" +", "_").toLowerCase(); + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java index 0838c146b..37c8cfcfe 100644 --- a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java +++ b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java @@ -1,6 +1,8 @@ package org.avni.server.web; - +import org.avni.server.service.DatabaseService; import org.avni.server.domain.accessControl.PrivilegeType; +import org.avni.server.dao.metabase.MetabaseConnector; +import org.avni.server.dao.metabase.DatabaseRepository; import org.avni.server.service.MetabaseService; import org.avni.server.service.UserService; import org.avni.server.service.accessControl.AccessControlService; @@ -9,10 +11,12 @@ @RestController @RequestMapping("/api/metabase") public class MetabaseController { + private final DatabaseService databaseService; private final MetabaseService metabaseService; private final AccessControlService accessControlService; - public MetabaseController(MetabaseService metabaseService, UserService userService,AccessControlService accessControlService) { + public MetabaseController(DatabaseService databaseService,MetabaseService metabaseService, MetabaseConnector metabaseConnector,DatabaseRepository databaseRepository, UserService userService,AccessControlService accessControlService) { + this.databaseService = databaseService; this.metabaseService = metabaseService; this.accessControlService= accessControlService; } @@ -21,5 +25,10 @@ public MetabaseController(MetabaseService metabaseService, UserService userServi public void setupMetabase() { accessControlService.checkPrivilege(PrivilegeType.EditOrganisationConfiguration); metabaseService.setupMetabase(); -} + } + + @PostMapping("/create-questions") + public void createQuestions() { + databaseService.createQuestionsForSubjectTypes(); + } } From 55393b52f6ba9463fec81f80d94b23f4a61836eb Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sat, 27 Jul 2024 16:48:57 +0530 Subject: [PATCH 02/34] avniproject#762 | Metabase package created , API to get status of DB sync created and waitforsync eliminated --- .../{ => metabase}/DatabaseService.java | 22 +++++-------------- .../{ => metabase}/MetabaseService.java | 3 ++- .../avni/server/web/MetabaseController.java | 9 ++++++-- 3 files changed, 14 insertions(+), 20 deletions(-) rename avni-server-api/src/main/java/org/avni/server/service/{ => metabase}/DatabaseService.java (90%) rename avni-server-api/src/main/java/org/avni/server/service/{ => metabase}/MetabaseService.java (97%) diff --git a/avni-server-api/src/main/java/org/avni/server/service/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java similarity index 90% rename from avni-server-api/src/main/java/org/avni/server/service/DatabaseService.java rename to avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 1817d8798..86dba4e01 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -1,4 +1,4 @@ -package org.avni.server.service; +package org.avni.server.service.metabase; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -56,21 +56,6 @@ public int getFieldIdByTableNameAndFieldName(int databaseId, String tableName, S return -1; } - public void waitForSyncCompletion(int databaseId) { - while (true) { - String syncStatus = getInitialSyncStatus(databaseId); - if ("complete".equals(syncStatus)) { - return; - } - try { - TimeUnit.SECONDS.sleep(5); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Thread was interrupted while waiting for sync completion", e); - } - } - } - public String getInitialSyncStatus(int databaseId) { JsonNode responseBody = databaseRepository.getInitialSyncStatus(databaseId); return responseBody.path("initial_sync_status").asText(); @@ -100,7 +85,10 @@ public void createQuestionsForSubjectTypes() { int databaseId = metabaseService.getGlobalDatabaseId(); int collectionId = metabaseService.getGlobalCollectionId(); - waitForSyncCompletion(databaseId); + String syncStatus = getInitialSyncStatus(databaseId); + if (!"complete".equals(syncStatus)) { + throw new RuntimeException("Database initial sync is not complete."); + } List subjectTypeNames = getSubjectTypeNames(databaseId); diff --git a/avni-server-api/src/main/java/org/avni/server/service/MetabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java similarity index 97% rename from avni-server-api/src/main/java/org/avni/server/service/MetabaseService.java rename to avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java index 6d8e75997..c1a08c55d 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/MetabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java @@ -1,4 +1,4 @@ -package org.avni.server.service; +package org.avni.server.service.metabase; import org.avni.server.dao.metabase.CollectionPermissionsRepository; import org.avni.server.dao.metabase.CollectionRepository; @@ -6,6 +6,7 @@ import org.avni.server.dao.metabase.GroupPermissionsRepository; import org.avni.server.domain.Organisation; import org.avni.server.domain.metabase.*; +import org.avni.server.service.OrganisationService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; diff --git a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java index 37c8cfcfe..01dc7fde3 100644 --- a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java +++ b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java @@ -1,9 +1,9 @@ package org.avni.server.web; -import org.avni.server.service.DatabaseService; +import org.avni.server.service.metabase.DatabaseService; import org.avni.server.domain.accessControl.PrivilegeType; import org.avni.server.dao.metabase.MetabaseConnector; import org.avni.server.dao.metabase.DatabaseRepository; -import org.avni.server.service.MetabaseService; +import org.avni.server.service.metabase.MetabaseService; import org.avni.server.service.UserService; import org.avni.server.service.accessControl.AccessControlService; import org.springframework.web.bind.annotation.*; @@ -31,4 +31,9 @@ public void setupMetabase() { public void createQuestions() { databaseService.createQuestionsForSubjectTypes(); } + + @GetMapping("/sync-status") + public String getSyncStatus() { + return databaseService.getInitialSyncStatus(metabaseService.getGlobalDatabaseId()); + } } From 1817387f6d6955429474109c7f02131f6d7ffbb0 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Mon, 5 Aug 2024 23:53:49 +0530 Subject: [PATCH 03/34] avniproject#762|Question creation for Programs and Encounters with Address table automated --- .../service/metabase/DatabaseService.java | 132 ++++++++++++------ .../avni/server/web/MetabaseController.java | 2 + .../src/main/resources/application.properties | 4 +- .../V1_339_1__GroupPrivilegeConstraint.sql | 35 +++++ .../V1_339_2__OrganisationStatus.sql | 1 + .../V1_342__UpdateStandardCardTypeNames.sql | 9 ++ 6 files changed, 140 insertions(+), 43 deletions(-) create mode 100644 avni-server-api/src/main/resources/db/migration/V1_339_1__GroupPrivilegeConstraint.sql create mode 100644 avni-server-api/src/main/resources/db/migration/V1_339_2__OrganisationStatus.sql create mode 100644 avni-server-api/src/main/resources/db/migration/V1_342__UpdateStandardCardTypeNames.sql diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 86dba4e01..031d73d0c 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -92,48 +92,98 @@ public void createQuestionsForSubjectTypes() { List subjectTypeNames = getSubjectTypeNames(databaseId); - int addressTableId = getTableIdByName(databaseId, "Address"); - int joinFieldId1 = getFieldIdByTableNameAndFieldName(databaseId, "Address", "id"); - for (String subjectTypeName : subjectTypeNames) { - int subjectTableId = getTableIdByName(databaseId, subjectTypeName); - int joinFieldId2 = getFieldIdByTableNameAndFieldName(databaseId, subjectTypeName, "address_id"); - - ObjectNode datasetQuery = objectMapper.createObjectNode(); - datasetQuery.put("database", databaseId); - datasetQuery.put("type", "query"); - - ObjectNode query = objectMapper.createObjectNode(); - query.put("source-table", addressTableId); - - ArrayNode joins = objectMapper.createArrayNode(); - ObjectNode join = objectMapper.createObjectNode(); - join.put("fields", "all"); - join.put("alias", subjectTypeName); - - ArrayNode condition = objectMapper.createArrayNode(); - condition.add("="); - condition.add(objectMapper.createArrayNode().add("field").add(joinFieldId1).add(objectMapper.createObjectNode().put("base-type", "type/Integer"))); - condition.add(objectMapper.createArrayNode().add("field").add(joinFieldId2).add(objectMapper.createObjectNode().put("base-type", "type/Integer").put("join-alias", subjectTypeName))); - - join.set("condition", condition); - join.put("source-table", subjectTableId); - joins.add(join); - - query.set("joins", joins); - datasetQuery.set("query", query); - - ObjectNode body = objectMapper.createObjectNode(); - body.put("name", "Address + " + subjectTypeName); - body.set("dataset_query", datasetQuery); - body.put("display", "table"); - body.putNull("description"); - body.set("visualization_settings", objectMapper.createObjectNode()); - body.put("collection_id", collectionId); - body.putNull("collection_position"); - body.putNull("result_metadata"); - - databaseRepository.postForObject(metabaseApiUrl + "/card", body, JsonNode.class); + createQuestionForTable(databaseId, collectionId, subjectTypeName, "Address", "id", "address_id"); + } + } + + private List getProgramNamesFromOperationalProgramsTable(int databaseId) { + int operationalProgramsTableId = getTableIdByName(databaseId, "All Operational Programs"); + + String requestBody = "{\"database\":" + databaseId + ",\"query\":{\"source-table\":" + operationalProgramsTableId + "},\"type\":\"query\",\"parameters\":[]}"; + JsonNode response = databaseRepository.getDataset(requestBody); + + List programNames = new ArrayList<>(); + JsonNode rows = response.path("data").path("rows"); + for (JsonNode row : rows) { + String programName = row.get(1).asText(); + programNames.add(programName); + } + return programNames; + } + + private List extractTableNames(JsonNode databaseDetails) { + List tableNames = new ArrayList<>(); + JsonNode tablesArray = databaseDetails.path("tables"); + for (JsonNode tableNode : tablesArray) { + tableNames.add(tableNode.path("display_name").asText()); + } + return tableNames; + } + + private void createQuestionForTable(int databaseId, int collectionId, String tableName, String addressTableName, String addressField, String tableField) { + int addressTableId = getTableIdByName(databaseId, addressTableName); + int joinFieldId1 = getFieldIdByTableNameAndFieldName(databaseId, addressTableName, addressField); + int tableId = getTableIdByName(databaseId, tableName); + int joinFieldId2 = getFieldIdByTableNameAndFieldName(databaseId, tableName, tableField); + + ObjectNode datasetQuery = objectMapper.createObjectNode(); + datasetQuery.put("database", databaseId); + datasetQuery.put("type", "query"); + + ObjectNode query = objectMapper.createObjectNode(); + query.put("source-table", addressTableId); + + ArrayNode joins = objectMapper.createArrayNode(); + ObjectNode join = objectMapper.createObjectNode(); + join.put("fields", "all"); + join.put("alias", tableName); + + ArrayNode condition = objectMapper.createArrayNode(); + condition.add("="); + condition.add(objectMapper.createArrayNode().add("field").add(joinFieldId1).add(objectMapper.createObjectNode().put("base-type", "type/Integer"))); + condition.add(objectMapper.createArrayNode().add("field").add(joinFieldId2).add(objectMapper.createObjectNode().put("base-type", "type/Integer").put("join-alias", tableName))); + + join.set("condition", condition); + join.put("source-table", tableId); + joins.add(join); + + query.set("joins", joins); + datasetQuery.set("query", query); + + ObjectNode body = objectMapper.createObjectNode(); + body.put("name", "Address + " + tableName); + body.set("dataset_query", datasetQuery); + body.put("display", "table"); + body.putNull("description"); + body.set("visualization_settings", objectMapper.createObjectNode()); + body.put("collection_id", collectionId); + body.putNull("collection_position"); + body.putNull("result_metadata"); + + databaseRepository.postForObject(metabaseApiUrl + "/card", body, JsonNode.class); + } + + public void createQuestionsForProgramsAndEncounters() { + int databaseId = metabaseService.getGlobalDatabaseId(); + int collectionId = metabaseService.getGlobalCollectionId(); + + String syncStatus = getInitialSyncStatus(databaseId); + if (!"complete".equals(syncStatus)) { + throw new RuntimeException("Database initial sync is not complete."); + } + + List programNames = getProgramNamesFromOperationalProgramsTable(databaseId); + + JsonNode databaseDetails = databaseRepository.getDatabaseDetails(databaseId); + List allTableNames = extractTableNames(databaseDetails); + + for (String programName : programNames) { + for (String tableName : allTableNames) { + if (tableName.contains(programName)) { + createQuestionForTable(databaseId, collectionId, tableName, "Address", "id", "address_id"); + } + } } } } diff --git a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java index 01dc7fde3..ac7eadb70 100644 --- a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java +++ b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java @@ -30,10 +30,12 @@ public void setupMetabase() { @PostMapping("/create-questions") public void createQuestions() { databaseService.createQuestionsForSubjectTypes(); + databaseService.createQuestionsForProgramsAndEncounters(); } @GetMapping("/sync-status") public String getSyncStatus() { return databaseService.getInitialSyncStatus(metabaseService.getGlobalDatabaseId()); } + } diff --git a/avni-server-api/src/main/resources/application.properties b/avni-server-api/src/main/resources/application.properties index a76a2ac77..a48af9f96 100644 --- a/avni-server-api/src/main/resources/application.properties +++ b/avni-server-api/src/main/resources/application.properties @@ -69,7 +69,7 @@ spring.jackson.serialization.fail-on-empty-beans=false spring.jackson.mapper.accept-case-insensitive-enums=true # Application -debug=false +debug=true avni.defaultUserName=${OPENCHS_USER_NAME:admin} spring.servlet.multipart.max-file-size=10028KB spring.servlet.multipart.max-request-size=10028KB @@ -77,7 +77,7 @@ spring.servlet.multipart.max-request-size=10028KB # Network server.tomcat.protocol-header=x-forwarded-proto spring.security.require-ssl=true -server.port=${OPENCHS_SERVER_PORT:8021} +server.port=${OPENCHS_SERVER_PORT:8080} server.compression.enabled=true server.compression.min-response-size=2048 server.compression.mime-types=application/json,application/xml,text/html,text/xml,text/plain,application/hal+json,application/javascript diff --git a/avni-server-api/src/main/resources/db/migration/V1_339_1__GroupPrivilegeConstraint.sql b/avni-server-api/src/main/resources/db/migration/V1_339_1__GroupPrivilegeConstraint.sql new file mode 100644 index 000000000..2223e75af --- /dev/null +++ b/avni-server-api/src/main/resources/db/migration/V1_339_1__GroupPrivilegeConstraint.sql @@ -0,0 +1,35 @@ +alter table group_privilege add column if not exists impl_version int not null default 1; + +update group_privilege +set impl_version = 0 +where is_voided = true; + +create or replace function check_group_privilege_uniqueness(groupPrivilegeId int, groupId int, privilegeId int, subjectTypeId int, programId int, programEncounterTypeId int, encounterTypeId int, checklistDetailId int, implVersion int) returns boolean + language plpgsql +as +$$ +declare +begin + if exists (select gp.* + from public.group_privilege gp + where gp.group_id = groupId + and gp.privilege_id = privilegeId + and (gp.subject_type_id = subjectTypeId or (gp.subject_type_id is null and subjectTypeId is null)) + and (gp.program_id = programId or (gp.program_id is null and programId is null)) + and (gp.program_encounter_type_id = programEncounterTypeId or (gp.program_encounter_type_id is null and programEncounterTypeId is null)) + and (gp.encounter_type_id = encounterTypeId or (gp.encounter_type_id is null and encounterTypeId is null)) + and (gp.checklist_detail_id = checklistDetailId or (gp.checklist_detail_id is null and checklistDetailId is null)) + and gp.id <> groupPrivilegeId + and gp.impl_version = 1 + and implVersion = 1 + ) then + raise 'Duplicate group privilege exists for: id: %, group_id: %, privilege_id: % subject_type_id: %, program_id: %, program_encounter_type_id: %, encounter_type_id: %, checklist_detail_id: %', groupPrivilegeId, groupId, privilegeId, subjectTypeId, programId, programEncounterTypeId, encounterTypeId, checklistDetailId; + end if; + + return true; +end +$$; + +alter table group_privilege + add constraint check_group_privilege_unique + check (check_group_privilege_uniqueness(id, group_id, privilege_id, subject_type_id, program_id, program_encounter_type_id, encounter_type_id, checklist_detail_id, impl_version)); \ No newline at end of file diff --git a/avni-server-api/src/main/resources/db/migration/V1_339_2__OrganisationStatus.sql b/avni-server-api/src/main/resources/db/migration/V1_339_2__OrganisationStatus.sql new file mode 100644 index 000000000..223dedd65 --- /dev/null +++ b/avni-server-api/src/main/resources/db/migration/V1_339_2__OrganisationStatus.sql @@ -0,0 +1 @@ +alter table organisation add column status varchar(255) not null default 'Live'; diff --git a/avni-server-api/src/main/resources/db/migration/V1_342__UpdateStandardCardTypeNames.sql b/avni-server-api/src/main/resources/db/migration/V1_342__UpdateStandardCardTypeNames.sql new file mode 100644 index 000000000..f2ead5142 --- /dev/null +++ b/avni-server-api/src/main/resources/db/migration/V1_342__UpdateStandardCardTypeNames.sql @@ -0,0 +1,9 @@ +UPDATE standard_report_card_type +SET name = 'Recent registrations', description = 'Recent registrations', last_modified_date_time = current_timestamp + WHERE name = 'Last 24 hours registrations'; +UPDATE standard_report_card_type +SET name = 'Recent enrolments', description = 'Recent enrolments', last_modified_date_time = current_timestamp + WHERE name = 'Last 24 hours enrolments'; +UPDATE standard_report_card_type +SET name = 'Recent visits', description = 'Recent visits', last_modified_date_time = current_timestamp + WHERE name = 'Last 24 hours visits'; From e8ec0c5f6c4ddacf68c74ebfca9722d8b7239771 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Tue, 6 Aug 2024 22:33:12 +0530 Subject: [PATCH 04/34] avniproject#762 | Some Refactoring changes in order to follow good coding practises --- .../service/metabase/DatabaseService.java | 100 ++++++++++-------- .../avni/server/web/MetabaseController.java | 2 +- 2 files changed, 58 insertions(+), 44 deletions(-) diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 031d73d0c..025681e15 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -12,7 +12,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeUnit; @Service public class DatabaseService { @@ -20,6 +19,8 @@ public class DatabaseService { private final DatabaseRepository databaseRepository; private final ObjectMapper objectMapper; private final MetabaseService metabaseService; + private Integer databaseId; + private Integer collectionId; @Value("${metabase.api.url}") private String metabaseApiUrl; @@ -34,8 +35,22 @@ public DatabaseService(DatabaseRepository databaseRepository, ObjectMapper objec this.metabaseService = metabaseService; } - public int getTableIdByName(int databaseId, String tableName) { - JsonNode rootNode = databaseRepository.getDatabaseDetails(databaseId); + private int getDatabaseId() { + if (databaseId == null) { + databaseId = metabaseService.getGlobalDatabaseId(); + } + return databaseId; + } + + private int getCollectionId() { + if (collectionId == null) { + collectionId = metabaseService.getGlobalCollectionId(); + } + return collectionId; + } + + public int getTableIdByName(String tableName) { + JsonNode rootNode = databaseRepository.getDatabaseDetails(getDatabaseId()); JsonNode tablesArray = rootNode.path("tables"); for (JsonNode tableNode : tablesArray) { if (tableName.equals(tableNode.path("display_name").asText())) { @@ -45,8 +60,12 @@ public int getTableIdByName(int databaseId, String tableName) { return -1; } - public int getFieldIdByTableNameAndFieldName(int databaseId, String tableName, String fieldName) { - JsonNode fieldsArray = databaseRepository.getFields(databaseId); + private String createRequestBodyForDataset(int sourceTableId) { + return "{\"database\":" + getDatabaseId() + ",\"query\":{\"source-table\":" + sourceTableId + "},\"type\":\"query\",\"parameters\":[]}"; + } + + public int getFieldIdByTableNameAndFieldName(String tableName, String fieldName) { + JsonNode fieldsArray = databaseRepository.getFields(getDatabaseId()); String snakeCaseTableName = StringUtils.toSnakeCase(tableName); for (JsonNode fieldNode : fieldsArray) { if (snakeCaseTableName.equals(fieldNode.path("table_name").asText()) && fieldName.equals(fieldNode.path("name").asText())) { @@ -56,14 +75,14 @@ public int getFieldIdByTableNameAndFieldName(int databaseId, String tableName, S return -1; } - public String getInitialSyncStatus(int databaseId) { - JsonNode responseBody = databaseRepository.getInitialSyncStatus(databaseId); + public String getInitialSyncStatus() { + JsonNode responseBody = databaseRepository.getInitialSyncStatus(getDatabaseId()); return responseBody.path("initial_sync_status").asText(); } - public List getSubjectTypeNames(int databaseId) { - int tableId = getTableIdByName(databaseId, "Subject Type"); - String requestBody = "{\"database\":" + databaseId + ",\"query\":{\"source-table\":" + tableId + "},\"type\":\"query\",\"parameters\":[]}"; + public List getSubjectTypeNames() { + int tableId = getTableIdByName("Subject Type"); + String requestBody = createRequestBodyForDataset(tableId); JsonNode response = databaseRepository.getDataset(requestBody); @@ -81,32 +100,16 @@ public List getSubjectTypeNames(int databaseId) { return subjectTypeNames; } - public void createQuestionsForSubjectTypes() { - int databaseId = metabaseService.getGlobalDatabaseId(); - int collectionId = metabaseService.getGlobalCollectionId(); - - String syncStatus = getInitialSyncStatus(databaseId); - if (!"complete".equals(syncStatus)) { - throw new RuntimeException("Database initial sync is not complete."); - } - - List subjectTypeNames = getSubjectTypeNames(databaseId); + private List getProgramNamesFromOperationalProgramsTable() { + int operationalProgramsTableId = getTableIdByName("All Operational Programs"); - for (String subjectTypeName : subjectTypeNames) { - createQuestionForTable(databaseId, collectionId, subjectTypeName, "Address", "id", "address_id"); - } - } - - private List getProgramNamesFromOperationalProgramsTable(int databaseId) { - int operationalProgramsTableId = getTableIdByName(databaseId, "All Operational Programs"); - - String requestBody = "{\"database\":" + databaseId + ",\"query\":{\"source-table\":" + operationalProgramsTableId + "},\"type\":\"query\",\"parameters\":[]}"; + String requestBody = createRequestBodyForDataset(operationalProgramsTableId); JsonNode response = databaseRepository.getDataset(requestBody); List programNames = new ArrayList<>(); JsonNode rows = response.path("data").path("rows"); for (JsonNode row : rows) { - String programName = row.get(1).asText(); + String programName = row.get(1).asText(); programNames.add(programName); } return programNames; @@ -121,14 +124,14 @@ private List extractTableNames(JsonNode databaseDetails) { return tableNames; } - private void createQuestionForTable(int databaseId, int collectionId, String tableName, String addressTableName, String addressField, String tableField) { - int addressTableId = getTableIdByName(databaseId, addressTableName); - int joinFieldId1 = getFieldIdByTableNameAndFieldName(databaseId, addressTableName, addressField); - int tableId = getTableIdByName(databaseId, tableName); - int joinFieldId2 = getFieldIdByTableNameAndFieldName(databaseId, tableName, tableField); + private void createQuestionForTable(String tableName, String addressTableName, String addressField, String tableField) { + int addressTableId = getTableIdByName(addressTableName); + int joinFieldId1 = getFieldIdByTableNameAndFieldName(addressTableName, addressField); + int tableId = getTableIdByName(tableName); + int joinFieldId2 = getFieldIdByTableNameAndFieldName(tableName, tableField); ObjectNode datasetQuery = objectMapper.createObjectNode(); - datasetQuery.put("database", databaseId); + datasetQuery.put("database", getDatabaseId()); datasetQuery.put("type", "query"); ObjectNode query = objectMapper.createObjectNode(); @@ -157,23 +160,34 @@ private void createQuestionForTable(int databaseId, int collectionId, String tab body.put("display", "table"); body.putNull("description"); body.set("visualization_settings", objectMapper.createObjectNode()); - body.put("collection_id", collectionId); + body.put("collection_id", getCollectionId()); body.putNull("collection_position"); body.putNull("result_metadata"); databaseRepository.postForObject(metabaseApiUrl + "/card", body, JsonNode.class); } - public void createQuestionsForProgramsAndEncounters() { - int databaseId = metabaseService.getGlobalDatabaseId(); - int collectionId = metabaseService.getGlobalCollectionId(); + public void createQuestionsForSubjectTypes() { - String syncStatus = getInitialSyncStatus(databaseId); + String syncStatus = getInitialSyncStatus(); + if (!"complete".equals(syncStatus)) { + throw new RuntimeException("Database initial sync is not complete."); + } + + List subjectTypeNames = getSubjectTypeNames(); + + for (String subjectTypeName : subjectTypeNames) { + createQuestionForTable(subjectTypeName, "Address", "id", "address_id"); + } + } + + public void createQuestionsForProgramsAndEncounters() { + String syncStatus = getInitialSyncStatus(); if (!"complete".equals(syncStatus)) { throw new RuntimeException("Database initial sync is not complete."); } - List programNames = getProgramNamesFromOperationalProgramsTable(databaseId); + List programNames = getProgramNamesFromOperationalProgramsTable(); JsonNode databaseDetails = databaseRepository.getDatabaseDetails(databaseId); List allTableNames = extractTableNames(databaseDetails); @@ -181,7 +195,7 @@ public void createQuestionsForProgramsAndEncounters() { for (String programName : programNames) { for (String tableName : allTableNames) { if (tableName.contains(programName)) { - createQuestionForTable(databaseId, collectionId, tableName, "Address", "id", "address_id"); + createQuestionForTable(tableName, "Address", "id", "address_id"); } } } diff --git a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java index ac7eadb70..652b26fa8 100644 --- a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java +++ b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java @@ -35,7 +35,7 @@ public void createQuestions() { @GetMapping("/sync-status") public String getSyncStatus() { - return databaseService.getInitialSyncStatus(metabaseService.getGlobalDatabaseId()); + return databaseService.getInitialSyncStatus(); } } From b5bc0c68ee623c1723dfbd13a8776a75340cbeb1 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sun, 11 Aug 2024 23:44:57 +0530 Subject: [PATCH 05/34] avniproject#762 | Automated questions for Address,Media,Sync Telemetry --- .../service/metabase/DatabaseService.java | 65 ++++++++++++++++++- .../avni/server/web/MetabaseController.java | 1 + 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 025681e15..533b547ee 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -9,7 +9,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; - +import java.util.Arrays; import java.util.ArrayList; import java.util.List; @@ -60,6 +60,29 @@ public int getTableIdByName(String tableName) { return -1; } + public int getTableIdByName(String tableName, String schema) { + JsonNode rootNode = databaseRepository.getDatabaseDetails( + getDatabaseId() + ); + JsonNode tablesArray = rootNode.path("tables"); + + for (JsonNode tableNode : tablesArray) { + String tableSchema = tableNode.path("schema").asText(); + + boolean schemaMatches = schema.equals("public") + ? "public".equals(tableSchema) + : !"public".equals(tableSchema); + + if ( + tableName.equals(tableNode.path("display_name").asText()) && + schemaMatches + ) { + return tableNode.path("id").asInt(); + } + } + return -1; + } + private String createRequestBodyForDataset(int sourceTableId) { return "{\"database\":" + getDatabaseId() + ",\"query\":{\"source-table\":" + sourceTableId + "},\"type\":\"query\",\"parameters\":[]}"; } @@ -167,6 +190,34 @@ private void createQuestionForTable(String tableName, String addressTableName, S databaseRepository.postForObject(metabaseApiUrl + "/card", body, JsonNode.class); } + private void createQuestionForTable(String tableName, String schema) { + int tableId = getTableIdByName(tableName, schema); + + ObjectNode datasetQuery = objectMapper.createObjectNode(); + datasetQuery.put("database", getDatabaseId()); + datasetQuery.put("type", "query"); + + ObjectNode query = objectMapper.createObjectNode(); + query.put("source-table", tableId); + datasetQuery.set("query", query); + + ObjectNode body = objectMapper.createObjectNode(); + body.put("name", tableName); + body.set("dataset_query", datasetQuery); + body.put("display", "table"); + body.putNull("description"); + body.set("visualization_settings", objectMapper.createObjectNode()); + body.put("collection_id", getCollectionId()); + body.putNull("collection_position"); + body.putNull("result_metadata"); + + databaseRepository.postForObject( + metabaseApiUrl + "/card", + body, + JsonNode.class + ); + } + public void createQuestionsForSubjectTypes() { String syncStatus = getInitialSyncStatus(); @@ -200,4 +251,16 @@ public void createQuestionsForProgramsAndEncounters() { } } } + + public void createQuestionsForIndivdualTables() { + List tablesToCreateQuestionsFor = Arrays.asList( + "Address", + "Media", + "Sync Telemetry" + ); + + for (String tableName : tablesToCreateQuestionsFor) { + createQuestionForTable(tableName, "!public"); + } + } } diff --git a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java index 652b26fa8..6eb423fbf 100644 --- a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java +++ b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java @@ -31,6 +31,7 @@ public void setupMetabase() { public void createQuestions() { databaseService.createQuestionsForSubjectTypes(); databaseService.createQuestionsForProgramsAndEncounters(); + databaseService.createQuestionsForIndivdualTables(); } @GetMapping("/sync-status") From d67f40fee886673901eff4603e36af22928215e5 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Wed, 21 Aug 2024 09:37:04 +0530 Subject: [PATCH 06/34] avniproject#762 | Reverted the changes made to application.properties --- avni-server-api/src/main/resources/application.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/avni-server-api/src/main/resources/application.properties b/avni-server-api/src/main/resources/application.properties index a48af9f96..a76a2ac77 100644 --- a/avni-server-api/src/main/resources/application.properties +++ b/avni-server-api/src/main/resources/application.properties @@ -69,7 +69,7 @@ spring.jackson.serialization.fail-on-empty-beans=false spring.jackson.mapper.accept-case-insensitive-enums=true # Application -debug=true +debug=false avni.defaultUserName=${OPENCHS_USER_NAME:admin} spring.servlet.multipart.max-file-size=10028KB spring.servlet.multipart.max-request-size=10028KB @@ -77,7 +77,7 @@ spring.servlet.multipart.max-request-size=10028KB # Network server.tomcat.protocol-header=x-forwarded-proto spring.security.require-ssl=true -server.port=${OPENCHS_SERVER_PORT:8080} +server.port=${OPENCHS_SERVER_PORT:8021} server.compression.enabled=true server.compression.min-response-size=2048 server.compression.mime-types=application/json,application/xml,text/html,text/xml,text/plain,application/hal+json,application/javascript From bd2ba3485a3e1cb27ba2d97cd18060c6e8983679 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Thu, 22 Aug 2024 00:14:41 +0530 Subject: [PATCH 07/34] avniproject#762 | Table Metadata is now used for finding the tables.More modifications will be done during the day --- .../service/metabase/DatabaseService.java | 56 ++++++++++++------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 533b547ee..5ac6cd9ec 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -103,37 +103,51 @@ public String getInitialSyncStatus() { return responseBody.path("initial_sync_status").asText(); } - public List getSubjectTypeNames() { - int tableId = getTableIdByName("Subject Type"); - String requestBody = createRequestBodyForDataset(tableId); + private JsonNode getTableMetadata() { + int tableMetadataId = getTableIdByName("Table Metadata"); + String requestBody = createRequestBodyForDataset(tableMetadataId); + return databaseRepository.getDataset(requestBody); + } + + private String formatName(String rawName) { + String[] parts = rawName.split("_"); + StringBuilder formattedName = new StringBuilder(); - JsonNode response = databaseRepository.getDataset(requestBody); + for (String part : parts) { + formattedName.append(part.substring(0, 1).toUpperCase()) + .append(part.substring(1)) + .append(" "); + } - JsonNode dataNode = response.path("data"); - JsonNode rows = dataNode.path("rows"); + return formattedName.toString().trim(); + } + public List getSubjectTypeNames() { + JsonNode tableMetadata = getTableMetadata(); List subjectTypeNames = new ArrayList<>(); + + JsonNode rows = tableMetadata.path("data").path("rows"); for (JsonNode row : rows) { - String name = row.get(4).asText(); - boolean isVoided = row.get(6).asBoolean(); - if (!isVoided) { - subjectTypeNames.add(name); + String type = row.get(2).asText(); + if (Arrays.asList("Individual", "Household", "Group", "Person").contains(type)) { + String rawName = row.get(1).asText(); + subjectTypeNames.add(formatName(rawName)); } } return subjectTypeNames; } - private List getProgramNamesFromOperationalProgramsTable() { - int operationalProgramsTableId = getTableIdByName("All Operational Programs"); - - String requestBody = createRequestBodyForDataset(operationalProgramsTableId); - JsonNode response = databaseRepository.getDataset(requestBody); - + public List getProgramAndEncounterNames() { + JsonNode tableMetadata = getTableMetadata(); List programNames = new ArrayList<>(); - JsonNode rows = response.path("data").path("rows"); + + JsonNode rows = tableMetadata.path("data").path("rows"); for (JsonNode row : rows) { - String programName = row.get(1).asText(); - programNames.add(programName); + String type = row.get(2).asText(); + if (Arrays.asList("ProgramEncounter", "ProgramEnrolment").contains(type)) { + String rawName = row.get(1).asText(); + programNames.add(formatName(rawName)); + } } return programNames; } @@ -238,9 +252,9 @@ public void createQuestionsForProgramsAndEncounters() { throw new RuntimeException("Database initial sync is not complete."); } - List programNames = getProgramNamesFromOperationalProgramsTable(); + List programNames = getProgramAndEncounterNames(); - JsonNode databaseDetails = databaseRepository.getDatabaseDetails(databaseId); + JsonNode databaseDetails = databaseRepository.getDatabaseDetails(getDatabaseId()); List allTableNames = extractTableNames(databaseDetails); for (String programName : programNames) { From bb9a681fdafe7e0541a82b30472b9c1dd5dc7a6b Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Thu, 22 Aug 2024 13:01:50 +0530 Subject: [PATCH 08/34] avniproject#762 | Minor fix when checking for tables --- .../java/org/avni/server/service/metabase/DatabaseService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 5ac6cd9ec..ec72c9a74 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -259,7 +259,7 @@ public void createQuestionsForProgramsAndEncounters() { for (String programName : programNames) { for (String tableName : allTableNames) { - if (tableName.contains(programName)) { + if (tableName.equalsIgnoreCase(programName)) { createQuestionForTable(tableName, "Address", "id", "address_id"); } } From b9dc476beb374a9ac2f9f9987a50b0043bdcada7 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Thu, 22 Aug 2024 23:49:13 +0530 Subject: [PATCH 09/34] avniproject#762 | Enums added --- .../server/domain/metabase/TableType.java | 29 +++++++++++++++++++ .../service/metabase/DatabaseService.java | 10 +++++-- 2 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/TableType.java diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableType.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableType.java new file mode 100644 index 000000000..f361ed671 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableType.java @@ -0,0 +1,29 @@ +package org.avni.server.domain.metabase; + +public enum TableType { + INDIVIDUAL("Individual"), + HOUSEHOLD("Household"), + GROUP("Group"), + PERSON("Person"), + PROGRAM_ENCOUNTER("ProgramEncounter"), + PROGRAM_ENROLMENT("ProgramEnrolment"); + + private final String typeName; + + TableType(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + public static TableType fromString(String typeName) { + for (TableType type : TableType.values()) { + if (type.getTypeName().equalsIgnoreCase(typeName)) { + return type; + } + } + throw new IllegalArgumentException("Unknown table type: " + typeName); + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index ec72c9a74..d41ee35a5 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -1,4 +1,4 @@ -package org.avni.server.service.metabase; +ckage org.avni.server.service.metabase; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -129,14 +129,16 @@ public List getSubjectTypeNames() { JsonNode rows = tableMetadata.path("data").path("rows"); for (JsonNode row : rows) { String type = row.get(2).asText(); - if (Arrays.asList("Individual", "Household", "Group", "Person").contains(type)) { + if (Arrays.asList(TableType.INDIVIDUAL.getTypeName(), TableType.HOUSEHOLD.getTypeName(), TableType.GROUP.getTypeName(), TableType.PERSON.getTypeName()).contains(type)) { String rawName = row.get(1).asText(); subjectTypeNames.add(formatName(rawName)); } } + System.out.println("The subject type names::" + subjectTypeNames); return subjectTypeNames; } + public List getProgramAndEncounterNames() { JsonNode tableMetadata = getTableMetadata(); List programNames = new ArrayList<>(); @@ -144,14 +146,16 @@ public List getProgramAndEncounterNames() { JsonNode rows = tableMetadata.path("data").path("rows"); for (JsonNode row : rows) { String type = row.get(2).asText(); - if (Arrays.asList("ProgramEncounter", "ProgramEnrolment").contains(type)) { + if (Arrays.asList(TableType.PROGRAM_ENCOUNTER.getTypeName(), TableType.PROGRAM_ENROLMENT.getTypeName()).contains(type)) { String rawName = row.get(1).asText(); programNames.add(formatName(rawName)); } } + System.out.println("The program and encounter::" + programNames); return programNames; } + private List extractTableNames(JsonNode databaseDetails) { List tableNames = new ArrayList<>(); JsonNode tablesArray = databaseDetails.path("tables"); From 6df633a3e4a0d33dce968d2856a5c36df3d58936 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sun, 25 Aug 2024 00:05:41 +0530 Subject: [PATCH 10/34] avniproject#762 | MetabaseJoin And MetabaseQuery created --- .../server/domain/metabase/MetabaseJoin.java | 28 ++++++++++++ .../server/domain/metabase/MetabaseQuery.java | 22 +++++++++ .../service/metabase/DatabaseService.java | 45 +++++++++---------- .../service/metabase/MetabaseService.java | 2 +- .../avni/server/web/MetabaseController.java | 2 +- 5 files changed, 73 insertions(+), 26 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java new file mode 100644 index 000000000..9e5b913e9 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java @@ -0,0 +1,28 @@ +package org.avni.server.domain.metabase; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class MetabaseJoin { + private final String fields; + private final String alias; + private final int sourceTable; + private final JsonNode condition; + + public MetabaseJoin(String fields, String alias, int sourceTable, JsonNode condition) { + this.fields = fields; + this.alias = alias; + this.sourceTable = sourceTable; + this.condition = condition; + } + + public ObjectNode toJson(ObjectMapper objectMapper) { + ObjectNode joinNode = objectMapper.createObjectNode(); + joinNode.put("fields", this.fields); + joinNode.put("alias", this.alias); + joinNode.put("source-table", this.sourceTable); + joinNode.set("condition", this.condition); + return joinNode; + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java new file mode 100644 index 000000000..a2084cfa2 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java @@ -0,0 +1,22 @@ +package org.avni.server.domain.metabase; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class MetabaseQuery { + private final int sourceTable; + private final ArrayNode joins; + + public MetabaseQuery(int sourceTable, ArrayNode joins) { + this.sourceTable = sourceTable; + this.joins = joins; + } + + public ObjectNode toJson(ObjectMapper objectMapper) { + ObjectNode queryNode = objectMapper.createObjectNode(); + queryNode.put("source-table", this.sourceTable); + queryNode.set("joins", this.joins); + return queryNode; + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index d41ee35a5..75ff22f9b 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -1,10 +1,13 @@ -ckage org.avni.server.service.metabase; +package org.avni.server.service.metabase; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.avni.server.dao.metabase.DatabaseRepository; +import org.avni.server.domain.metabase.TableType; +import org.avni.server.domain.metabase.MetabaseJoin; +import org.avni.server.domain.metabase.MetabaseQuery; import org.avni.server.util.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -12,6 +15,7 @@ import java.util.Arrays; import java.util.ArrayList; import java.util.List; +import java.util.HashMap; @Service public class DatabaseService { @@ -165,35 +169,26 @@ private List extractTableNames(JsonNode databaseDetails) { return tableNames; } - private void createQuestionForTable(String tableName, String addressTableName, String addressField, String tableField) { + private void createQuestionForTable(String tableName, String addressTableName, String addressField, String tableField) throws Exception { int addressTableId = getTableIdByName(addressTableName); int joinFieldId1 = getFieldIdByTableNameAndFieldName(addressTableName, addressField); int tableId = getTableIdByName(tableName); int joinFieldId2 = getFieldIdByTableNameAndFieldName(tableName, tableField); - ObjectNode datasetQuery = objectMapper.createObjectNode(); - datasetQuery.put("database", getDatabaseId()); - datasetQuery.put("type", "query"); - - ObjectNode query = objectMapper.createObjectNode(); - query.put("source-table", addressTableId); - - ArrayNode joins = objectMapper.createArrayNode(); - ObjectNode join = objectMapper.createObjectNode(); - join.put("fields", "all"); - join.put("alias", tableName); + JsonNode conditionNode = objectMapper.readTree( + "[\"=\", [\"field\", " + joinFieldId1 + ", {\"base-type\": \"type/Integer\"}], [\"field\", " + joinFieldId2 + ", {\"base-type\": \"type/Integer\", \"join-alias\": \"" + tableName + "\"}]]" + ); + MetabaseJoin join = new MetabaseJoin("all", tableName, tableId, conditionNode); // Change: tableId as integer - ArrayNode condition = objectMapper.createArrayNode(); - condition.add("="); - condition.add(objectMapper.createArrayNode().add("field").add(joinFieldId1).add(objectMapper.createObjectNode().put("base-type", "type/Integer"))); - condition.add(objectMapper.createArrayNode().add("field").add(joinFieldId2).add(objectMapper.createObjectNode().put("base-type", "type/Integer").put("join-alias", tableName))); + ArrayNode joinsArray = objectMapper.createArrayNode(); + joinsArray.add(join.toJson(objectMapper)); - join.set("condition", condition); - join.put("source-table", tableId); - joins.add(join); + MetabaseQuery query = new MetabaseQuery(addressTableId, joinsArray); // Change: addressTableId as integer - query.set("joins", joins); - datasetQuery.set("query", query); + ObjectNode datasetQuery = objectMapper.createObjectNode(); + datasetQuery.put("database", getDatabaseId()); + datasetQuery.put("type", "query"); + datasetQuery.set("query", query.toJson(objectMapper)); ObjectNode body = objectMapper.createObjectNode(); body.put("name", "Address + " + tableName); @@ -208,6 +203,8 @@ private void createQuestionForTable(String tableName, String addressTableName, S databaseRepository.postForObject(metabaseApiUrl + "/card", body, JsonNode.class); } + + private void createQuestionForTable(String tableName, String schema) { int tableId = getTableIdByName(tableName, schema); @@ -236,7 +233,7 @@ private void createQuestionForTable(String tableName, String schema) { ); } - public void createQuestionsForSubjectTypes() { + public void createQuestionsForSubjectTypes() throws Exception { String syncStatus = getInitialSyncStatus(); if (!"complete".equals(syncStatus)) { @@ -250,7 +247,7 @@ public void createQuestionsForSubjectTypes() { } } - public void createQuestionsForProgramsAndEncounters() { + public void createQuestionsForProgramsAndEncounters() throws Exception{ String syncStatus = getInitialSyncStatus(); if (!"complete".equals(syncStatus)) { throw new RuntimeException("Database initial sync is not complete."); diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java index c1a08c55d..fad55d9ea 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java @@ -64,7 +64,7 @@ public void setupMetabase() { collectionPermissionsRepository.updateCollectionPermissions(collectionPermissions, metabaseGroup.getId(), metabaseCollection.getId()); } - public void createQuestionsForSubjectTypes() { + public void createQuestionsForSubjectTypes() throws Exception{ databaseService.createQuestionsForSubjectTypes(); } diff --git a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java index 6eb423fbf..f5624ff1f 100644 --- a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java +++ b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java @@ -28,7 +28,7 @@ public void setupMetabase() { } @PostMapping("/create-questions") - public void createQuestions() { + public void createQuestions() throws Exception{ databaseService.createQuestionsForSubjectTypes(); databaseService.createQuestionsForProgramsAndEncounters(); databaseService.createQuestionsForIndivdualTables(); From 812b4bde503b86a4a91e385631cf21e1095508ee Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sun, 25 Aug 2024 12:20:19 +0530 Subject: [PATCH 11/34] avniproject#762 | MetabaseRequestBody added --- .../server/domain/metabase/MetabaseQuery.java | 17 +++++-- .../domain/metabase/MetabaseRequestBody.java | 45 +++++++++++++++++++ .../service/metabase/DatabaseService.java | 25 +++-------- 3 files changed, 64 insertions(+), 23 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java index a2084cfa2..3e031cd4e 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java @@ -1,22 +1,31 @@ package org.avni.server.domain.metabase; -import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; public class MetabaseQuery { + private final int databaseId; private final int sourceTable; private final ArrayNode joins; + private final String type = "query"; - public MetabaseQuery(int sourceTable, ArrayNode joins) { + public MetabaseQuery(int databaseId,int sourceTable, ArrayNode joins) { + this.databaseId = databaseId; this.sourceTable = sourceTable; this.joins = joins; } + public int getDatabaseId() { + return databaseId; + } + public ObjectNode toJson(ObjectMapper objectMapper) { ObjectNode queryNode = objectMapper.createObjectNode(); - queryNode.put("source-table", this.sourceTable); - queryNode.set("joins", this.joins); + queryNode.put("source-table", sourceTable); + queryNode.set("joins", joins); + queryNode.put("type", type); + return queryNode; } } diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java new file mode 100644 index 000000000..acd919abe --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java @@ -0,0 +1,45 @@ +package org.avni.server.domain.metabase; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class MetabaseRequestBody { + private String name; + private MetabaseQuery datasetQuery; + private String display; + private String description; + private ObjectNode visualizationSettings; + private int collectionId; + private Integer collectionPosition; + private JsonNode resultMetadata; + + public MetabaseRequestBody(String name, MetabaseQuery datasetQuery, String display, String description, ObjectNode visualizationSettings, int collectionId) { + this.name = name; + this.datasetQuery = datasetQuery; + this.display = display; + this.description = description; + this.visualizationSettings = visualizationSettings; + this.collectionId = collectionId; + } + + public ObjectNode toJson(ObjectMapper objectMapper) { + ObjectNode rootNode = objectMapper.createObjectNode(); + rootNode.put("name", name); + + ObjectNode datasetQueryNode = objectMapper.createObjectNode(); + datasetQueryNode.put("database", datasetQuery.getDatabaseId()); + datasetQueryNode.put("type", "query"); + datasetQueryNode.set("query", datasetQuery.toJson(objectMapper)); + + rootNode.set("dataset_query", datasetQueryNode); + rootNode.put("display", display); + rootNode.putNull("description"); + rootNode.set("visualization_settings", visualizationSettings); + rootNode.put("collection_id", collectionId); + rootNode.putNull("collection_position"); + rootNode.putNull("result_metadata"); + + return rootNode; + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 75ff22f9b..11e246d78 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -8,6 +8,7 @@ import org.avni.server.domain.metabase.TableType; import org.avni.server.domain.metabase.MetabaseJoin; import org.avni.server.domain.metabase.MetabaseQuery; +import org.avni.server.domain.metabase.MetabaseRequestBody; import org.avni.server.util.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -178,33 +179,19 @@ private void createQuestionForTable(String tableName, String addressTableName, S JsonNode conditionNode = objectMapper.readTree( "[\"=\", [\"field\", " + joinFieldId1 + ", {\"base-type\": \"type/Integer\"}], [\"field\", " + joinFieldId2 + ", {\"base-type\": \"type/Integer\", \"join-alias\": \"" + tableName + "\"}]]" ); - MetabaseJoin join = new MetabaseJoin("all", tableName, tableId, conditionNode); // Change: tableId as integer + MetabaseJoin join = new MetabaseJoin("all", tableName, tableId, conditionNode); ArrayNode joinsArray = objectMapper.createArrayNode(); joinsArray.add(join.toJson(objectMapper)); - MetabaseQuery query = new MetabaseQuery(addressTableId, joinsArray); // Change: addressTableId as integer + MetabaseQuery query = new MetabaseQuery(getDatabaseId(),addressTableId, joinsArray); - ObjectNode datasetQuery = objectMapper.createObjectNode(); - datasetQuery.put("database", getDatabaseId()); - datasetQuery.put("type", "query"); - datasetQuery.set("query", query.toJson(objectMapper)); - - ObjectNode body = objectMapper.createObjectNode(); - body.put("name", "Address + " + tableName); - body.set("dataset_query", datasetQuery); - body.put("display", "table"); - body.putNull("description"); - body.set("visualization_settings", objectMapper.createObjectNode()); - body.put("collection_id", getCollectionId()); - body.putNull("collection_position"); - body.putNull("result_metadata"); + MetabaseRequestBody requestBody = new MetabaseRequestBody( + "Address + " + tableName, query, "table", null, objectMapper.createObjectNode(), getCollectionId()); - databaseRepository.postForObject(metabaseApiUrl + "/card", body, JsonNode.class); + databaseRepository.postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); } - - private void createQuestionForTable(String tableName, String schema) { int tableId = getTableIdByName(tableName, schema); From 3f8968c9ba827ff680a90ffa496a741cceedf57c Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sun, 25 Aug 2024 12:54:55 +0530 Subject: [PATCH 12/34] avniproject#762 | Shifted condition node to MetabaseJoin --- .../org/avni/server/domain/metabase/MetabaseJoin.java | 10 ++++++++-- .../avni/server/service/metabase/DatabaseService.java | 5 +---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java index 9e5b913e9..884189476 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java @@ -10,11 +10,17 @@ public class MetabaseJoin { private final int sourceTable; private final JsonNode condition; - public MetabaseJoin(String fields, String alias, int sourceTable, JsonNode condition) { + public MetabaseJoin(String fields, String alias, int sourceTable, int joinFieldId1, int joinFieldId2, String tableName, ObjectMapper objectMapper) throws Exception { this.fields = fields; this.alias = alias; this.sourceTable = sourceTable; - this.condition = condition; + this.condition = createConditionNode(joinFieldId1, joinFieldId2, tableName, objectMapper); + } + + private JsonNode createConditionNode(int joinFieldId1, int joinFieldId2, String tableName, ObjectMapper objectMapper) throws Exception { + return objectMapper.readTree( + "[\"=\", [\"field\", " + joinFieldId1 + ", {\"base-type\": \"type/Integer\"}], [\"field\", " + joinFieldId2 + ", {\"base-type\": \"type/Integer\", \"join-alias\": \"" + tableName + "\"}]]" + ); } public ObjectNode toJson(ObjectMapper objectMapper) { diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 11e246d78..8f9c2d0ca 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -176,10 +176,7 @@ private void createQuestionForTable(String tableName, String addressTableName, S int tableId = getTableIdByName(tableName); int joinFieldId2 = getFieldIdByTableNameAndFieldName(tableName, tableField); - JsonNode conditionNode = objectMapper.readTree( - "[\"=\", [\"field\", " + joinFieldId1 + ", {\"base-type\": \"type/Integer\"}], [\"field\", " + joinFieldId2 + ", {\"base-type\": \"type/Integer\", \"join-alias\": \"" + tableName + "\"}]]" - ); - MetabaseJoin join = new MetabaseJoin("all", tableName, tableId, conditionNode); + MetabaseJoin join = new MetabaseJoin("all", tableName, tableId, joinFieldId1, joinFieldId2, tableName, objectMapper); ArrayNode joinsArray = objectMapper.createArrayNode(); joinsArray.add(join.toJson(objectMapper)); From 08f3705f8670783def191b5b892a4eed92086ff1 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Wed, 28 Aug 2024 16:19:47 +0530 Subject: [PATCH 13/34] avniproject#762 | Enums Added --- .../avni/server/domain/metabase/BaseType.java | 17 ++++++++++++ .../avni/server/domain/metabase/CardType.java | 15 +++++++++++ .../server/domain/metabase/ConditionType.java | 16 ++++++++++++ .../server/domain/metabase/MetabaseJoin.java | 23 +++++++++------- .../server/domain/metabase/MetabaseQuery.java | 6 +++-- .../domain/metabase/MetabaseRequestBody.java | 9 ++++--- .../server/domain/metabase/QueryType.java | 21 +++++++++++++++ .../server/domain/metabase/SyncStatus.java | 26 +++++++++++++++++++ .../domain/metabase/VisualizationType.java | 21 +++++++++++++++ .../service/metabase/DatabaseService.java | 21 +++++++-------- .../avni/server/web/MetabaseController.java | 3 ++- 11 files changed, 151 insertions(+), 27 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/BaseType.java create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/CardType.java create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/ConditionType.java create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/QueryType.java create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/SyncStatus.java create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/VisualizationType.java diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/BaseType.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/BaseType.java new file mode 100644 index 000000000..e9404dd72 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/BaseType.java @@ -0,0 +1,17 @@ +package org.avni.server.domain.metabase; + +public enum BaseType { + INTEGER("type/Integer"), + TEXT("type/Text"), + BOOLEAN("type/Boolean"); + + private final String typeName; + + BaseType(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/CardType.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/CardType.java new file mode 100644 index 000000000..6afc67c28 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/CardType.java @@ -0,0 +1,15 @@ +package org.avni.server.domain.metabase; + +public enum CardType { + QUESTION("question"); + + private final String type; + + CardType(String type) { + this.type = type; + } + + public String getType() { + return type; + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/ConditionType.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/ConditionType.java new file mode 100644 index 000000000..0a9a1c6c1 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/ConditionType.java @@ -0,0 +1,16 @@ +package org.avni.server.domain.metabase; + +public enum ConditionType { + EQUAL("="), + NOT_EQUAL("!="); + + private final String operator; + + ConditionType(String operator) { + this.operator = operator; + } + + public String getOperator() { + return operator; + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java index 884189476..f3bde2044 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java @@ -2,25 +2,29 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; public class MetabaseJoin { - private final String fields; - private final String alias; - private final int sourceTable; - private final JsonNode condition; + private String fields; + private String alias; + private int sourceTable; + private JsonNode condition; public MetabaseJoin(String fields, String alias, int sourceTable, int joinFieldId1, int joinFieldId2, String tableName, ObjectMapper objectMapper) throws Exception { this.fields = fields; this.alias = alias; this.sourceTable = sourceTable; - this.condition = createConditionNode(joinFieldId1, joinFieldId2, tableName, objectMapper); + + this.condition = createConditionNode(joinFieldId1, joinFieldId2, tableName, BaseType.INTEGER, objectMapper); } - private JsonNode createConditionNode(int joinFieldId1, int joinFieldId2, String tableName, ObjectMapper objectMapper) throws Exception { - return objectMapper.readTree( - "[\"=\", [\"field\", " + joinFieldId1 + ", {\"base-type\": \"type/Integer\"}], [\"field\", " + joinFieldId2 + ", {\"base-type\": \"type/Integer\", \"join-alias\": \"" + tableName + "\"}]]" - ); + private JsonNode createConditionNode(int joinFieldId1, int joinFieldId2, String tableName, BaseType baseType, ObjectMapper objectMapper) throws Exception { + ArrayNode conditionNode = objectMapper.createArrayNode(); + conditionNode.add(ConditionType.EQUAL.getOperator()); + conditionNode.add(objectMapper.createArrayNode().add("field").add(joinFieldId1).add(objectMapper.createObjectNode().put("base-type", baseType.getTypeName()))); + conditionNode.add(objectMapper.createArrayNode().add("field").add(joinFieldId2).add(objectMapper.createObjectNode().put("base-type", baseType.getTypeName()).put("join-alias", tableName))); + return conditionNode; } public ObjectNode toJson(ObjectMapper objectMapper) { @@ -29,6 +33,7 @@ public ObjectNode toJson(ObjectMapper objectMapper) { joinNode.put("alias", this.alias); joinNode.put("source-table", this.sourceTable); joinNode.set("condition", this.condition); + return joinNode; } } diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java index 3e031cd4e..bdbb6a6ce 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java @@ -8,7 +8,8 @@ public class MetabaseQuery { private final int databaseId; private final int sourceTable; private final ArrayNode joins; - private final String type = "query"; + private final QueryType type = QueryType.QUERY; + public MetabaseQuery(int databaseId,int sourceTable, ArrayNode joins) { this.databaseId = databaseId; @@ -24,7 +25,8 @@ public ObjectNode toJson(ObjectMapper objectMapper) { ObjectNode queryNode = objectMapper.createObjectNode(); queryNode.put("source-table", sourceTable); queryNode.set("joins", joins); - queryNode.put("type", type); + queryNode.put("type", type.toString()); + return queryNode; } diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java index acd919abe..68b3e4572 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java @@ -7,20 +7,22 @@ public class MetabaseRequestBody { private String name; private MetabaseQuery datasetQuery; - private String display; + private VisualizationType display; private String description; private ObjectNode visualizationSettings; private int collectionId; private Integer collectionPosition; private JsonNode resultMetadata; + private CardType cardType; - public MetabaseRequestBody(String name, MetabaseQuery datasetQuery, String display, String description, ObjectNode visualizationSettings, int collectionId) { + public MetabaseRequestBody(String name, MetabaseQuery datasetQuery, VisualizationType display, String description, ObjectNode visualizationSettings, int collectionId, CardType cardType) { this.name = name; this.datasetQuery = datasetQuery; this.display = display; this.description = description; this.visualizationSettings = visualizationSettings; this.collectionId = collectionId; + this.cardType = cardType; } public ObjectNode toJson(ObjectMapper objectMapper) { @@ -33,7 +35,8 @@ public ObjectNode toJson(ObjectMapper objectMapper) { datasetQueryNode.set("query", datasetQuery.toJson(objectMapper)); rootNode.set("dataset_query", datasetQueryNode); - rootNode.put("display", display); + rootNode.put("display", display.toString()); + rootNode.put("type", cardType.getType()); rootNode.putNull("description"); rootNode.set("visualization_settings", visualizationSettings); rootNode.put("collection_id", collectionId); diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/QueryType.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/QueryType.java new file mode 100644 index 000000000..afe4c7758 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/QueryType.java @@ -0,0 +1,21 @@ +package org.avni.server.domain.metabase; + +public enum QueryType { + QUERY("query"), + DASHBOARD("dashboard"); + + private final String typeName; + + QueryType(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + @Override + public String toString() { + return typeName; + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/SyncStatus.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/SyncStatus.java new file mode 100644 index 000000000..b9b732b8d --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/SyncStatus.java @@ -0,0 +1,26 @@ +package org.avni.server.domain.metabase; + +public enum SyncStatus { + COMPLETE("complete"), + INCOMPLETE("incomplete"); + + private final String status; + + SyncStatus(String status) { + this.status = status; + } + + public String getStatus() { + return status; + } + + public static SyncStatus fromString(String status) { + for (SyncStatus s : SyncStatus.values()) { + if (s.getStatus().equalsIgnoreCase(status)) { + return s; + } + } + return null; + } + +} diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/VisualizationType.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/VisualizationType.java new file mode 100644 index 000000000..94dc0611c --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/VisualizationType.java @@ -0,0 +1,21 @@ +package org.avni.server.domain.metabase; + +public enum VisualizationType { + TABLE("table"), + CHART("chart"); + + private final String typeName; + + VisualizationType(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + @Override + public String toString() { + return typeName; + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 8f9c2d0ca..fa8f5d4ec 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -5,10 +5,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.avni.server.dao.metabase.DatabaseRepository; -import org.avni.server.domain.metabase.TableType; -import org.avni.server.domain.metabase.MetabaseJoin; -import org.avni.server.domain.metabase.MetabaseQuery; -import org.avni.server.domain.metabase.MetabaseRequestBody; +import org.avni.server.domain.metabase.*; import org.avni.server.util.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -103,9 +100,10 @@ public int getFieldIdByTableNameAndFieldName(String tableName, String fieldName) return -1; } - public String getInitialSyncStatus() { + public SyncStatus getInitialSyncStatus() { JsonNode responseBody = databaseRepository.getInitialSyncStatus(getDatabaseId()); - return responseBody.path("initial_sync_status").asText(); + String status = responseBody.path("initial_sync_status").asText(); + return SyncStatus.fromString(status); } private JsonNode getTableMetadata() { @@ -184,7 +182,7 @@ private void createQuestionForTable(String tableName, String addressTableName, S MetabaseQuery query = new MetabaseQuery(getDatabaseId(),addressTableId, joinsArray); MetabaseRequestBody requestBody = new MetabaseRequestBody( - "Address + " + tableName, query, "table", null, objectMapper.createObjectNode(), getCollectionId()); + "Address + " + tableName, query, VisualizationType.TABLE, null, objectMapper.createObjectNode(), getCollectionId(), CardType.QUESTION); databaseRepository.postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); } @@ -218,9 +216,8 @@ private void createQuestionForTable(String tableName, String schema) { } public void createQuestionsForSubjectTypes() throws Exception { - - String syncStatus = getInitialSyncStatus(); - if (!"complete".equals(syncStatus)) { + SyncStatus syncStatus = getInitialSyncStatus(); + if (syncStatus != SyncStatus.COMPLETE) { throw new RuntimeException("Database initial sync is not complete."); } @@ -232,8 +229,8 @@ public void createQuestionsForSubjectTypes() throws Exception { } public void createQuestionsForProgramsAndEncounters() throws Exception{ - String syncStatus = getInitialSyncStatus(); - if (!"complete".equals(syncStatus)) { + SyncStatus syncStatus = getInitialSyncStatus(); + if (syncStatus != SyncStatus.COMPLETE) { throw new RuntimeException("Database initial sync is not complete."); } diff --git a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java index f5624ff1f..d8ba5ff11 100644 --- a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java +++ b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java @@ -1,4 +1,5 @@ package org.avni.server.web; +import org.avni.server.domain.metabase.SyncStatus; import org.avni.server.service.metabase.DatabaseService; import org.avni.server.domain.accessControl.PrivilegeType; import org.avni.server.dao.metabase.MetabaseConnector; @@ -35,7 +36,7 @@ public void createQuestions() throws Exception{ } @GetMapping("/sync-status") - public String getSyncStatus() { + public SyncStatus getSyncStatus() { return databaseService.getInitialSyncStatus(); } From 7dba7f808493338fd5f41e5b89ceb3ac5ade39a6 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Wed, 28 Aug 2024 19:06:41 +0530 Subject: [PATCH 14/34] avniproject#762 | StringUtils shifted to S.java --- .../service/metabase/DatabaseService.java | 23 ++++--------------- .../src/main/java/org/avni/server/util/S.java | 20 ++++++++++++++++ .../org/avni/server/util/StringUtils.java | 10 -------- 3 files changed, 25 insertions(+), 28 deletions(-) delete mode 100644 avni-server-api/src/main/java/org/avni/server/util/StringUtils.java diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index fa8f5d4ec..97a30d74f 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -6,14 +6,14 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import org.avni.server.dao.metabase.DatabaseRepository; import org.avni.server.domain.metabase.*; -import org.avni.server.util.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.util.Arrays; import java.util.ArrayList; import java.util.List; -import java.util.HashMap; + +import org.avni.server.util.S; @Service public class DatabaseService { @@ -91,7 +91,7 @@ private String createRequestBodyForDataset(int sourceTableId) { public int getFieldIdByTableNameAndFieldName(String tableName, String fieldName) { JsonNode fieldsArray = databaseRepository.getFields(getDatabaseId()); - String snakeCaseTableName = StringUtils.toSnakeCase(tableName); + String snakeCaseTableName = S.toSnakeCase(tableName); for (JsonNode fieldNode : fieldsArray) { if (snakeCaseTableName.equals(fieldNode.path("table_name").asText()) && fieldName.equals(fieldNode.path("name").asText())) { return fieldNode.path("id").asInt(); @@ -112,19 +112,6 @@ private JsonNode getTableMetadata() { return databaseRepository.getDataset(requestBody); } - private String formatName(String rawName) { - String[] parts = rawName.split("_"); - StringBuilder formattedName = new StringBuilder(); - - for (String part : parts) { - formattedName.append(part.substring(0, 1).toUpperCase()) - .append(part.substring(1)) - .append(" "); - } - - return formattedName.toString().trim(); - } - public List getSubjectTypeNames() { JsonNode tableMetadata = getTableMetadata(); List subjectTypeNames = new ArrayList<>(); @@ -134,7 +121,7 @@ public List getSubjectTypeNames() { String type = row.get(2).asText(); if (Arrays.asList(TableType.INDIVIDUAL.getTypeName(), TableType.HOUSEHOLD.getTypeName(), TableType.GROUP.getTypeName(), TableType.PERSON.getTypeName()).contains(type)) { String rawName = row.get(1).asText(); - subjectTypeNames.add(formatName(rawName)); + subjectTypeNames.add(S.formatName(rawName)); } } System.out.println("The subject type names::" + subjectTypeNames); @@ -151,7 +138,7 @@ public List getProgramAndEncounterNames() { String type = row.get(2).asText(); if (Arrays.asList(TableType.PROGRAM_ENCOUNTER.getTypeName(), TableType.PROGRAM_ENROLMENT.getTypeName()).contains(type)) { String rawName = row.get(1).asText(); - programNames.add(formatName(rawName)); + programNames.add(S.formatName(rawName)); } } System.out.println("The program and encounter::" + programNames); diff --git a/avni-server-api/src/main/java/org/avni/server/util/S.java b/avni-server-api/src/main/java/org/avni/server/util/S.java index 7d83bab0b..8022d10df 100644 --- a/avni-server-api/src/main/java/org/avni/server/util/S.java +++ b/avni-server-api/src/main/java/org/avni/server/util/S.java @@ -29,4 +29,24 @@ public static String[] splitMultiSelectAnswer(String answerValue) { .toArray(String[]::new); } + public static String toSnakeCase(String input) { + if (input == null) { + return null; + } + return input.trim().replaceAll(" +", "_").toLowerCase(); + } + + public static String formatName(String rawName) { + String[] parts = rawName.split("_"); + StringBuilder formattedName = new StringBuilder(); + + for (String part : parts) { + formattedName.append(part.substring(0, 1).toUpperCase()) + .append(part.substring(1)) + .append(" "); + } + + return formattedName.toString().trim(); + } + } diff --git a/avni-server-api/src/main/java/org/avni/server/util/StringUtils.java b/avni-server-api/src/main/java/org/avni/server/util/StringUtils.java deleted file mode 100644 index 98e592c21..000000000 --- a/avni-server-api/src/main/java/org/avni/server/util/StringUtils.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.avni.server.util; - -public class StringUtils { - public static String toSnakeCase(String input) { - if (input == null) { - return null; - } - return input.trim().replaceAll(" +", "_").toLowerCase(); - } -} From 6b4edfb809c39ba59684816f2295bd3c084151c7 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sat, 31 Aug 2024 00:52:00 +0530 Subject: [PATCH 15/34] avniproject#762 | Created a proper object for getDatabaseDetails --- .../dao/metabase/DatabaseRepository.java | 13 +++- .../domain/metabase/MetabaseDatabaseInfo.java | 70 ++++++++++++++++++ .../server/domain/metabase/TableDetails.java | 72 +++++++++++++++++++ .../service/metabase/DatabaseService.java | 45 ++++++------ 4 files changed, 176 insertions(+), 24 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseDatabaseInfo.java create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java index ac7755e7b..c01236900 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java @@ -2,13 +2,17 @@ import com.fasterxml.jackson.databind.JsonNode; import org.avni.server.domain.metabase.Database; +import org.avni.server.domain.metabase.MetabaseDatabaseInfo; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.stereotype.Repository; +import com.fasterxml.jackson.databind.ObjectMapper; @Repository public class DatabaseRepository extends MetabaseConnector { + private final ObjectMapper objectMapper; public DatabaseRepository(RestTemplateBuilder restTemplateBuilder) { super(restTemplateBuilder); + this.objectMapper = new ObjectMapper(); } public Database save(Database database) { @@ -18,9 +22,14 @@ public Database save(Database database) { return database; } - public JsonNode getDatabaseDetails(int databaseId) { + public MetabaseDatabaseInfo getDatabaseDetails(int databaseId) { String url = metabaseApiUrl + "/database/" + databaseId + "?include=tables"; - return getForObject(url, JsonNode.class); + String jsonResponse = getForObject(url, String.class); + try { + return objectMapper.readValue(jsonResponse, MetabaseDatabaseInfo.class); + } catch (Exception e) { + throw new RuntimeException("Failed to parse database details", e); + } } public JsonNode getFields(int databaseId) { diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseDatabaseInfo.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseDatabaseInfo.java new file mode 100644 index 000000000..28ef8f5c3 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseDatabaseInfo.java @@ -0,0 +1,70 @@ +package org.avni.server.domain.metabase; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class MetabaseDatabaseInfo { + + @JsonProperty("description") + private String description; + + @JsonProperty("features") + private List features; + + @JsonProperty("cache_field_values_schedule") + private String cacheFieldValuesSchedule; + + @JsonProperty("timezone") + private String timezone; + + @JsonProperty("auto_run_queries") + private boolean autoRunQueries; + + @JsonProperty("metadata_sync_schedule") + private String metadataSyncSchedule; + + @JsonProperty("name") + private String name; + + @JsonProperty("tables") + private List tables; + + public String getDescription() { + return description; + } + + public List getFeatures() { + return features; + } + + public String getCacheFieldValuesSchedule() { + return cacheFieldValuesSchedule; + } + + public String getTimezone() { + return timezone; + } + + public void setTimezone(String timezone) { + this.timezone = timezone; + } + + public boolean isAutoRunQueries() { + return autoRunQueries; + } + + public String getMetadataSyncSchedule() { + return metadataSyncSchedule; + } + + public String getName() { + return name; + } + + public List getTables() { + return tables; + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java new file mode 100644 index 000000000..d57694aeb --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java @@ -0,0 +1,72 @@ +package org.avni.server.domain.metabase; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class TableDetails { + + @JsonProperty("description") + private String description; + + @JsonProperty("entity_type") + private String entityType; + + @JsonProperty("schema") + private String schema; + + @JsonProperty("name") + private String name; + + @JsonProperty("active") + private boolean active; + + @JsonProperty("id") + private int id; + + @JsonProperty("db_id") + private int dbId; + + @JsonProperty("initial_sync_status") + private String initialSyncStatus; + + @JsonProperty("display_name") + private String displayName; + + public String getDescription() { + return description; + } + + public String getEntityType() { + return entityType; + } + + public String getSchema() { + return schema; + } + + public String getName() { + return name; + } + + public boolean isActive() { + return active; + } + + public int getId() { + return id; + } + + public int getDbId() { + return dbId; + } + + public String getInitialSyncStatus() { + return initialSyncStatus; + } + + public String getDisplayName() { + return displayName; + } + +} diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 97a30d74f..6ebbbf8ae 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -12,6 +12,7 @@ import java.util.Arrays; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import org.avni.server.util.S; @@ -52,39 +53,36 @@ private int getCollectionId() { } public int getTableIdByName(String tableName) { - JsonNode rootNode = databaseRepository.getDatabaseDetails(getDatabaseId()); - JsonNode tablesArray = rootNode.path("tables"); - for (JsonNode tableNode : tablesArray) { - if (tableName.equals(tableNode.path("display_name").asText())) { - return tableNode.path("id").asInt(); + MetabaseDatabaseInfo databaseInfo = databaseRepository.getDatabaseDetails(getDatabaseId()); + List tables = databaseInfo.getTables(); + + for (TableDetails table : tables) { + if (tableName.equals(table.getDisplayName())) { + return table.getId(); } } return -1; } public int getTableIdByName(String tableName, String schema) { - JsonNode rootNode = databaseRepository.getDatabaseDetails( - getDatabaseId() - ); - JsonNode tablesArray = rootNode.path("tables"); + MetabaseDatabaseInfo databaseInfo = databaseRepository.getDatabaseDetails(getDatabaseId()); + List tables = databaseInfo.getTables(); - for (JsonNode tableNode : tablesArray) { - String tableSchema = tableNode.path("schema").asText(); + for (TableDetails table : tables) { + String tableSchema = table.getSchema(); boolean schemaMatches = schema.equals("public") - ? "public".equals(tableSchema) - : !"public".equals(tableSchema); - - if ( - tableName.equals(tableNode.path("display_name").asText()) && - schemaMatches - ) { - return tableNode.path("id").asInt(); + ? "public".equals(tableSchema) + : !"public".equals(tableSchema); + + if (tableName.equals(table.getDisplayName()) && schemaMatches) { + return table.getId(); } } return -1; } + private String createRequestBodyForDataset(int sourceTableId) { return "{\"database\":" + getDatabaseId() + ",\"query\":{\"source-table\":" + sourceTableId + "},\"type\":\"query\",\"parameters\":[]}"; } @@ -215,7 +213,7 @@ public void createQuestionsForSubjectTypes() throws Exception { } } - public void createQuestionsForProgramsAndEncounters() throws Exception{ + public void createQuestionsForProgramsAndEncounters() throws Exception { SyncStatus syncStatus = getInitialSyncStatus(); if (syncStatus != SyncStatus.COMPLETE) { throw new RuntimeException("Database initial sync is not complete."); @@ -223,8 +221,11 @@ public void createQuestionsForProgramsAndEncounters() throws Exception{ List programNames = getProgramAndEncounterNames(); - JsonNode databaseDetails = databaseRepository.getDatabaseDetails(getDatabaseId()); - List allTableNames = extractTableNames(databaseDetails); + MetabaseDatabaseInfo databaseInfo = databaseRepository.getDatabaseDetails(getDatabaseId()); + List allTableNames = databaseInfo.getTables() + .stream() + .map(TableDetails::getDisplayName) + .collect(Collectors.toList()); for (String programName : programNames) { for (String tableName : allTableNames) { From b4562323828065af20197e97daae5277afb01aca Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sat, 31 Aug 2024 11:10:37 +0530 Subject: [PATCH 16/34] avniproject#762 | Proper object for field details created --- .../dao/metabase/DatabaseRepository.java | 13 +++- .../server/domain/metabase/FieldDetails.java | 59 +++++++++++++++++++ .../service/metabase/DatabaseService.java | 9 +-- 3 files changed, 75 insertions(+), 6 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldDetails.java diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java index c01236900..395f26180 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java @@ -1,12 +1,16 @@ package org.avni.server.dao.metabase; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import org.avni.server.domain.metabase.Database; +import org.avni.server.domain.metabase.FieldDetails; import org.avni.server.domain.metabase.MetabaseDatabaseInfo; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.stereotype.Repository; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.List; + @Repository public class DatabaseRepository extends MetabaseConnector { private final ObjectMapper objectMapper; @@ -32,9 +36,14 @@ public MetabaseDatabaseInfo getDatabaseDetails(int databaseId) { } } - public JsonNode getFields(int databaseId) { + public List getFields(int databaseId) { String url = metabaseApiUrl + "/database/" + databaseId + "/fields"; - return getForObject(url, JsonNode.class); + String jsonResponse = getForObject(url, String.class); + try { + return objectMapper.readValue(jsonResponse, new TypeReference>() {}); + } catch (Exception e) { + throw new RuntimeException("Failed to parse fields", e); + } } public JsonNode getInitialSyncStatus(int databaseId) { diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldDetails.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldDetails.java new file mode 100644 index 000000000..94b8e5827 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldDetails.java @@ -0,0 +1,59 @@ +package org.avni.server.domain.metabase; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class FieldDetails { + + @JsonProperty("id") + private int id; + + @JsonProperty("name") + private String name; + + @JsonProperty("display_name") + private String displayName; + + @JsonProperty("base_type") + private String baseType; + + @JsonProperty("semantic_type") + private String semanticType; + + @JsonProperty("table_name") + private String tableName; + + @JsonProperty("schema") + private String schema; + + public int getId() { + return id; + } + + public String getName() { + return name; + } + + public String getDisplayName() { + return displayName; + } + + public String getBaseType() { + return baseType; + } + + public String getSemanticType() { + return semanticType; + } + + public String getTableName() { + return tableName; + } + + + public String getSchema() { + return schema; + } + +} diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 6ebbbf8ae..e478394da 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -88,11 +88,12 @@ private String createRequestBodyForDataset(int sourceTableId) { } public int getFieldIdByTableNameAndFieldName(String tableName, String fieldName) { - JsonNode fieldsArray = databaseRepository.getFields(getDatabaseId()); + List fieldsList = databaseRepository.getFields(getDatabaseId()); String snakeCaseTableName = S.toSnakeCase(tableName); - for (JsonNode fieldNode : fieldsArray) { - if (snakeCaseTableName.equals(fieldNode.path("table_name").asText()) && fieldName.equals(fieldNode.path("name").asText())) { - return fieldNode.path("id").asInt(); + + for (FieldDetails field : fieldsList) { + if (snakeCaseTableName.equals(field.getTableName()) && fieldName.equals(field.getName())) { + return field.getId(); } } return -1; From e21282de5bfa4c0aa1ca0931397515b6c78dbf15 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sat, 31 Aug 2024 15:53:47 +0530 Subject: [PATCH 17/34] avnirproject#762 | Created DatabaseSyncStatus for getInitialSyncStatus --- .../dao/metabase/DatabaseRepository.java | 10 +++- .../domain/metabase/DatabaseSyncStatus.java | 49 +++++++++++++++++++ .../service/metabase/DatabaseService.java | 5 +- 3 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/DatabaseSyncStatus.java diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java index 395f26180..ef2d70690 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import org.avni.server.domain.metabase.Database; +import org.avni.server.domain.metabase.DatabaseSyncStatus; import org.avni.server.domain.metabase.FieldDetails; import org.avni.server.domain.metabase.MetabaseDatabaseInfo; import org.springframework.boot.web.client.RestTemplateBuilder; @@ -46,9 +47,14 @@ public List getFields(int databaseId) { } } - public JsonNode getInitialSyncStatus(int databaseId) { + public DatabaseSyncStatus getInitialSyncStatus(int databaseId) { String url = metabaseApiUrl + "/database/" + databaseId; - return getForObject(url, JsonNode.class); + String jsonResponse = getForObject(url, String.class); + try { + return objectMapper.readValue(jsonResponse, DatabaseSyncStatus.class); + } catch (Exception e) { + throw new RuntimeException("Failed to parse sync status", e); + } } public JsonNode getDataset(String requestBody) { diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatabaseSyncStatus.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatabaseSyncStatus.java new file mode 100644 index 000000000..832efd201 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatabaseSyncStatus.java @@ -0,0 +1,49 @@ +package org.avni.server.domain.metabase; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class DatabaseSyncStatus { + + @JsonProperty("initial_sync_status") + private String initialSyncStatus; + + @JsonProperty("name") + private String name; + + @JsonProperty("is_full_sync") + private boolean isFullSync; + + @JsonProperty("id") + private int id; + + @JsonProperty("engine") + private String engine; + + + public String getInitialSyncStatus() { + return initialSyncStatus; + } + + + public String getName() { + return name; + } + + + public boolean isFullSync() { + return isFullSync; + } + + + public int getId() { + return id; + } + + + public String getEngine() { + return engine; + } + +} diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index e478394da..e5c7c0ec0 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -100,11 +100,12 @@ public int getFieldIdByTableNameAndFieldName(String tableName, String fieldName) } public SyncStatus getInitialSyncStatus() { - JsonNode responseBody = databaseRepository.getInitialSyncStatus(getDatabaseId()); - String status = responseBody.path("initial_sync_status").asText(); + DatabaseSyncStatus databaseSyncStatus = databaseRepository.getInitialSyncStatus(getDatabaseId()); + String status = databaseSyncStatus.getInitialSyncStatus(); return SyncStatus.fromString(status); } + private JsonNode getTableMetadata() { int tableMetadataId = getTableIdByName("Table Metadata"); String requestBody = createRequestBodyForDataset(tableMetadataId); From 935706b100c52d6bff2fbc2d799ee5f2beed5968 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sat, 31 Aug 2024 16:25:39 +0530 Subject: [PATCH 18/34] avniproject#762 | DatasetResponse added --- .../dao/metabase/DatabaseRepository.java | 14 ++++--- .../domain/metabase/DatasetResponse.java | 35 ++++++++++++++++ .../service/metabase/DatabaseService.java | 42 ++++++++++++------- 3 files changed, 70 insertions(+), 21 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/DatasetResponse.java diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java index ef2d70690..d715d0088 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java @@ -2,10 +2,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; -import org.avni.server.domain.metabase.Database; -import org.avni.server.domain.metabase.DatabaseSyncStatus; -import org.avni.server.domain.metabase.FieldDetails; -import org.avni.server.domain.metabase.MetabaseDatabaseInfo; +import org.avni.server.domain.metabase.*; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.stereotype.Repository; import com.fasterxml.jackson.databind.ObjectMapper; @@ -57,8 +54,13 @@ public DatabaseSyncStatus getInitialSyncStatus(int databaseId) { } } - public JsonNode getDataset(String requestBody) { + public DatasetResponse getDataset(String requestBody) { String url = metabaseApiUrl + "/dataset"; - return postForObject(url, requestBody, JsonNode.class); + String jsonResponse = postForObject(url, requestBody, String.class); + try { + return objectMapper.readValue(jsonResponse, DatasetResponse.class); + } catch (Exception e) { + throw new RuntimeException("Failed to parse dataset response", e); + } } } diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatasetResponse.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatasetResponse.java new file mode 100644 index 000000000..7bf33ba90 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatasetResponse.java @@ -0,0 +1,35 @@ +package org.avni.server.domain.metabase; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class DatasetResponse { + + @JsonProperty("data") + private Data data; + + public Data getData() { + return data; + } + + public void setData(Data data) { + this.data = data; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Data { + + @JsonProperty("rows") + private List> rows; + + public List> getRows() { + return rows; + } + + public void setRows(List> rows) { + this.rows = rows; + } + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index e5c7c0ec0..cb801687d 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -106,46 +106,58 @@ public SyncStatus getInitialSyncStatus() { } - private JsonNode getTableMetadata() { + private DatasetResponse getTableMetadata() { int tableMetadataId = getTableIdByName("Table Metadata"); String requestBody = createRequestBodyForDataset(tableMetadataId); return databaseRepository.getDataset(requestBody); } public List getSubjectTypeNames() { - JsonNode tableMetadata = getTableMetadata(); + DatasetResponse tableMetadata = getTableMetadata(); List subjectTypeNames = new ArrayList<>(); - JsonNode rows = tableMetadata.path("data").path("rows"); - for (JsonNode row : rows) { - String type = row.get(2).asText(); - if (Arrays.asList(TableType.INDIVIDUAL.getTypeName(), TableType.HOUSEHOLD.getTypeName(), TableType.GROUP.getTypeName(), TableType.PERSON.getTypeName()).contains(type)) { - String rawName = row.get(1).asText(); + List> rows = tableMetadata.getData().getRows(); + for (List row : rows) { + String type = row.get(2); + if (Arrays.asList( + TableType.INDIVIDUAL.getTypeName(), + TableType.HOUSEHOLD.getTypeName(), + TableType.GROUP.getTypeName(), + TableType.PERSON.getTypeName() + ).contains(type)) { + String rawName = row.get(1); subjectTypeNames.add(S.formatName(rawName)); } } - System.out.println("The subject type names::" + subjectTypeNames); + + System.out.println("The subject type names: " + subjectTypeNames); return subjectTypeNames; } + public List getProgramAndEncounterNames() { - JsonNode tableMetadata = getTableMetadata(); + DatasetResponse tableMetadata = getTableMetadata(); List programNames = new ArrayList<>(); - JsonNode rows = tableMetadata.path("data").path("rows"); - for (JsonNode row : rows) { - String type = row.get(2).asText(); - if (Arrays.asList(TableType.PROGRAM_ENCOUNTER.getTypeName(), TableType.PROGRAM_ENROLMENT.getTypeName()).contains(type)) { - String rawName = row.get(1).asText(); + List> rows = tableMetadata.getData().getRows(); + for (List row : rows) { + String type = row.get(2); + if (Arrays.asList( + TableType.PROGRAM_ENCOUNTER.getTypeName(), + TableType.PROGRAM_ENROLMENT.getTypeName() + ).contains(type)) { + String rawName = row.get(1); programNames.add(S.formatName(rawName)); } } - System.out.println("The program and encounter::" + programNames); + + System.out.println("The program and encounter names: " + programNames); return programNames; } + private List extractTableNames(JsonNode databaseDetails) { List tableNames = new ArrayList<>(); JsonNode tablesArray = databaseDetails.path("tables"); From 5f58deb26d21214279860b6debe89437c18935d6 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sat, 31 Aug 2024 17:40:29 +0530 Subject: [PATCH 19/34] avniproject#762 | Added AddressQuestionCreationService and QuestionCreationService --- .../AddressQuestionCreationService.java | 74 +++++++++++++++++++ .../service/metabase/DatabaseService.java | 12 +-- .../metabase/QuestionCreationService.java | 6 ++ 3 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java create mode 100644 avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java new file mode 100644 index 000000000..4f061266e --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java @@ -0,0 +1,74 @@ +package org.avni.server.service.metabase; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.avni.server.dao.metabase.DatabaseRepository; +import org.avni.server.domain.metabase.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; + +@Service +public class AddressQuestionCreationService implements QuestionCreationService { + + private final DatabaseService databaseService; + private final DatabaseRepository databaseRepository; + private final ObjectMapper objectMapper; + private final String metabaseApiUrl; + + @Autowired + public AddressQuestionCreationService(@Lazy DatabaseService databaseService, DatabaseRepository databaseRepository, ObjectMapper objectMapper, @Value("${metabase.api.url}") String metabaseApiUrl) { + this.databaseService = databaseService; + this.databaseRepository = databaseRepository; + this.objectMapper = objectMapper; + this.metabaseApiUrl = metabaseApiUrl; + } + + @Override + public void createQuestionForTable(String tableName, String addressTableName, String addressField, String tableField) throws Exception { + int addressTableId = databaseService.getTableIdByName(addressTableName); + int joinFieldId1 = databaseService.getFieldIdByTableNameAndFieldName(addressTableName, addressField); + int tableId = databaseService.getTableIdByName(tableName); + int joinFieldId2 = databaseService.getFieldIdByTableNameAndFieldName(tableName, tableField); + + MetabaseJoin join = new MetabaseJoin("all", tableName, tableId, joinFieldId1, joinFieldId2, tableName, objectMapper); + + ArrayNode joinsArray = objectMapper.createArrayNode(); + joinsArray.add(join.toJson(objectMapper)); + + MetabaseQuery query = new MetabaseQuery(databaseService.getDatabaseId(), addressTableId, joinsArray); + + MetabaseRequestBody requestBody = new MetabaseRequestBody( + "Address + " + tableName, query, VisualizationType.TABLE, null, objectMapper.createObjectNode(), databaseService.getCollectionId(), CardType.QUESTION); + + databaseRepository.postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); + } + + @Override + public void createQuestionForTable(String tableName, String schema) { + int tableId = databaseService.getTableIdByName(tableName, schema); + + ObjectNode datasetQuery = objectMapper.createObjectNode(); + datasetQuery.put("database", databaseService.getDatabaseId()); + datasetQuery.put("type", "query"); + + ObjectNode query = objectMapper.createObjectNode(); + query.put("source-table", tableId); + datasetQuery.set("query", query); + + ObjectNode body = objectMapper.createObjectNode(); + body.put("name", tableName); + body.set("dataset_query", datasetQuery); + body.put("display", "table"); + body.putNull("description"); + body.set("visualization_settings", objectMapper.createObjectNode()); + body.put("collection_id", databaseService.getCollectionId()); + body.putNull("collection_position"); + body.putNull("result_metadata"); + + databaseRepository.postForObject(metabaseApiUrl + "/card", body, JsonNode.class); + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index cb801687d..100626eb5 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -22,6 +22,7 @@ public class DatabaseService { private final DatabaseRepository databaseRepository; private final ObjectMapper objectMapper; private final MetabaseService metabaseService; + private final AddressQuestionCreationService addressQuestionCreationService; private Integer databaseId; private Integer collectionId; @@ -32,20 +33,21 @@ public class DatabaseService { private String apiKey; @Autowired - public DatabaseService(DatabaseRepository databaseRepository, ObjectMapper objectMapper, MetabaseService metabaseService) { + public DatabaseService(DatabaseRepository databaseRepository, ObjectMapper objectMapper, MetabaseService metabaseService, AddressQuestionCreationService addressQuestionCreationService) { this.databaseRepository = databaseRepository; this.objectMapper = objectMapper; this.metabaseService = metabaseService; + this.addressQuestionCreationService = addressQuestionCreationService; } - private int getDatabaseId() { + public int getDatabaseId() { if (databaseId == null) { databaseId = metabaseService.getGlobalDatabaseId(); } return databaseId; } - private int getCollectionId() { + public int getCollectionId() { if (collectionId == null) { collectionId = metabaseService.getGlobalCollectionId(); } @@ -223,7 +225,7 @@ public void createQuestionsForSubjectTypes() throws Exception { List subjectTypeNames = getSubjectTypeNames(); for (String subjectTypeName : subjectTypeNames) { - createQuestionForTable(subjectTypeName, "Address", "id", "address_id"); + addressQuestionCreationService.createQuestionForTable(subjectTypeName, "Address", "id", "address_id"); } } @@ -244,7 +246,7 @@ public void createQuestionsForProgramsAndEncounters() throws Exception { for (String programName : programNames) { for (String tableName : allTableNames) { if (tableName.equalsIgnoreCase(programName)) { - createQuestionForTable(tableName, "Address", "id", "address_id"); + addressQuestionCreationService.createQuestionForTable(tableName, "Address", "id", "address_id"); } } } diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java new file mode 100644 index 000000000..58babec9f --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java @@ -0,0 +1,6 @@ +package org.avni.server.service.metabase; + +public interface QuestionCreationService { + void createQuestionForTable(String tableName, String addressTableName, String addressField, String tableField) throws Exception; + void createQuestionForTable(String tableName, String schema); +} From 1aa42d10be6559e055a5df5ae6c3b98a4785372e Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Wed, 4 Sep 2024 23:12:10 +0530 Subject: [PATCH 20/34] Removed db migration --- .../V1_339_1__GroupPrivilegeConstraint.sql | 35 ------------------- 1 file changed, 35 deletions(-) delete mode 100644 avni-server-api/src/main/resources/db/migration/V1_339_1__GroupPrivilegeConstraint.sql diff --git a/avni-server-api/src/main/resources/db/migration/V1_339_1__GroupPrivilegeConstraint.sql b/avni-server-api/src/main/resources/db/migration/V1_339_1__GroupPrivilegeConstraint.sql deleted file mode 100644 index 2223e75af..000000000 --- a/avni-server-api/src/main/resources/db/migration/V1_339_1__GroupPrivilegeConstraint.sql +++ /dev/null @@ -1,35 +0,0 @@ -alter table group_privilege add column if not exists impl_version int not null default 1; - -update group_privilege -set impl_version = 0 -where is_voided = true; - -create or replace function check_group_privilege_uniqueness(groupPrivilegeId int, groupId int, privilegeId int, subjectTypeId int, programId int, programEncounterTypeId int, encounterTypeId int, checklistDetailId int, implVersion int) returns boolean - language plpgsql -as -$$ -declare -begin - if exists (select gp.* - from public.group_privilege gp - where gp.group_id = groupId - and gp.privilege_id = privilegeId - and (gp.subject_type_id = subjectTypeId or (gp.subject_type_id is null and subjectTypeId is null)) - and (gp.program_id = programId or (gp.program_id is null and programId is null)) - and (gp.program_encounter_type_id = programEncounterTypeId or (gp.program_encounter_type_id is null and programEncounterTypeId is null)) - and (gp.encounter_type_id = encounterTypeId or (gp.encounter_type_id is null and encounterTypeId is null)) - and (gp.checklist_detail_id = checklistDetailId or (gp.checklist_detail_id is null and checklistDetailId is null)) - and gp.id <> groupPrivilegeId - and gp.impl_version = 1 - and implVersion = 1 - ) then - raise 'Duplicate group privilege exists for: id: %, group_id: %, privilege_id: % subject_type_id: %, program_id: %, program_encounter_type_id: %, encounter_type_id: %, checklist_detail_id: %', groupPrivilegeId, groupId, privilegeId, subjectTypeId, programId, programEncounterTypeId, encounterTypeId, checklistDetailId; - end if; - - return true; -end -$$; - -alter table group_privilege - add constraint check_group_privilege_unique - check (check_group_privilege_uniqueness(id, group_id, privilege_id, subject_type_id, program_id, program_encounter_type_id, encounter_type_id, checklist_detail_id, impl_version)); \ No newline at end of file From 267eb694093f3c84266e57602abaa1d9f1b8f3ae Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Wed, 4 Sep 2024 23:12:47 +0530 Subject: [PATCH 21/34] removed db migration --- .../main/resources/db/migration/V1_339_2__OrganisationStatus.sql | 1 - 1 file changed, 1 deletion(-) delete mode 100644 avni-server-api/src/main/resources/db/migration/V1_339_2__OrganisationStatus.sql diff --git a/avni-server-api/src/main/resources/db/migration/V1_339_2__OrganisationStatus.sql b/avni-server-api/src/main/resources/db/migration/V1_339_2__OrganisationStatus.sql deleted file mode 100644 index 223dedd65..000000000 --- a/avni-server-api/src/main/resources/db/migration/V1_339_2__OrganisationStatus.sql +++ /dev/null @@ -1 +0,0 @@ -alter table organisation add column status varchar(255) not null default 'Live'; From 4d8697cb69e4684624dd562b7c05dcd2c7871f7b Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Wed, 4 Sep 2024 23:13:09 +0530 Subject: [PATCH 22/34] removed db migration --- .../db/migration/V1_342__UpdateStandardCardTypeNames.sql | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 avni-server-api/src/main/resources/db/migration/V1_342__UpdateStandardCardTypeNames.sql diff --git a/avni-server-api/src/main/resources/db/migration/V1_342__UpdateStandardCardTypeNames.sql b/avni-server-api/src/main/resources/db/migration/V1_342__UpdateStandardCardTypeNames.sql deleted file mode 100644 index f2ead5142..000000000 --- a/avni-server-api/src/main/resources/db/migration/V1_342__UpdateStandardCardTypeNames.sql +++ /dev/null @@ -1,9 +0,0 @@ -UPDATE standard_report_card_type -SET name = 'Recent registrations', description = 'Recent registrations', last_modified_date_time = current_timestamp - WHERE name = 'Last 24 hours registrations'; -UPDATE standard_report_card_type -SET name = 'Recent enrolments', description = 'Recent enrolments', last_modified_date_time = current_timestamp - WHERE name = 'Last 24 hours enrolments'; -UPDATE standard_report_card_type -SET name = 'Recent visits', description = 'Recent visits', last_modified_date_time = current_timestamp - WHERE name = 'Last 24 hours visits'; From 910ffb57b5135763379256257015464cb85bcc19 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Fri, 6 Sep 2024 04:37:15 +0530 Subject: [PATCH 23/34] avniproject#762 | MCI and MQB added and others modified --- .../dao/metabase/DatabaseRepository.java | 92 ++++++++++- .../metabase/MetabaseCollectionInfo.java | 37 +++++ .../server/domain/metabase/MetabaseQuery.java | 19 +-- .../domain/metabase/MetabaseQueryBuilder.java | 41 +++++ .../server/domain/metabase/TableDetails.java | 3 + .../AddressQuestionCreationService.java | 40 +---- .../service/metabase/DatabaseService.java | 146 +++++++++--------- .../service/metabase/MetabaseService.java | 12 +- .../avni/server/web/MetabaseController.java | 4 +- 9 files changed, 267 insertions(+), 127 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseCollectionInfo.java create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java index d715d0088..45a2f511f 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java @@ -2,7 +2,9 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import org.avni.server.domain.metabase.*; +import org.avni.server.util.S; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.stereotype.Repository; import com.fasterxml.jackson.databind.ObjectMapper; @@ -24,9 +26,73 @@ public Database save(Database database) { return database; } - public MetabaseDatabaseInfo getDatabaseDetails(int databaseId) { - String url = metabaseApiUrl + "/database/" + databaseId + "?include=tables"; + public Database getDatabaseByName(String databaseName) { + String url = metabaseApiUrl + "/database"; + String jsonResponse = getForObject(url, String.class); + try { + List databases = objectMapper.readValue(jsonResponse, new TypeReference>() {}); + return databases.stream() + .filter(db -> db.getName().equals(databaseName)) + .findFirst() + .orElseThrow(() -> new RuntimeException("Database with name " + databaseName + " not found.")); + } catch (Exception e) { + throw new RuntimeException("Failed to retrieve database", e); + } + } + + public MetabaseCollectionInfo getCollectionByName(String collectionName) { + String url = metabaseApiUrl + "/collection"; + String jsonResponse = getForObject(url, String.class); + + try { + List collections = objectMapper.readValue(jsonResponse, new TypeReference>() {}); + return collections.stream() + .filter(coll -> coll.getName().equals(collectionName)) + .findFirst() + .orElseThrow(() -> new RuntimeException("Collection with name " + collectionName + " not found.")); + } catch (Exception e) { + throw new RuntimeException("Failed to retrieve collection", e); + } + } + + public void createQuestionForTable(Database database, TableDetails tableDetails, String addressTableName, String addressField, String tableField) throws Exception { + TableDetails addressTableDetails = getTableDetailsByDisplayName(database, addressTableName); + FieldDetails joinField1 = getFieldDetailsByName(database, addressTableName, addressField); + FieldDetails joinField2 = getFieldDetailsByName(database, tableDetails.getName(), tableField); + + ArrayNode joinsArray = objectMapper.createArrayNode(); + MetabaseQuery query = new MetabaseQueryBuilder(database, joinsArray) + .forTable(tableDetails) + .joinWith(addressTableDetails, joinField1, joinField2) + .build(); + + MetabaseRequestBody requestBody = new MetabaseRequestBody( + "Address + " + tableDetails.getDisplayName(), + query, + VisualizationType.TABLE, + null, + objectMapper.createObjectNode(), + getCollectionByName(database.getName()).getId(), + CardType.QUESTION + ); + + postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); + } + + public FieldDetails getFieldDetailsByName(Database database, String tableName, String fieldName) { + List fieldsList = getFields(database); + String snakeCaseTableName = S.toSnakeCase(tableName); + + return fieldsList.stream() + .filter(field -> snakeCaseTableName.equals(field.getTableName()) && fieldName.equals(field.getName())) + .findFirst() + .orElseThrow(() -> new RuntimeException("Field " + fieldName + " not found in table " + tableName)); + } + + public MetabaseDatabaseInfo getDatabaseDetails(Database database) { + String url = metabaseApiUrl + "/database/" + database.getId() + "?include=tables"; String jsonResponse = getForObject(url, String.class); + try { return objectMapper.readValue(jsonResponse, MetabaseDatabaseInfo.class); } catch (Exception e) { @@ -34,9 +100,10 @@ public MetabaseDatabaseInfo getDatabaseDetails(int databaseId) { } } - public List getFields(int databaseId) { - String url = metabaseApiUrl + "/database/" + databaseId + "/fields"; + public List getFields(Database database) { + String url = metabaseApiUrl + "/database/" + database.getId() + "/fields"; String jsonResponse = getForObject(url, String.class); + try { return objectMapper.readValue(jsonResponse, new TypeReference>() {}); } catch (Exception e) { @@ -44,6 +111,14 @@ public List getFields(int databaseId) { } } + public TableDetails getTableDetailsByDisplayName(Database database, String tableName) { + MetabaseDatabaseInfo databaseInfo = getDatabaseDetails(database); + return databaseInfo.getTables().stream() + .filter(tableDetail -> tableDetail.nameMatches(tableName)) + .findFirst() + .orElseThrow(() -> new RuntimeException("Table with name " + tableName + " not found.")); + } + public DatabaseSyncStatus getInitialSyncStatus(int databaseId) { String url = metabaseApiUrl + "/database/" + databaseId; String jsonResponse = getForObject(url, String.class); @@ -63,4 +138,13 @@ public DatasetResponse getDataset(String requestBody) { throw new RuntimeException("Failed to parse dataset response", e); } } + + public DatasetResponse findAll(TableDetails table, Database database) { + String requestBody = createRequestBodyForDataset(database, table); + return getDataset(requestBody); + } + + private String createRequestBodyForDataset(Database database,TableDetails table) { + return "{\"database\":" + database.getId() + ",\"query\":{\"source-table\":" + table.getId() + "},\"type\":\"query\",\"parameters\":[]}"; + } } diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseCollectionInfo.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseCollectionInfo.java new file mode 100644 index 000000000..e67046408 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseCollectionInfo.java @@ -0,0 +1,37 @@ +package org.avni.server.domain.metabase; + +public class MetabaseCollectionInfo { + private String name; + private int id; + private boolean isPersonal; + + public MetabaseCollectionInfo(String name, int id, boolean isPersonal) { + this.name = name; + this.id = id; + this.isPersonal = isPersonal; + } + + public String getName() { + return name; + } + + public int getId() { + return id; + } + + public boolean isPersonal() { + return isPersonal; + } + + public void setName(String name) { + this.name = name; + } + + public void setId(int id) { + this.id = id; + } + + public void setPersonal(boolean personal) { + isPersonal = personal; + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java index bdbb6a6ce..5b2db827c 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java @@ -5,29 +5,24 @@ import com.fasterxml.jackson.databind.node.ObjectNode; public class MetabaseQuery { - private final int databaseId; - private final int sourceTable; + private final Database database; private final ArrayNode joins; - private final QueryType type = QueryType.QUERY; - - public MetabaseQuery(int databaseId,int sourceTable, ArrayNode joins) { - this.databaseId = databaseId; - this.sourceTable = sourceTable; + public MetabaseQuery(Database database, ArrayNode joins) { + this.database = database; this.joins = joins; } public int getDatabaseId() { - return databaseId; + return database.getId(); } + public ObjectNode toJson(ObjectMapper objectMapper) { ObjectNode queryNode = objectMapper.createObjectNode(); - queryNode.put("source-table", sourceTable); + queryNode.put("database", database.getId()); queryNode.set("joins", joins); - queryNode.put("type", type.toString()); - - + queryNode.put("type", "query"); return queryNode; } } diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java new file mode 100644 index 000000000..28460c685 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java @@ -0,0 +1,41 @@ +// to be completed +package org.avni.server.domain.metabase; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class MetabaseQueryBuilder { + private final Database database; + private ArrayNode joins; + + public MetabaseQueryBuilder(Database database, ArrayNode joins) { + this.database = database; + this.joins = joins; + } + + public MetabaseQueryBuilder forTable(TableDetails tableDetails) { + // code to be added + return this; + } + + + public MetabaseQueryBuilder joinWith(TableDetails joinTable, FieldDetails originField, FieldDetails destinationField) { + // Build the join condition and add to the joins array + ObjectMapper objectMapper = new ObjectMapper(); + ArrayNode joinCondition = objectMapper.createArrayNode(); + + joinCondition.add(ConditionType.EQUAL.getOperator()); + joinCondition.add(objectMapper.createArrayNode().add("field").add(originField.getId())); + joinCondition.add(objectMapper.createArrayNode().add("field").add(destinationField.getId())); + + joins.add(joinCondition); + return this; + } + + + public MetabaseQuery build() { + return new MetabaseQuery(database, joins); + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java index d57694aeb..b07844512 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java @@ -69,4 +69,7 @@ public String getDisplayName() { return displayName; } + public boolean nameMatches(String tableName) { + return tableName.equals(getDisplayName()); + } } diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java index 4f061266e..fca124e07 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java @@ -29,46 +29,14 @@ public AddressQuestionCreationService(@Lazy DatabaseService databaseService, Dat @Override public void createQuestionForTable(String tableName, String addressTableName, String addressField, String tableField) throws Exception { - int addressTableId = databaseService.getTableIdByName(addressTableName); - int joinFieldId1 = databaseService.getFieldIdByTableNameAndFieldName(addressTableName, addressField); - int tableId = databaseService.getTableIdByName(tableName); - int joinFieldId2 = databaseService.getFieldIdByTableNameAndFieldName(tableName, tableField); + Database database = databaseService.getGlobalDatabase(); + TableDetails tableDetails = databaseRepository.getTableDetailsByDisplayName(database, tableName); - MetabaseJoin join = new MetabaseJoin("all", tableName, tableId, joinFieldId1, joinFieldId2, tableName, objectMapper); - - ArrayNode joinsArray = objectMapper.createArrayNode(); - joinsArray.add(join.toJson(objectMapper)); - - MetabaseQuery query = new MetabaseQuery(databaseService.getDatabaseId(), addressTableId, joinsArray); - - MetabaseRequestBody requestBody = new MetabaseRequestBody( - "Address + " + tableName, query, VisualizationType.TABLE, null, objectMapper.createObjectNode(), databaseService.getCollectionId(), CardType.QUESTION); - - databaseRepository.postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); + databaseRepository.createQuestionForTable(database, tableDetails, addressTableName, addressField, tableField); } @Override public void createQuestionForTable(String tableName, String schema) { - int tableId = databaseService.getTableIdByName(tableName, schema); - - ObjectNode datasetQuery = objectMapper.createObjectNode(); - datasetQuery.put("database", databaseService.getDatabaseId()); - datasetQuery.put("type", "query"); - - ObjectNode query = objectMapper.createObjectNode(); - query.put("source-table", tableId); - datasetQuery.set("query", query); - - ObjectNode body = objectMapper.createObjectNode(); - body.put("name", tableName); - body.set("dataset_query", datasetQuery); - body.put("display", "table"); - body.putNull("description"); - body.set("visualization_settings", objectMapper.createObjectNode()); - body.put("collection_id", databaseService.getCollectionId()); - body.putNull("collection_position"); - body.putNull("result_metadata"); - - databaseRepository.postForObject(metabaseApiUrl + "/card", body, JsonNode.class); + // to be added } } diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 100626eb5..586529ace 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -12,6 +12,7 @@ import java.util.Arrays; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import org.avni.server.util.S; @@ -40,6 +41,11 @@ public DatabaseService(DatabaseRepository databaseRepository, ObjectMapper objec this.addressQuestionCreationService = addressQuestionCreationService; } + public Database getGlobalDatabase() { + int databaseId = metabaseService.getGlobalDatabaseId(); + return databaseRepository.getDatabaseByName(String.valueOf(databaseId)); + } + public int getDatabaseId() { if (databaseId == null) { databaseId = metabaseService.getGlobalDatabaseId(); @@ -54,8 +60,8 @@ public int getCollectionId() { return collectionId; } - public int getTableIdByName(String tableName) { - MetabaseDatabaseInfo databaseInfo = databaseRepository.getDatabaseDetails(getDatabaseId()); + public int getTableIdByDisplayName(String tableName) { + MetabaseDatabaseInfo databaseInfo = databaseRepository.getDatabaseDetails(getGlobalDatabase()); List tables = databaseInfo.getTables(); for (TableDetails table : tables) { @@ -66,8 +72,8 @@ public int getTableIdByName(String tableName) { return -1; } - public int getTableIdByName(String tableName, String schema) { - MetabaseDatabaseInfo databaseInfo = databaseRepository.getDatabaseDetails(getDatabaseId()); + public int getTableIdByDisplayName(String tableName, String schema) { + MetabaseDatabaseInfo databaseInfo = databaseRepository.getDatabaseDetails(getGlobalDatabase()); List tables = databaseInfo.getTables(); for (TableDetails table : tables) { @@ -84,13 +90,26 @@ public int getTableIdByName(String tableName, String schema) { return -1; } + public TableDetails getTableDetailsByDisplayName(String tableName) { + MetabaseDatabaseInfo databaseInfo = databaseRepository.getDatabaseDetails(getGlobalDatabase()); + List tables = databaseInfo.getTables(); + + // Handle Optional properly + Optional tableDetailsOptional = tables.stream() + .filter(tableDetail -> tableDetail.nameMatches(tableName)) + .findFirst(); + + return tableDetailsOptional.orElseThrow(() -> + new RuntimeException("Table not found: " + tableName)); + } + private String createRequestBodyForDataset(int sourceTableId) { return "{\"database\":" + getDatabaseId() + ",\"query\":{\"source-table\":" + sourceTableId + "},\"type\":\"query\",\"parameters\":[]}"; } public int getFieldIdByTableNameAndFieldName(String tableName, String fieldName) { - List fieldsList = databaseRepository.getFields(getDatabaseId()); + List fieldsList = databaseRepository.getFields(getGlobalDatabase()); String snakeCaseTableName = S.toSnakeCase(tableName); for (FieldDetails field : fieldsList) { @@ -108,53 +127,52 @@ public SyncStatus getInitialSyncStatus() { } - private DatasetResponse getTableMetadata() { - int tableMetadataId = getTableIdByName("Table Metadata"); - String requestBody = createRequestBodyForDataset(tableMetadataId); - return databaseRepository.getDataset(requestBody); + private DatasetResponse getTableMetadata(Database database) { + TableDetails metadataTable = databaseRepository.getTableDetailsByDisplayName(database, "table_metadata"); + return databaseRepository.findAll(metadataTable, database); } public List getSubjectTypeNames() { - DatasetResponse tableMetadata = getTableMetadata(); + Database database = getGlobalDatabase(); + TableDetails metadataTable = databaseRepository.getTableDetailsByDisplayName(database, "table_metadata"); + + DatasetResponse datasetResponse = databaseRepository.findAll(metadataTable, database); + List> rows = datasetResponse.getData().getRows(); + List subjectTypeNames = new ArrayList<>(); - List> rows = tableMetadata.getData().getRows(); for (List row : rows) { String type = row.get(2); - if (Arrays.asList( - TableType.INDIVIDUAL.getTypeName(), - TableType.HOUSEHOLD.getTypeName(), - TableType.GROUP.getTypeName(), - TableType.PERSON.getTypeName() - ).contains(type)) { - String rawName = row.get(1); - subjectTypeNames.add(S.formatName(rawName)); + if (type.equalsIgnoreCase(TableType.INDIVIDUAL.getTypeName()) || + type.equalsIgnoreCase(TableType.HOUSEHOLD.getTypeName()) || + type.equalsIgnoreCase(TableType.GROUP.getTypeName()) || + type.equalsIgnoreCase(TableType.PERSON.getTypeName())) { + subjectTypeNames.add(row.get(1)); } } - System.out.println("The subject type names: " + subjectTypeNames); return subjectTypeNames; } public List getProgramAndEncounterNames() { - DatasetResponse tableMetadata = getTableMetadata(); + Database database = getGlobalDatabase(); + TableDetails metadataTable = databaseRepository.getTableDetailsByDisplayName(database, "table_metadata"); + + DatasetResponse datasetResponse = databaseRepository.findAll(metadataTable, database); + List> rows = datasetResponse.getData().getRows(); + List programNames = new ArrayList<>(); - List> rows = tableMetadata.getData().getRows(); for (List row : rows) { String type = row.get(2); - if (Arrays.asList( - TableType.PROGRAM_ENCOUNTER.getTypeName(), - TableType.PROGRAM_ENROLMENT.getTypeName() - ).contains(type)) { - String rawName = row.get(1); - programNames.add(S.formatName(rawName)); + if (type.equalsIgnoreCase(TableType.PROGRAM_ENCOUNTER.getTypeName()) || + type.equalsIgnoreCase(TableType.PROGRAM_ENROLMENT.getTypeName())) { + programNames.add(row.get(1)); } } - System.out.println("The program and encounter names: " + programNames); return programNames; } @@ -169,27 +187,8 @@ private List extractTableNames(JsonNode databaseDetails) { return tableNames; } - private void createQuestionForTable(String tableName, String addressTableName, String addressField, String tableField) throws Exception { - int addressTableId = getTableIdByName(addressTableName); - int joinFieldId1 = getFieldIdByTableNameAndFieldName(addressTableName, addressField); - int tableId = getTableIdByName(tableName); - int joinFieldId2 = getFieldIdByTableNameAndFieldName(tableName, tableField); - - MetabaseJoin join = new MetabaseJoin("all", tableName, tableId, joinFieldId1, joinFieldId2, tableName, objectMapper); - - ArrayNode joinsArray = objectMapper.createArrayNode(); - joinsArray.add(join.toJson(objectMapper)); - - MetabaseQuery query = new MetabaseQuery(getDatabaseId(),addressTableId, joinsArray); - - MetabaseRequestBody requestBody = new MetabaseRequestBody( - "Address + " + tableName, query, VisualizationType.TABLE, null, objectMapper.createObjectNode(), getCollectionId(), CardType.QUESTION); - - databaseRepository.postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); - } - private void createQuestionForTable(String tableName, String schema) { - int tableId = getTableIdByName(tableName, schema); + int tableId = getTableIdByDisplayName(tableName, schema); ObjectNode datasetQuery = objectMapper.createObjectNode(); datasetQuery.put("database", getDatabaseId()); @@ -219,12 +218,15 @@ private void createQuestionForTable(String tableName, String schema) { public void createQuestionsForSubjectTypes() throws Exception { SyncStatus syncStatus = getInitialSyncStatus(); if (syncStatus != SyncStatus.COMPLETE) { - throw new RuntimeException("Database initial sync is not complete."); + throw new RuntimeException("Database sync is not complete. Cannot create questions."); } + Database database = getGlobalDatabase(); List subjectTypeNames = getSubjectTypeNames(); for (String subjectTypeName : subjectTypeNames) { + TableDetails subjectTable = databaseRepository.getTableDetailsByDisplayName(database, subjectTypeName); + addressQuestionCreationService.createQuestionForTable(subjectTypeName, "Address", "id", "address_id"); } } @@ -232,35 +234,39 @@ public void createQuestionsForSubjectTypes() throws Exception { public void createQuestionsForProgramsAndEncounters() throws Exception { SyncStatus syncStatus = getInitialSyncStatus(); if (syncStatus != SyncStatus.COMPLETE) { - throw new RuntimeException("Database initial sync is not complete."); + throw new RuntimeException("Database sync is not complete. Cannot create questions."); } - List programNames = getProgramAndEncounterNames(); + Database database = getGlobalDatabase(); - MetabaseDatabaseInfo databaseInfo = databaseRepository.getDatabaseDetails(getDatabaseId()); - List allTableNames = databaseInfo.getTables() - .stream() - .map(TableDetails::getDisplayName) - .collect(Collectors.toList()); + List programAndEncounterNames = getProgramAndEncounterNames(); - for (String programName : programNames) { - for (String tableName : allTableNames) { - if (tableName.equalsIgnoreCase(programName)) { - addressQuestionCreationService.createQuestionForTable(tableName, "Address", "id", "address_id"); - } - } + for (String programName : programAndEncounterNames) { + TableDetails programTable = databaseRepository.getTableDetailsByDisplayName(database, programName); + addressQuestionCreationService.createQuestionForTable(programName, "Address", "id", "address_id"); } } - public void createQuestionsForIndivdualTables() { - List tablesToCreateQuestionsFor = Arrays.asList( - "Address", - "Media", - "Sync Telemetry" - ); + public void createQuestionsForIndividualTables() throws Exception { + SyncStatus syncStatus = getInitialSyncStatus(); + if (syncStatus != SyncStatus.COMPLETE) { + throw new RuntimeException("Database sync is not complete. Cannot create questions."); + } + + Database database = getGlobalDatabase(); + List tablesToCreateQuestionsFor = Arrays.asList("Address", "Media", "Sync Telemetry"); for (String tableName : tablesToCreateQuestionsFor) { - createQuestionForTable(tableName, "!public"); + TableDetails tableDetails = databaseRepository.getTableDetailsByDisplayName(database, tableName); + databaseRepository.createQuestionForTable(database, tableDetails, "Address", "id", "address_id"); } } + + public void createQuestions() throws Exception { + createQuestionsForSubjectTypes(); + + createQuestionsForProgramsAndEncounters(); + + createQuestionsForIndividualTables(); + } } diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java index fad55d9ea..4ac97ec05 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java @@ -23,7 +23,7 @@ public class MetabaseService { private final CollectionPermissionsRepository collectionPermissionsRepository; private final CollectionRepository collectionRepository; private Database globalDatabase; - private CollectionResponse globalCollection; + private MetabaseCollectionInfo globalCollection; @Autowired public MetabaseService(OrganisationService organisationService, @@ -51,7 +51,7 @@ public void setupMetabase() { this.globalDatabase = database; CollectionResponse metabaseCollection = collectionRepository.save(new Collection(name, name + " collection")); - this.globalCollection = metabaseCollection; + this.globalCollection = new MetabaseCollectionInfo(null, metabaseCollection.getId(), false); Group metabaseGroup = groupPermissionsRepository.save(new Group(name)); @@ -69,10 +69,18 @@ public void createQuestionsForSubjectTypes() throws Exception{ } public int getGlobalDatabaseId() { + if (globalDatabase == null) { + Organisation currentOrganisation = organisationService.getCurrentOrganisation(); + globalDatabase = databaseRepository.getDatabaseByName(currentOrganisation.getName()); + } return globalDatabase.getId(); } public int getGlobalCollectionId() { + if (globalCollection == null) { + Organisation currentOrganisation = organisationService.getCurrentOrganisation(); + globalCollection = databaseRepository.getCollectionByName(currentOrganisation.getName() + " collection"); + } return globalCollection.getId(); } } diff --git a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java index d8ba5ff11..3aac32847 100644 --- a/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java +++ b/avni-server-api/src/main/java/org/avni/server/web/MetabaseController.java @@ -30,9 +30,7 @@ public void setupMetabase() { @PostMapping("/create-questions") public void createQuestions() throws Exception{ - databaseService.createQuestionsForSubjectTypes(); - databaseService.createQuestionsForProgramsAndEncounters(); - databaseService.createQuestionsForIndivdualTables(); + databaseService.createQuestions(); } @GetMapping("/sync-status") From de870a15673ce14c5cf62afa09577206f89d5bb1 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sat, 7 Sep 2024 12:25:55 +0530 Subject: [PATCH 24/34] avniproject#762 | Working code with MetabaseQueryBuilder and QuestionCreationService --- .../dao/metabase/CollectionRepository.java | 6 +- .../dao/metabase/DatabaseRepository.java | 63 ++++--- .../metabase/CollectionInfoResponse.java | 56 +++++++ ...tion.java => CreateCollectionRequest.java} | 4 +- .../avni/server/domain/metabase/Database.java | 3 + .../domain/metabase/DatabaseDetails.java | 3 + .../metabase/MetabaseCollectionInfo.java | 37 ----- .../server/domain/metabase/MetabaseQuery.java | 22 +-- .../domain/metabase/MetabaseQueryBuilder.java | 54 +++--- .../domain/metabase/MetabaseRequestBody.java | 2 +- .../AddressQuestionCreationService.java | 26 ++- .../service/metabase/DatabaseService.java | 156 ++++-------------- .../service/metabase/MetabaseService.java | 12 +- .../metabase/QuestionCreationService.java | 2 +- ...a => CreateCollectionRequestUtilTest.java} | 2 +- 15 files changed, 213 insertions(+), 235 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/CollectionInfoResponse.java rename avni-server-api/src/main/java/org/avni/server/domain/metabase/{Collection.java => CreateCollectionRequest.java} (73%) delete mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseCollectionInfo.java rename avni-server-api/src/test/java/org/avni/server/util/{CollectionUtilTest.java => CreateCollectionRequestUtilTest.java} (93%) diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/CollectionRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/CollectionRepository.java index 50c28e816..ac07057c2 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/CollectionRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/CollectionRepository.java @@ -1,6 +1,6 @@ package org.avni.server.dao.metabase; -import org.avni.server.domain.metabase.Collection; +import org.avni.server.domain.metabase.CreateCollectionRequest; import org.avni.server.domain.metabase.CollectionResponse; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.stereotype.Repository; @@ -11,8 +11,8 @@ public CollectionRepository(RestTemplateBuilder restTemplateBuilder) { super(restTemplateBuilder); } - public CollectionResponse save(Collection collection) { + public CollectionResponse save(CreateCollectionRequest createCollectionRequest) { String url = metabaseApiUrl + "/collection"; - return postForObject(url, collection, CollectionResponse.class); + return postForObject(url, createCollectionRequest, CollectionResponse.class); } } diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java index 45a2f511f..25e532b04 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java @@ -8,7 +8,6 @@ import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.stereotype.Repository; import com.fasterxml.jackson.databind.ObjectMapper; - import java.util.List; @Repository @@ -26,28 +25,52 @@ public Database save(Database database) { return database; } - public Database getDatabaseByName(String databaseName) { + public Database getDatabaseById(Database database) { + int id = database.getId(); + String url = metabaseApiUrl + "/database/" + id; + try { + String jsonResponse = getForObject(url, String.class); + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.readValue(jsonResponse, Database.class); + } catch (Exception e) { + throw new RuntimeException("Failed to retrieve database with ID " + id, e); + } + } + + public Database getDatabaseByName(String name) { String url = metabaseApiUrl + "/database"; + String jsonResponse = getForObject(url, String.class); + try { - List databases = objectMapper.readValue(jsonResponse, new TypeReference>() {}); - return databases.stream() - .filter(db -> db.getName().equals(databaseName)) - .findFirst() - .orElseThrow(() -> new RuntimeException("Database with name " + databaseName + " not found.")); + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode rootNode = objectMapper.readTree(jsonResponse); + JsonNode dataArray = rootNode.path("data"); + + for (JsonNode dbNode : dataArray) { + Database db = objectMapper.treeToValue(dbNode, Database.class); + if (db.getName().equals(name)) { + return db; + } + } + throw new RuntimeException("Database with name " + name + " not found."); } catch (Exception e) { throw new RuntimeException("Failed to retrieve database", e); } } - public MetabaseCollectionInfo getCollectionByName(String collectionName) { + public CollectionInfoResponse getCollectionByName(String collectionName) { String url = metabaseApiUrl + "/collection"; - String jsonResponse = getForObject(url, String.class); - try { - List collections = objectMapper.readValue(jsonResponse, new TypeReference>() {}); + String jsonResponse = getForObject(url, String.class); + + ObjectMapper objectMapper = new ObjectMapper(); + List collections = objectMapper.readValue( + jsonResponse, new TypeReference>() {} + ); + return collections.stream() - .filter(coll -> coll.getName().equals(collectionName)) + .filter(collection -> collection.getName().equals(collectionName)) .findFirst() .orElseThrow(() -> new RuntimeException("Collection with name " + collectionName + " not found.")); } catch (Exception e) { @@ -55,13 +78,13 @@ public MetabaseCollectionInfo getCollectionByName(String collectionName) { } } - public void createQuestionForTable(Database database, TableDetails tableDetails, String addressTableName, String addressField, String tableField) throws Exception { - TableDetails addressTableDetails = getTableDetailsByDisplayName(database, addressTableName); + public void createQuestionForTable(Database database, TableDetails tableDetails, String addressTableName, String addressField, String tableField) { + TableDetails addressTableDetails = getTableDetailsByName(database, addressTableName); FieldDetails joinField1 = getFieldDetailsByName(database, addressTableName, addressField); FieldDetails joinField2 = getFieldDetailsByName(database, tableDetails.getName(), tableField); ArrayNode joinsArray = objectMapper.createArrayNode(); - MetabaseQuery query = new MetabaseQueryBuilder(database, joinsArray) + MetabaseQuery query = new MetabaseQueryBuilder(database, joinsArray, objectMapper) .forTable(tableDetails) .joinWith(addressTableDetails, joinField1, joinField2) .build(); @@ -72,7 +95,7 @@ public void createQuestionForTable(Database database, TableDetails tableDetails, VisualizationType.TABLE, null, objectMapper.createObjectNode(), - getCollectionByName(database.getName()).getId(), + getCollectionByName(database.getName()).getIdAsInt(), CardType.QUESTION ); @@ -111,16 +134,16 @@ public List getFields(Database database) { } } - public TableDetails getTableDetailsByDisplayName(Database database, String tableName) { + public TableDetails getTableDetailsByName(Database database, String tableName) { MetabaseDatabaseInfo databaseInfo = getDatabaseDetails(database); return databaseInfo.getTables().stream() - .filter(tableDetail -> tableDetail.nameMatches(tableName)) + .filter(tableDetail -> tableDetail.getName().equalsIgnoreCase(tableName)) .findFirst() .orElseThrow(() -> new RuntimeException("Table with name " + tableName + " not found.")); } - public DatabaseSyncStatus getInitialSyncStatus(int databaseId) { - String url = metabaseApiUrl + "/database/" + databaseId; + public DatabaseSyncStatus getInitialSyncStatus(Database database) { + String url = metabaseApiUrl + "/database/" + database.getId(); String jsonResponse = getForObject(url, String.class); try { return objectMapper.readValue(jsonResponse, DatabaseSyncStatus.class); diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/CollectionInfoResponse.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/CollectionInfoResponse.java new file mode 100644 index 000000000..322aff320 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/CollectionInfoResponse.java @@ -0,0 +1,56 @@ +package org.avni.server.domain.metabase; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class CollectionInfoResponse { + + private String name; + private String id; + private boolean isPersonal; + + public CollectionInfoResponse() { + } + + public CollectionInfoResponse(@JsonProperty("name") String name, + @JsonProperty("id") Object id, + @JsonProperty("is_personal") boolean isPersonal) { + this.name = name; + this.isPersonal = isPersonal; + + if (id instanceof Integer) { + this.id = String.valueOf(id); + } else { + this.id = id.toString(); + } + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getId() { + return id; + } + + public int getIdAsInt() { + try { + return Integer.parseInt(id); + } catch (NumberFormatException e) { + throw new RuntimeException("Failed to convert id to integer: " + id, e); + } + } + + public boolean isPersonal() { + return isPersonal; + } + + public void setPersonal(boolean personal) { + isPersonal = personal; + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/Collection.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/CreateCollectionRequest.java similarity index 73% rename from avni-server-api/src/main/java/org/avni/server/domain/metabase/Collection.java rename to avni-server-api/src/main/java/org/avni/server/domain/metabase/CreateCollectionRequest.java index 3b24bfa2c..b9fc8e6c2 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/Collection.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/CreateCollectionRequest.java @@ -1,10 +1,10 @@ package org.avni.server.domain.metabase; -public class Collection { +public class CreateCollectionRequest { private String name; private String description; - public Collection(String name, String description) { + public CreateCollectionRequest(String name, String description) { this.name = name; this.description = description; } diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/Database.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/Database.java index 86b0ce604..009f07762 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/Database.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/Database.java @@ -1,5 +1,8 @@ package org.avni.server.domain.metabase; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) public class Database { private Integer id; private String name; diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatabaseDetails.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatabaseDetails.java index 5844499a2..519f002e6 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatabaseDetails.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatabaseDetails.java @@ -1,5 +1,8 @@ package org.avni.server.domain.metabase; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) public class DatabaseDetails { private String host; private String port; diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseCollectionInfo.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseCollectionInfo.java deleted file mode 100644 index e67046408..000000000 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseCollectionInfo.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.avni.server.domain.metabase; - -public class MetabaseCollectionInfo { - private String name; - private int id; - private boolean isPersonal; - - public MetabaseCollectionInfo(String name, int id, boolean isPersonal) { - this.name = name; - this.id = id; - this.isPersonal = isPersonal; - } - - public String getName() { - return name; - } - - public int getId() { - return id; - } - - public boolean isPersonal() { - return isPersonal; - } - - public void setName(String name) { - this.name = name; - } - - public void setId(int id) { - this.id = id; - } - - public void setPersonal(boolean personal) { - isPersonal = personal; - } -} diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java index 5b2db827c..8323d79b0 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQuery.java @@ -1,28 +1,20 @@ package org.avni.server.domain.metabase; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; public class MetabaseQuery { - private final Database database; - private final ArrayNode joins; + private final int databaseId; + private final ObjectNode queryNode; - public MetabaseQuery(Database database, ArrayNode joins) { - this.database = database; - this.joins = joins; + public MetabaseQuery(int databaseId, ObjectNode queryNode) { + this.databaseId = databaseId; + this.queryNode = queryNode; } public int getDatabaseId() { - return database.getId(); + return databaseId; } - - public ObjectNode toJson(ObjectMapper objectMapper) { - ObjectNode queryNode = objectMapper.createObjectNode(); - queryNode.put("database", database.getId()); - queryNode.set("joins", joins); - queryNode.put("type", "query"); + public ObjectNode toJson() { return queryNode; } } diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java index 28460c685..e48ab94bc 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java @@ -1,41 +1,57 @@ -// to be completed package org.avni.server.domain.metabase; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; public class MetabaseQueryBuilder { private final Database database; - private ArrayNode joins; + private final ArrayNode joinsArray; + private final ObjectMapper objectMapper; + private ObjectNode queryNode; - public MetabaseQueryBuilder(Database database, ArrayNode joins) { + public MetabaseQueryBuilder(Database database, ArrayNode joinsArray, ObjectMapper objectMapper) { this.database = database; - this.joins = joins; + this.joinsArray = joinsArray; + this.objectMapper = objectMapper; + this.queryNode = objectMapper.createObjectNode(); } public MetabaseQueryBuilder forTable(TableDetails tableDetails) { - // code to be added + queryNode.put("source-table", tableDetails.getId()); + queryNode.put("database", database.getId()); return this; } - - public MetabaseQueryBuilder joinWith(TableDetails joinTable, FieldDetails originField, FieldDetails destinationField) { - // Build the join condition and add to the joins array - ObjectMapper objectMapper = new ObjectMapper(); - ArrayNode joinCondition = objectMapper.createArrayNode(); - - joinCondition.add(ConditionType.EQUAL.getOperator()); - joinCondition.add(objectMapper.createArrayNode().add("field").add(originField.getId())); - joinCondition.add(objectMapper.createArrayNode().add("field").add(destinationField.getId())); - - joins.add(joinCondition); + public MetabaseQueryBuilder joinWith(TableDetails addressTable, FieldDetails joinField1, FieldDetails joinField2) { + ObjectNode joinNode = objectMapper.createObjectNode(); + joinNode.put("fields", "all"); + joinNode.put("alias", addressTable.getName()); + joinNode.put("source-table", addressTable.getId()); + + ArrayNode conditionArray = objectMapper.createArrayNode(); + conditionArray.add("="); + + ArrayNode leftField = objectMapper.createArrayNode(); + leftField.add("field"); + leftField.add(joinField1.getId()); + leftField.add(objectMapper.createObjectNode().put("base-type", "type/Integer")); + conditionArray.add(leftField); + + ArrayNode rightField = objectMapper.createArrayNode(); + rightField.add("field"); + rightField.add(joinField2.getId()); + rightField.add(objectMapper.createObjectNode().put("base-type", "type/Integer").put("join-alias", addressTable.getName())); + conditionArray.add(rightField); + + joinNode.set("condition", conditionArray); + joinsArray.add(joinNode); + queryNode.set("joins", joinsArray); return this; } - public MetabaseQuery build() { - return new MetabaseQuery(database, joins); + queryNode.put("type", "query"); + return new MetabaseQuery(database.getId(), queryNode); } } diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java index 68b3e4572..d5ab1fbc1 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java @@ -32,7 +32,7 @@ public ObjectNode toJson(ObjectMapper objectMapper) { ObjectNode datasetQueryNode = objectMapper.createObjectNode(); datasetQueryNode.put("database", datasetQuery.getDatabaseId()); datasetQueryNode.put("type", "query"); - datasetQueryNode.set("query", datasetQuery.toJson(objectMapper)); + datasetQueryNode.set("query", datasetQuery.toJson()); rootNode.set("dataset_query", datasetQueryNode); rootNode.put("display", display.toString()); diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java index fca124e07..4c59f3636 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import org.avni.server.dao.metabase.DatabaseRepository; import org.avni.server.domain.metabase.*; import org.springframework.beans.factory.annotation.Autowired; @@ -30,13 +29,32 @@ public AddressQuestionCreationService(@Lazy DatabaseService databaseService, Dat @Override public void createQuestionForTable(String tableName, String addressTableName, String addressField, String tableField) throws Exception { Database database = databaseService.getGlobalDatabase(); - TableDetails tableDetails = databaseRepository.getTableDetailsByDisplayName(database, tableName); + TableDetails tableDetails = databaseRepository.getTableDetailsByName(database, tableName); databaseRepository.createQuestionForTable(database, tableDetails, addressTableName, addressField, tableField); } @Override - public void createQuestionForTable(String tableName, String schema) { - // to be added + public void createQuestionForTable(String tableName, String schema) throws Exception { + Database database = databaseService.getGlobalDatabase(); + TableDetails tableDetails = databaseRepository.getTableDetailsByName(database, tableName); + + ArrayNode joinsArray = objectMapper.createArrayNode(); + + MetabaseQuery query = new MetabaseQueryBuilder(database, joinsArray, objectMapper) + .forTable(tableDetails) + .build(); + + MetabaseRequestBody requestBody = new MetabaseRequestBody( + tableName, + query, + VisualizationType.TABLE, + null, + objectMapper.createObjectNode(), + databaseService.getCollectionId(), + CardType.QUESTION + ); + databaseRepository.postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); } + } diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 586529ace..c30b1350a 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -2,8 +2,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import org.avni.server.dao.metabase.DatabaseRepository; import org.avni.server.domain.metabase.*; import org.springframework.beans.factory.annotation.Autowired; @@ -12,10 +10,6 @@ import java.util.Arrays; import java.util.ArrayList; import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -import org.avni.server.util.S; @Service public class DatabaseService { @@ -42,8 +36,8 @@ public DatabaseService(DatabaseRepository databaseRepository, ObjectMapper objec } public Database getGlobalDatabase() { - int databaseId = metabaseService.getGlobalDatabaseId(); - return databaseRepository.getDatabaseByName(String.valueOf(databaseId)); + Database globalDatabase = metabaseService.getGlobalDatabase(); + return databaseRepository.getDatabaseById(globalDatabase); } public int getDatabaseId() { @@ -60,81 +54,16 @@ public int getCollectionId() { return collectionId; } - public int getTableIdByDisplayName(String tableName) { - MetabaseDatabaseInfo databaseInfo = databaseRepository.getDatabaseDetails(getGlobalDatabase()); - List tables = databaseInfo.getTables(); - - for (TableDetails table : tables) { - if (tableName.equals(table.getDisplayName())) { - return table.getId(); - } - } - return -1; - } - - public int getTableIdByDisplayName(String tableName, String schema) { - MetabaseDatabaseInfo databaseInfo = databaseRepository.getDatabaseDetails(getGlobalDatabase()); - List tables = databaseInfo.getTables(); - - for (TableDetails table : tables) { - String tableSchema = table.getSchema(); - - boolean schemaMatches = schema.equals("public") - ? "public".equals(tableSchema) - : !"public".equals(tableSchema); - - if (tableName.equals(table.getDisplayName()) && schemaMatches) { - return table.getId(); - } - } - return -1; - } - - public TableDetails getTableDetailsByDisplayName(String tableName) { - MetabaseDatabaseInfo databaseInfo = databaseRepository.getDatabaseDetails(getGlobalDatabase()); - List tables = databaseInfo.getTables(); - - // Handle Optional properly - Optional tableDetailsOptional = tables.stream() - .filter(tableDetail -> tableDetail.nameMatches(tableName)) - .findFirst(); - - return tableDetailsOptional.orElseThrow(() -> - new RuntimeException("Table not found: " + tableName)); - } - - - private String createRequestBodyForDataset(int sourceTableId) { - return "{\"database\":" + getDatabaseId() + ",\"query\":{\"source-table\":" + sourceTableId + "},\"type\":\"query\",\"parameters\":[]}"; - } - - public int getFieldIdByTableNameAndFieldName(String tableName, String fieldName) { - List fieldsList = databaseRepository.getFields(getGlobalDatabase()); - String snakeCaseTableName = S.toSnakeCase(tableName); - - for (FieldDetails field : fieldsList) { - if (snakeCaseTableName.equals(field.getTableName()) && fieldName.equals(field.getName())) { - return field.getId(); - } - } - return -1; - } - public SyncStatus getInitialSyncStatus() { - DatabaseSyncStatus databaseSyncStatus = databaseRepository.getInitialSyncStatus(getDatabaseId()); + Database globalDatabase = metabaseService.getGlobalDatabase(); + DatabaseSyncStatus databaseSyncStatus = databaseRepository.getInitialSyncStatus(globalDatabase); String status = databaseSyncStatus.getInitialSyncStatus(); return SyncStatus.fromString(status); } - - private DatasetResponse getTableMetadata(Database database) { - TableDetails metadataTable = databaseRepository.getTableDetailsByDisplayName(database, "table_metadata"); - return databaseRepository.findAll(metadataTable, database); - } - public List getSubjectTypeNames() { Database database = getGlobalDatabase(); - TableDetails metadataTable = databaseRepository.getTableDetailsByDisplayName(database, "table_metadata"); + TableDetails metadataTable = databaseRepository.getTableDetailsByName(database, "table_metadata"); DatasetResponse datasetResponse = databaseRepository.findAll(metadataTable, database); List> rows = datasetResponse.getData().getRows(); @@ -158,7 +87,7 @@ public List getSubjectTypeNames() { public List getProgramAndEncounterNames() { Database database = getGlobalDatabase(); - TableDetails metadataTable = databaseRepository.getTableDetailsByDisplayName(database, "table_metadata"); + TableDetails metadataTable = databaseRepository.getTableDetailsByName(database, "table_metadata"); DatasetResponse datasetResponse = databaseRepository.findAll(metadataTable, database); List> rows = datasetResponse.getData().getRows(); @@ -176,57 +105,14 @@ public List getProgramAndEncounterNames() { return programNames; } - - - private List extractTableNames(JsonNode databaseDetails) { - List tableNames = new ArrayList<>(); - JsonNode tablesArray = databaseDetails.path("tables"); - for (JsonNode tableNode : tablesArray) { - tableNames.add(tableNode.path("display_name").asText()); - } - return tableNames; - } - - private void createQuestionForTable(String tableName, String schema) { - int tableId = getTableIdByDisplayName(tableName, schema); - - ObjectNode datasetQuery = objectMapper.createObjectNode(); - datasetQuery.put("database", getDatabaseId()); - datasetQuery.put("type", "query"); - - ObjectNode query = objectMapper.createObjectNode(); - query.put("source-table", tableId); - datasetQuery.set("query", query); - - ObjectNode body = objectMapper.createObjectNode(); - body.put("name", tableName); - body.set("dataset_query", datasetQuery); - body.put("display", "table"); - body.putNull("description"); - body.set("visualization_settings", objectMapper.createObjectNode()); - body.put("collection_id", getCollectionId()); - body.putNull("collection_position"); - body.putNull("result_metadata"); - - databaseRepository.postForObject( - metabaseApiUrl + "/card", - body, - JsonNode.class - ); - } - public void createQuestionsForSubjectTypes() throws Exception { SyncStatus syncStatus = getInitialSyncStatus(); if (syncStatus != SyncStatus.COMPLETE) { throw new RuntimeException("Database sync is not complete. Cannot create questions."); } - - Database database = getGlobalDatabase(); List subjectTypeNames = getSubjectTypeNames(); for (String subjectTypeName : subjectTypeNames) { - TableDetails subjectTable = databaseRepository.getTableDetailsByDisplayName(database, subjectTypeName); - addressQuestionCreationService.createQuestionForTable(subjectTypeName, "Address", "id", "address_id"); } } @@ -237,31 +123,45 @@ public void createQuestionsForProgramsAndEncounters() throws Exception { throw new RuntimeException("Database sync is not complete. Cannot create questions."); } - Database database = getGlobalDatabase(); - List programAndEncounterNames = getProgramAndEncounterNames(); for (String programName : programAndEncounterNames) { - TableDetails programTable = databaseRepository.getTableDetailsByDisplayName(database, programName); addressQuestionCreationService.createQuestionForTable(programName, "Address", "id", "address_id"); } } - public void createQuestionsForIndividualTables() throws Exception { + public void createQuestionsForIndividualTables() { SyncStatus syncStatus = getInitialSyncStatus(); if (syncStatus != SyncStatus.COMPLETE) { throw new RuntimeException("Database sync is not complete. Cannot create questions."); } Database database = getGlobalDatabase(); - List tablesToCreateQuestionsFor = Arrays.asList("Address", "Media", "Sync Telemetry"); - for (String tableName : tablesToCreateQuestionsFor) { - TableDetails tableDetails = databaseRepository.getTableDetailsByDisplayName(database, tableName); - databaseRepository.createQuestionForTable(database, tableDetails, "Address", "id", "address_id"); + List individualTables = Arrays.asList("address", "media", "sync_telemetry"); // Adjust these as needed + + for (String tableName : individualTables) { + TableDetails tableDetails = databaseRepository.getTableDetailsByName(database, tableName); + + MetabaseQuery query = new MetabaseQueryBuilder(database, objectMapper.createArrayNode(), objectMapper) + .forTable(tableDetails) + .build(); + + MetabaseRequestBody requestBody = new MetabaseRequestBody( + tableName, + query, + VisualizationType.TABLE, + null, + objectMapper.createObjectNode(), + databaseRepository.getCollectionByName(database.getName()).getIdAsInt(), + CardType.QUESTION + ); + + databaseRepository.postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); } } + public void createQuestions() throws Exception { createQuestionsForSubjectTypes(); diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java index 4ac97ec05..746267e41 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java @@ -23,7 +23,7 @@ public class MetabaseService { private final CollectionPermissionsRepository collectionPermissionsRepository; private final CollectionRepository collectionRepository; private Database globalDatabase; - private MetabaseCollectionInfo globalCollection; + private CollectionInfoResponse globalCollection; @Autowired public MetabaseService(OrganisationService organisationService, @@ -50,8 +50,8 @@ public void setupMetabase() { Database database = databaseRepository.save(new Database(name, "postgres", new DatabaseDetails(avniDatabase, dbUser))); this.globalDatabase = database; - CollectionResponse metabaseCollection = collectionRepository.save(new Collection(name, name + " collection")); - this.globalCollection = new MetabaseCollectionInfo(null, metabaseCollection.getId(), false); + CollectionResponse metabaseCollection = collectionRepository.save(new CreateCollectionRequest(name, name + " collection")); + this.globalCollection = new CollectionInfoResponse(null, metabaseCollection.getId(), false); Group metabaseGroup = groupPermissionsRepository.save(new Group(name)); @@ -81,6 +81,10 @@ public int getGlobalCollectionId() { Organisation currentOrganisation = organisationService.getCurrentOrganisation(); globalCollection = databaseRepository.getCollectionByName(currentOrganisation.getName() + " collection"); } - return globalCollection.getId(); + return globalCollection.getIdAsInt(); + } + + public Database getGlobalDatabase() { + return globalDatabase; } } diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java index 58babec9f..121bfe158 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java @@ -2,5 +2,5 @@ public interface QuestionCreationService { void createQuestionForTable(String tableName, String addressTableName, String addressField, String tableField) throws Exception; - void createQuestionForTable(String tableName, String schema); + void createQuestionForTable(String tableName, String schema) throws Exception; } diff --git a/avni-server-api/src/test/java/org/avni/server/util/CollectionUtilTest.java b/avni-server-api/src/test/java/org/avni/server/util/CreateCollectionRequestUtilTest.java similarity index 93% rename from avni-server-api/src/test/java/org/avni/server/util/CollectionUtilTest.java rename to avni-server-api/src/test/java/org/avni/server/util/CreateCollectionRequestUtilTest.java index d6219aa2f..b0928ecb8 100644 --- a/avni-server-api/src/test/java/org/avni/server/util/CollectionUtilTest.java +++ b/avni-server-api/src/test/java/org/avni/server/util/CreateCollectionRequestUtilTest.java @@ -5,7 +5,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -public class CollectionUtilTest { +public class CreateCollectionRequestUtilTest { @Test public void hasOnlyTrailingEmptyStrings() { assertTrue(CollectionUtil.hasOnlyTrailingEmptyStrings(Arrays.asList("a", "b", ""))); From 74ec09d87c0150c5e8a0aba804a0fc9d100c2c2b Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sat, 7 Sep 2024 21:56:18 +0530 Subject: [PATCH 25/34] avniproject#762 | renamed getTableDetailsByName to findTableDetailsByName and removed Primitive obsession --- .../dao/metabase/DatabaseRepository.java | 11 ++-- .../server/domain/metabase/TableDetails.java | 4 ++ .../AddressQuestionCreationService.java | 18 +++--- .../service/metabase/DatabaseService.java | 55 ++++++++++++++++--- .../metabase/QuestionCreationService.java | 3 +- 5 files changed, 66 insertions(+), 25 deletions(-) diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java index 25e532b04..3b20ad76d 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java @@ -78,9 +78,8 @@ public CollectionInfoResponse getCollectionByName(String collectionName) { } } - public void createQuestionForTable(Database database, TableDetails tableDetails, String addressTableName, String addressField, String tableField) { - TableDetails addressTableDetails = getTableDetailsByName(database, addressTableName); - FieldDetails joinField1 = getFieldDetailsByName(database, addressTableName, addressField); + public void createQuestionForTable(Database database, TableDetails tableDetails, TableDetails addressTableDetails, String addressField, String tableField) { + FieldDetails joinField1 = getFieldDetailsByName(database, addressTableDetails.getName(), addressField); FieldDetails joinField2 = getFieldDetailsByName(database, tableDetails.getName(), tableField); ArrayNode joinsArray = objectMapper.createArrayNode(); @@ -134,12 +133,12 @@ public List getFields(Database database) { } } - public TableDetails getTableDetailsByName(Database database, String tableName) { + public TableDetails findTableDetailsByName(Database database, TableDetails targetTable) { MetabaseDatabaseInfo databaseInfo = getDatabaseDetails(database); return databaseInfo.getTables().stream() - .filter(tableDetail -> tableDetail.getName().equalsIgnoreCase(tableName)) + .filter(tableDetail -> tableDetail.getName().equalsIgnoreCase(targetTable.getName())) .findFirst() - .orElseThrow(() -> new RuntimeException("Table with name " + tableName + " not found.")); + .orElseThrow(() -> new RuntimeException("Table with name " + targetTable.getName() + " not found.")); } public DatabaseSyncStatus getInitialSyncStatus(Database database) { diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java index b07844512..f2178fbc3 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java @@ -72,4 +72,8 @@ public String getDisplayName() { public boolean nameMatches(String tableName) { return tableName.equals(getDisplayName()); } + + public void setName(String name) { + this.name = name; + } } diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java index 4c59f3636..2e8d35115 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; import org.avni.server.dao.metabase.DatabaseRepository; import org.avni.server.domain.metabase.*; import org.springframework.beans.factory.annotation.Autowired; @@ -27,22 +26,22 @@ public AddressQuestionCreationService(@Lazy DatabaseService databaseService, Dat } @Override - public void createQuestionForTable(String tableName, String addressTableName, String addressField, String tableField) throws Exception { + public void createQuestionForTable(TableDetails tableDetails, TableDetails addressTableDetails, String addressField, String tableField) throws Exception { Database database = databaseService.getGlobalDatabase(); - TableDetails tableDetails = databaseRepository.getTableDetailsByName(database, tableName); - - databaseRepository.createQuestionForTable(database, tableDetails, addressTableName, addressField, tableField); + databaseRepository.createQuestionForTable(database, tableDetails, addressTableDetails, addressField, tableField); } + @Override public void createQuestionForTable(String tableName, String schema) throws Exception { Database database = databaseService.getGlobalDatabase(); - TableDetails tableDetails = databaseRepository.getTableDetailsByName(database, tableName); - ArrayNode joinsArray = objectMapper.createArrayNode(); + TableDetails tableDetails = new TableDetails(); + tableDetails.setName(tableName); + TableDetails fetchedTableDetails = databaseRepository.findTableDetailsByName(database, tableDetails); - MetabaseQuery query = new MetabaseQueryBuilder(database, joinsArray, objectMapper) - .forTable(tableDetails) + MetabaseQuery query = new MetabaseQueryBuilder(database, objectMapper.createArrayNode(), objectMapper) + .forTable(fetchedTableDetails) .build(); MetabaseRequestBody requestBody = new MetabaseRequestBody( @@ -57,4 +56,5 @@ public void createQuestionForTable(String tableName, String schema) throws Excep databaseRepository.postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); } + } diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index c30b1350a..c7af8bb69 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -63,9 +63,12 @@ public SyncStatus getInitialSyncStatus() { public List getSubjectTypeNames() { Database database = getGlobalDatabase(); - TableDetails metadataTable = databaseRepository.getTableDetailsByName(database, "table_metadata"); + TableDetails metadataTable = new TableDetails(); + metadataTable.setName("table_metadata"); - DatasetResponse datasetResponse = databaseRepository.findAll(metadataTable, database); + TableDetails fetchedMetadataTable = databaseRepository.findTableDetailsByName(database, metadataTable); + + DatasetResponse datasetResponse = databaseRepository.findAll(fetchedMetadataTable, database); List> rows = datasetResponse.getData().getRows(); List subjectTypeNames = new ArrayList<>(); @@ -85,11 +88,15 @@ public List getSubjectTypeNames() { + public List getProgramAndEncounterNames() { Database database = getGlobalDatabase(); - TableDetails metadataTable = databaseRepository.getTableDetailsByName(database, "table_metadata"); + TableDetails metadataTable = new TableDetails(); + metadataTable.setName("table_metadata"); + + TableDetails fetchedMetadataTable = databaseRepository.findTableDetailsByName(database, metadataTable); - DatasetResponse datasetResponse = databaseRepository.findAll(metadataTable, database); + DatasetResponse datasetResponse = databaseRepository.findAll(fetchedMetadataTable, database); List> rows = datasetResponse.getData().getRows(); List programNames = new ArrayList<>(); @@ -105,31 +112,59 @@ public List getProgramAndEncounterNames() { return programNames; } + public void createQuestionsForSubjectTypes() throws Exception { SyncStatus syncStatus = getInitialSyncStatus(); if (syncStatus != SyncStatus.COMPLETE) { throw new RuntimeException("Database sync is not complete. Cannot create questions."); } + + Database database = getGlobalDatabase(); List subjectTypeNames = getSubjectTypeNames(); + TableDetails addressTableDetails = new TableDetails(); + addressTableDetails.setName("Address"); + + TableDetails fetchedAddressTableDetails = databaseRepository.findTableDetailsByName(database, addressTableDetails); + for (String subjectTypeName : subjectTypeNames) { - addressQuestionCreationService.createQuestionForTable(subjectTypeName, "Address", "id", "address_id"); + TableDetails subjectTableDetails = new TableDetails(); + subjectTableDetails.setName(subjectTypeName); + + TableDetails fetchedSubjectTableDetails = databaseRepository.findTableDetailsByName(database, subjectTableDetails); + + addressQuestionCreationService.createQuestionForTable(fetchedSubjectTableDetails, fetchedAddressTableDetails, "id", "address_id"); } } + + public void createQuestionsForProgramsAndEncounters() throws Exception { SyncStatus syncStatus = getInitialSyncStatus(); if (syncStatus != SyncStatus.COMPLETE) { throw new RuntimeException("Database sync is not complete. Cannot create questions."); } + Database database = getGlobalDatabase(); List programAndEncounterNames = getProgramAndEncounterNames(); + TableDetails addressTableDetails = new TableDetails(); + addressTableDetails.setName("Address"); + + TableDetails fetchedAddressTableDetails = databaseRepository.findTableDetailsByName(database, addressTableDetails); + for (String programName : programAndEncounterNames) { - addressQuestionCreationService.createQuestionForTable(programName, "Address", "id", "address_id"); + TableDetails programTableDetails = new TableDetails(); + programTableDetails.setName(programName); + + TableDetails fetchedProgramTableDetails = databaseRepository.findTableDetailsByName(database, programTableDetails); + + addressQuestionCreationService.createQuestionForTable(fetchedProgramTableDetails, fetchedAddressTableDetails, "id", "address_id"); } } + + public void createQuestionsForIndividualTables() { SyncStatus syncStatus = getInitialSyncStatus(); if (syncStatus != SyncStatus.COMPLETE) { @@ -138,13 +173,15 @@ public void createQuestionsForIndividualTables() { Database database = getGlobalDatabase(); - List individualTables = Arrays.asList("address", "media", "sync_telemetry"); // Adjust these as needed + List individualTables = Arrays.asList("address", "media", "sync_telemetry"); for (String tableName : individualTables) { - TableDetails tableDetails = databaseRepository.getTableDetailsByName(database, tableName); + TableDetails tableDetails = new TableDetails(); + tableDetails.setName(tableName); + TableDetails fetchedTableDetails = databaseRepository.findTableDetailsByName(database, tableDetails); MetabaseQuery query = new MetabaseQueryBuilder(database, objectMapper.createArrayNode(), objectMapper) - .forTable(tableDetails) + .forTable(fetchedTableDetails) .build(); MetabaseRequestBody requestBody = new MetabaseRequestBody( diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java index 121bfe158..1755ff964 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java @@ -1,6 +1,7 @@ package org.avni.server.service.metabase; +import org.avni.server.domain.metabase.TableDetails; public interface QuestionCreationService { - void createQuestionForTable(String tableName, String addressTableName, String addressField, String tableField) throws Exception; + void createQuestionForTable(TableDetails tableDetails, TableDetails addressTableDetails, String addressField, String tableField) throws Exception; void createQuestionForTable(String tableName, String schema) throws Exception; } From 19b66fd5515b5fe5c4472677e3dc7eb71cfb1421 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sun, 8 Sep 2024 12:08:01 +0530 Subject: [PATCH 26/34] avniproject#762 | Dataset Request Body Created --- .../dao/metabase/DatabaseRepository.java | 19 ++++++------ .../domain/metabase/DatasetRequestBody.java | 29 +++++++++++++++++++ .../server/domain/metabase/FieldDetails.java | 8 +++++ .../AddressQuestionCreationService.java | 4 +-- .../service/metabase/DatabaseService.java | 16 ++++++++-- .../metabase/QuestionCreationService.java | 3 +- 6 files changed, 65 insertions(+), 14 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/DatasetRequestBody.java diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java index 3b20ad76d..ce957a560 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java @@ -78,9 +78,9 @@ public CollectionInfoResponse getCollectionByName(String collectionName) { } } - public void createQuestionForTable(Database database, TableDetails tableDetails, TableDetails addressTableDetails, String addressField, String tableField) { - FieldDetails joinField1 = getFieldDetailsByName(database, addressTableDetails.getName(), addressField); - FieldDetails joinField2 = getFieldDetailsByName(database, tableDetails.getName(), tableField); + public void createQuestionForTable(Database database, TableDetails tableDetails, TableDetails addressTableDetails, FieldDetails addressField, FieldDetails tableField) { + FieldDetails joinField1 = getFieldDetailsByName(database, addressTableDetails, addressField); + FieldDetails joinField2 = getFieldDetailsByName(database, tableDetails, tableField); ArrayNode joinsArray = objectMapper.createArrayNode(); MetabaseQuery query = new MetabaseQueryBuilder(database, joinsArray, objectMapper) @@ -101,14 +101,14 @@ public void createQuestionForTable(Database database, TableDetails tableDetails, postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); } - public FieldDetails getFieldDetailsByName(Database database, String tableName, String fieldName) { + public FieldDetails getFieldDetailsByName(Database database, TableDetails tableDetails, FieldDetails fieldDetails) { List fieldsList = getFields(database); - String snakeCaseTableName = S.toSnakeCase(tableName); + String snakeCaseTableName = S.toSnakeCase(tableDetails.getName()); return fieldsList.stream() - .filter(field -> snakeCaseTableName.equals(field.getTableName()) && fieldName.equals(field.getName())) + .filter(field -> snakeCaseTableName.equals(field.getTableName()) && fieldDetails.getName().equals(field.getName())) .findFirst() - .orElseThrow(() -> new RuntimeException("Field " + fieldName + " not found in table " + tableName)); + .orElseThrow(() -> new RuntimeException("Field " + fieldDetails.getName() + " not found in table " + tableDetails.getName())); } public MetabaseDatabaseInfo getDatabaseDetails(Database database) { @@ -166,7 +166,8 @@ public DatasetResponse findAll(TableDetails table, Database database) { return getDataset(requestBody); } - private String createRequestBodyForDataset(Database database,TableDetails table) { - return "{\"database\":" + database.getId() + ",\"query\":{\"source-table\":" + table.getId() + "},\"type\":\"query\",\"parameters\":[]}"; + private String createRequestBodyForDataset(Database database, TableDetails table) { + DatasetRequestBody requestBody = new DatasetRequestBody(database, table); + return requestBody.toJson(objectMapper).toString(); } } diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatasetRequestBody.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatasetRequestBody.java new file mode 100644 index 000000000..75846eb05 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatasetRequestBody.java @@ -0,0 +1,29 @@ +package org.avni.server.domain.metabase; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class DatasetRequestBody { + + private final Database database; + private final TableDetails table; + + public DatasetRequestBody(Database database, TableDetails table) { + this.database = database; + this.table = table; + } + + public ObjectNode toJson(ObjectMapper objectMapper) { + ObjectNode rootNode = objectMapper.createObjectNode(); + rootNode.put("database", database.getId()); + + ObjectNode queryNode = objectMapper.createObjectNode(); + queryNode.put("source-table", table.getId()); + + rootNode.set("query", queryNode); + rootNode.put("type", "query"); + rootNode.set("parameters", objectMapper.createArrayNode()); + + return rootNode; + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldDetails.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldDetails.java index 94b8e5827..5b723307b 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldDetails.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldDetails.java @@ -9,9 +9,17 @@ public class FieldDetails { @JsonProperty("id") private int id; + public void setId(int id) { + this.id = id; + } + @JsonProperty("name") private String name; + public void setName(String name) { + this.name = name; + } + @JsonProperty("display_name") private String displayName; diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java index 2e8d35115..0449ebb79 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java @@ -26,9 +26,9 @@ public AddressQuestionCreationService(@Lazy DatabaseService databaseService, Dat } @Override - public void createQuestionForTable(TableDetails tableDetails, TableDetails addressTableDetails, String addressField, String tableField) throws Exception { + public void createQuestionForTable(TableDetails tableDetails, TableDetails addressTableDetails, FieldDetails addressFieldDetails, FieldDetails tableFieldDetails) throws Exception { Database database = databaseService.getGlobalDatabase(); - databaseRepository.createQuestionForTable(database, tableDetails, addressTableDetails, addressField, tableField); + databaseRepository.createQuestionForTable(database, tableDetails, addressTableDetails, addressFieldDetails, tableFieldDetails); } diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index c7af8bb69..d32c979d9 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -127,13 +127,19 @@ public void createQuestionsForSubjectTypes() throws Exception { TableDetails fetchedAddressTableDetails = databaseRepository.findTableDetailsByName(database, addressTableDetails); + FieldDetails addressFieldDetails = new FieldDetails(); + addressFieldDetails.setName("id"); + + FieldDetails subjectFieldDetails = new FieldDetails(); + subjectFieldDetails.setName("address_id"); + for (String subjectTypeName : subjectTypeNames) { TableDetails subjectTableDetails = new TableDetails(); subjectTableDetails.setName(subjectTypeName); TableDetails fetchedSubjectTableDetails = databaseRepository.findTableDetailsByName(database, subjectTableDetails); - addressQuestionCreationService.createQuestionForTable(fetchedSubjectTableDetails, fetchedAddressTableDetails, "id", "address_id"); + addressQuestionCreationService.createQuestionForTable(fetchedSubjectTableDetails, fetchedAddressTableDetails, addressFieldDetails, subjectFieldDetails); } } @@ -153,13 +159,19 @@ public void createQuestionsForProgramsAndEncounters() throws Exception { TableDetails fetchedAddressTableDetails = databaseRepository.findTableDetailsByName(database, addressTableDetails); + FieldDetails addressFieldDetails = new FieldDetails(); + addressFieldDetails.setName("id"); + + FieldDetails programFieldDetails = new FieldDetails(); + programFieldDetails.setName("address_id"); + for (String programName : programAndEncounterNames) { TableDetails programTableDetails = new TableDetails(); programTableDetails.setName(programName); TableDetails fetchedProgramTableDetails = databaseRepository.findTableDetailsByName(database, programTableDetails); - addressQuestionCreationService.createQuestionForTable(fetchedProgramTableDetails, fetchedAddressTableDetails, "id", "address_id"); + addressQuestionCreationService.createQuestionForTable(fetchedProgramTableDetails, fetchedAddressTableDetails, addressFieldDetails, programFieldDetails); } } diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java index 1755ff964..8ae178f68 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/QuestionCreationService.java @@ -1,7 +1,8 @@ package org.avni.server.service.metabase; +import org.avni.server.domain.metabase.FieldDetails; import org.avni.server.domain.metabase.TableDetails; public interface QuestionCreationService { - void createQuestionForTable(TableDetails tableDetails, TableDetails addressTableDetails, String addressField, String tableField) throws Exception; + void createQuestionForTable(TableDetails tableDetails, TableDetails addressTableDetails, FieldDetails addressFieldDetails, FieldDetails tableFieldDetails) throws Exception; void createQuestionForTable(String tableName, String schema) throws Exception; } From 05cf2ede0efa635aa0b45a24b6ef9cae0e976a05 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sun, 8 Sep 2024 18:12:20 +0530 Subject: [PATCH 27/34] avniproject#762 | Corrected the QuestionCreation RequestBody --- .../dao/metabase/DatabaseRepository.java | 21 +++++++++++-------- .../domain/metabase/MetabaseQueryBuilder.java | 10 ++++----- .../domain/metabase/MetabaseRequestBody.java | 5 +---- .../AddressQuestionCreationService.java | 3 +-- .../service/metabase/DatabaseService.java | 4 +--- .../service/metabase/MetabaseService.java | 6 +----- 6 files changed, 21 insertions(+), 28 deletions(-) diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java index ce957a560..3686e78c9 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java @@ -89,18 +89,21 @@ public void createQuestionForTable(Database database, TableDetails tableDetails, .build(); MetabaseRequestBody requestBody = new MetabaseRequestBody( - "Address + " + tableDetails.getDisplayName(), + tableDetails.getDisplayName(), query, VisualizationType.TABLE, null, objectMapper.createObjectNode(), - getCollectionByName(database.getName()).getIdAsInt(), - CardType.QUESTION + getCollectionByName(database.getName()).getIdAsInt() ); + System.out.println("Final Request Body: " + requestBody.toJson(objectMapper).toPrettyString()); + postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); } + + public FieldDetails getFieldDetailsByName(Database database, TableDetails tableDetails, FieldDetails fieldDetails) { List fieldsList = getFields(database); String snakeCaseTableName = S.toSnakeCase(tableDetails.getName()); @@ -151,9 +154,10 @@ public DatabaseSyncStatus getInitialSyncStatus(Database database) { } } - public DatasetResponse getDataset(String requestBody) { + public DatasetResponse getDataset(DatasetRequestBody requestBody) { String url = metabaseApiUrl + "/dataset"; - String jsonResponse = postForObject(url, requestBody, String.class); + String jsonRequestBody = requestBody.toJson(objectMapper).toString(); + String jsonResponse = postForObject(url, jsonRequestBody, String.class); try { return objectMapper.readValue(jsonResponse, DatasetResponse.class); } catch (Exception e) { @@ -162,12 +166,11 @@ public DatasetResponse getDataset(String requestBody) { } public DatasetResponse findAll(TableDetails table, Database database) { - String requestBody = createRequestBodyForDataset(database, table); + DatasetRequestBody requestBody = createRequestBodyForDataset(database, table); return getDataset(requestBody); } - private String createRequestBodyForDataset(Database database, TableDetails table) { - DatasetRequestBody requestBody = new DatasetRequestBody(database, table); - return requestBody.toJson(objectMapper).toString(); + private DatasetRequestBody createRequestBodyForDataset(Database database, TableDetails table) { + return new DatasetRequestBody(database, table); } } diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java index e48ab94bc..d5a417bd1 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java @@ -19,11 +19,10 @@ public MetabaseQueryBuilder(Database database, ArrayNode joinsArray, ObjectMappe public MetabaseQueryBuilder forTable(TableDetails tableDetails) { queryNode.put("source-table", tableDetails.getId()); - queryNode.put("database", database.getId()); return this; } - public MetabaseQueryBuilder joinWith(TableDetails addressTable, FieldDetails joinField1, FieldDetails joinField2) { + public MetabaseQueryBuilder joinWith(TableDetails addressTable, FieldDetails joinField1, FieldDetails joinField2) { ObjectNode joinNode = objectMapper.createObjectNode(); joinNode.put("fields", "all"); joinNode.put("alias", addressTable.getName()); @@ -34,24 +33,25 @@ public MetabaseQueryBuilder joinWith(TableDetails addressTable, FieldDetails jo ArrayNode leftField = objectMapper.createArrayNode(); leftField.add("field"); - leftField.add(joinField1.getId()); + leftField.add(joinField2.getId()); leftField.add(objectMapper.createObjectNode().put("base-type", "type/Integer")); conditionArray.add(leftField); ArrayNode rightField = objectMapper.createArrayNode(); rightField.add("field"); - rightField.add(joinField2.getId()); + rightField.add(joinField1.getId()); rightField.add(objectMapper.createObjectNode().put("base-type", "type/Integer").put("join-alias", addressTable.getName())); conditionArray.add(rightField); joinNode.set("condition", conditionArray); joinsArray.add(joinNode); queryNode.set("joins", joinsArray); + return this; } + public MetabaseQuery build() { - queryNode.put("type", "query"); return new MetabaseQuery(database.getId(), queryNode); } } diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java index d5ab1fbc1..6076c9ed2 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java @@ -13,16 +13,14 @@ public class MetabaseRequestBody { private int collectionId; private Integer collectionPosition; private JsonNode resultMetadata; - private CardType cardType; - public MetabaseRequestBody(String name, MetabaseQuery datasetQuery, VisualizationType display, String description, ObjectNode visualizationSettings, int collectionId, CardType cardType) { + public MetabaseRequestBody(String name, MetabaseQuery datasetQuery, VisualizationType display, String description, ObjectNode visualizationSettings, int collectionId) { this.name = name; this.datasetQuery = datasetQuery; this.display = display; this.description = description; this.visualizationSettings = visualizationSettings; this.collectionId = collectionId; - this.cardType = cardType; } public ObjectNode toJson(ObjectMapper objectMapper) { @@ -36,7 +34,6 @@ public ObjectNode toJson(ObjectMapper objectMapper) { rootNode.set("dataset_query", datasetQueryNode); rootNode.put("display", display.toString()); - rootNode.put("type", cardType.getType()); rootNode.putNull("description"); rootNode.set("visualization_settings", visualizationSettings); rootNode.put("collection_id", collectionId); diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java index 0449ebb79..b88db3481 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java @@ -50,8 +50,7 @@ public void createQuestionForTable(String tableName, String schema) throws Excep VisualizationType.TABLE, null, objectMapper.createObjectNode(), - databaseService.getCollectionId(), - CardType.QUESTION + databaseService.getCollectionId() ); databaseRepository.postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); } diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index d32c979d9..5bc128e8f 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -202,15 +202,13 @@ public void createQuestionsForIndividualTables() { VisualizationType.TABLE, null, objectMapper.createObjectNode(), - databaseRepository.getCollectionByName(database.getName()).getIdAsInt(), - CardType.QUESTION + databaseRepository.getCollectionByName(database.getName()).getIdAsInt() ); databaseRepository.postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); } } - public void createQuestions() throws Exception { createQuestionsForSubjectTypes(); diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java index 746267e41..cdd3677b9 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java @@ -64,10 +64,6 @@ public void setupMetabase() { collectionPermissionsRepository.updateCollectionPermissions(collectionPermissions, metabaseGroup.getId(), metabaseCollection.getId()); } - public void createQuestionsForSubjectTypes() throws Exception{ - databaseService.createQuestionsForSubjectTypes(); - } - public int getGlobalDatabaseId() { if (globalDatabase == null) { Organisation currentOrganisation = organisationService.getCurrentOrganisation(); @@ -79,7 +75,7 @@ public int getGlobalDatabaseId() { public int getGlobalCollectionId() { if (globalCollection == null) { Organisation currentOrganisation = organisationService.getCurrentOrganisation(); - globalCollection = databaseRepository.getCollectionByName(currentOrganisation.getName() + " collection"); + globalCollection = databaseRepository.getCollectionByName(currentOrganisation.getName()); } return globalCollection.getIdAsInt(); } From 9a185c448283c6ba30233b5f1a3b4da23f418bd1 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sun, 8 Sep 2024 18:36:04 +0530 Subject: [PATCH 28/34] avniproject#762 | Removed primitive obsession from getDatabaseByName --- .../avni/server/dao/metabase/DatabaseRepository.java | 10 +++------- .../avni/server/service/metabase/MetabaseService.java | 4 +++- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java index 3686e78c9..7aacd1eb7 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java @@ -37,7 +37,7 @@ public Database getDatabaseById(Database database) { } } - public Database getDatabaseByName(String name) { + public Database getDatabaseByName(Database database) { String url = metabaseApiUrl + "/database"; String jsonResponse = getForObject(url, String.class); @@ -49,11 +49,11 @@ public Database getDatabaseByName(String name) { for (JsonNode dbNode : dataArray) { Database db = objectMapper.treeToValue(dbNode, Database.class); - if (db.getName().equals(name)) { + if (db.getName().equals(database.getName())) { return db; } } - throw new RuntimeException("Database with name " + name + " not found."); + throw new RuntimeException("Database with name " + database.getName() + " not found."); } catch (Exception e) { throw new RuntimeException("Failed to retrieve database", e); } @@ -97,13 +97,9 @@ public void createQuestionForTable(Database database, TableDetails tableDetails, getCollectionByName(database.getName()).getIdAsInt() ); - System.out.println("Final Request Body: " + requestBody.toJson(objectMapper).toPrettyString()); - postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); } - - public FieldDetails getFieldDetailsByName(Database database, TableDetails tableDetails, FieldDetails fieldDetails) { List fieldsList = getFields(database); String snakeCaseTableName = S.toSnakeCase(tableDetails.getName()); diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java index cdd3677b9..96a03efdf 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java @@ -67,7 +67,9 @@ public void setupMetabase() { public int getGlobalDatabaseId() { if (globalDatabase == null) { Organisation currentOrganisation = organisationService.getCurrentOrganisation(); - globalDatabase = databaseRepository.getDatabaseByName(currentOrganisation.getName()); + Database database = new Database(); + database.setName(currentOrganisation.getName()); + globalDatabase = databaseRepository.getDatabaseByName(database); } return globalDatabase.getId(); } From 4f382cc97502d5b6f1bdc5473ae1df940b260b26 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sun, 8 Sep 2024 18:49:30 +0530 Subject: [PATCH 29/34] avniproject#762 | Removed primitive obsession from getCollectionByName --- .../org/avni/server/dao/metabase/DatabaseRepository.java | 8 ++++---- .../org/avni/server/service/metabase/DatabaseService.java | 2 +- .../org/avni/server/service/metabase/MetabaseService.java | 4 +++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java index 7aacd1eb7..066860a6e 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java @@ -59,7 +59,7 @@ public Database getDatabaseByName(Database database) { } } - public CollectionInfoResponse getCollectionByName(String collectionName) { + public CollectionInfoResponse getCollectionByName(Database database) { String url = metabaseApiUrl + "/collection"; try { String jsonResponse = getForObject(url, String.class); @@ -70,9 +70,9 @@ public CollectionInfoResponse getCollectionByName(String collectionName) { ); return collections.stream() - .filter(collection -> collection.getName().equals(collectionName)) + .filter(collection -> collection.getName().equals(database.getName())) .findFirst() - .orElseThrow(() -> new RuntimeException("Collection with name " + collectionName + " not found.")); + .orElseThrow(() -> new RuntimeException("Collection with name " + database.getName() + " not found.")); } catch (Exception e) { throw new RuntimeException("Failed to retrieve collection", e); } @@ -94,7 +94,7 @@ public void createQuestionForTable(Database database, TableDetails tableDetails, VisualizationType.TABLE, null, objectMapper.createObjectNode(), - getCollectionByName(database.getName()).getIdAsInt() + getCollectionByName(database).getIdAsInt() ); postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 5bc128e8f..20f9e2e63 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -202,7 +202,7 @@ public void createQuestionsForIndividualTables() { VisualizationType.TABLE, null, objectMapper.createObjectNode(), - databaseRepository.getCollectionByName(database.getName()).getIdAsInt() + databaseRepository.getCollectionByName(database).getIdAsInt() ); databaseRepository.postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java index 96a03efdf..429e7ad3c 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java @@ -77,7 +77,9 @@ public int getGlobalDatabaseId() { public int getGlobalCollectionId() { if (globalCollection == null) { Organisation currentOrganisation = organisationService.getCurrentOrganisation(); - globalCollection = databaseRepository.getCollectionByName(currentOrganisation.getName()); + Database database = new Database(); + database.setName(currentOrganisation.getName()); + globalCollection = databaseRepository.getCollectionByName(database); } return globalCollection.getIdAsInt(); } From 87c122f042092421fd716a86aa15e1f24b58a2bb Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Sun, 8 Sep 2024 23:14:21 +0530 Subject: [PATCH 30/34] avniproject#762 | DB and Collection are now fetched from DB if they arent cached --- .../avni/server/domain/metabase/Database.java | 11 +- .../server/domain/metabase/FieldDetails.java | 6 + .../server/domain/metabase/TableDetails.java | 6 + .../AddressQuestionCreationService.java | 3 - .../service/metabase/DatabaseService.java | 105 ++++++------------ .../service/metabase/MetabaseService.java | 20 +--- 6 files changed, 57 insertions(+), 94 deletions(-) diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/Database.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/Database.java index 009f07762..21d388917 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/Database.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/Database.java @@ -8,16 +8,21 @@ public class Database { private String name; private String engine; private DatabaseDetails details; - public Database() { + + } + + public Database(String name) { + this.name = name; } - + public Database(String name, String engine, DatabaseDetails details) { this(null,name,engine,details); } - public Database(Integer id,String name, String engine, DatabaseDetails details) { + + public Database(Integer id, String name, String engine, DatabaseDetails details) { this.id=id; this.name = name; this.engine = engine; diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldDetails.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldDetails.java index 5b723307b..9ae172d77 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldDetails.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldDetails.java @@ -5,6 +5,12 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class FieldDetails { + public FieldDetails() { + } + + public FieldDetails(String name) { + this.name = name; + } @JsonProperty("id") private int id; diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java index f2178fbc3..7bc0965fb 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableDetails.java @@ -5,6 +5,12 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class TableDetails { + public TableDetails() { + } + + public TableDetails(String name) { + this.name = name; + } @JsonProperty("description") private String description; diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java index b88db3481..306052614 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java @@ -31,7 +31,6 @@ public void createQuestionForTable(TableDetails tableDetails, TableDetails addre databaseRepository.createQuestionForTable(database, tableDetails, addressTableDetails, addressFieldDetails, tableFieldDetails); } - @Override public void createQuestionForTable(String tableName, String schema) throws Exception { Database database = databaseService.getGlobalDatabase(); @@ -54,6 +53,4 @@ public void createQuestionForTable(String tableName, String schema) throws Excep ); databaseRepository.postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); } - - } diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 20f9e2e63..f19dfdf00 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -18,8 +18,6 @@ public class DatabaseService { private final ObjectMapper objectMapper; private final MetabaseService metabaseService; private final AddressQuestionCreationService addressQuestionCreationService; - private Integer databaseId; - private Integer collectionId; @Value("${metabase.api.url}") private String metabaseApiUrl; @@ -27,6 +25,8 @@ public class DatabaseService { @Value("${metabase.api.key}") private String apiKey; + private static final String ADDRESS_TABLE = "Address"; + @Autowired public DatabaseService(DatabaseRepository databaseRepository, ObjectMapper objectMapper, MetabaseService metabaseService, AddressQuestionCreationService addressQuestionCreationService) { this.databaseRepository = databaseRepository; @@ -36,39 +36,32 @@ public DatabaseService(DatabaseRepository databaseRepository, ObjectMapper objec } public Database getGlobalDatabase() { - Database globalDatabase = metabaseService.getGlobalDatabase(); - return databaseRepository.getDatabaseById(globalDatabase); - } - - public int getDatabaseId() { - if (databaseId == null) { - databaseId = metabaseService.getGlobalDatabaseId(); - } - return databaseId; + return metabaseService.getGlobalDatabase(); } public int getCollectionId() { - if (collectionId == null) { - collectionId = metabaseService.getGlobalCollectionId(); - } - return collectionId; + return metabaseService.getGlobalCollection().getIdAsInt(); } public SyncStatus getInitialSyncStatus() { - Database globalDatabase = metabaseService.getGlobalDatabase(); - DatabaseSyncStatus databaseSyncStatus = databaseRepository.getInitialSyncStatus(globalDatabase); + DatabaseSyncStatus databaseSyncStatus = databaseRepository.getInitialSyncStatus(getGlobalDatabase()); String status = databaseSyncStatus.getInitialSyncStatus(); return SyncStatus.fromString(status); } + private void ensureSyncComplete() { + SyncStatus syncStatus = getInitialSyncStatus(); + if (syncStatus != SyncStatus.COMPLETE) { + throw new RuntimeException("Database sync is not complete. Cannot create questions."); + } + } + public List getSubjectTypeNames() { - Database database = getGlobalDatabase(); - TableDetails metadataTable = new TableDetails(); - metadataTable.setName("table_metadata"); + TableDetails metadataTable = new TableDetails("table_metadata"); - TableDetails fetchedMetadataTable = databaseRepository.findTableDetailsByName(database, metadataTable); + TableDetails fetchedMetadataTable = databaseRepository.findTableDetailsByName(getGlobalDatabase(), metadataTable); - DatasetResponse datasetResponse = databaseRepository.findAll(fetchedMetadataTable, database); + DatasetResponse datasetResponse = databaseRepository.findAll(fetchedMetadataTable, getGlobalDatabase()); List> rows = datasetResponse.getData().getRows(); List subjectTypeNames = new ArrayList<>(); @@ -86,17 +79,12 @@ public List getSubjectTypeNames() { return subjectTypeNames; } - - - public List getProgramAndEncounterNames() { - Database database = getGlobalDatabase(); - TableDetails metadataTable = new TableDetails(); - metadataTable.setName("table_metadata"); + TableDetails metadataTable = new TableDetails("table_metadata"); - TableDetails fetchedMetadataTable = databaseRepository.findTableDetailsByName(database, metadataTable); + TableDetails fetchedMetadataTable = databaseRepository.findTableDetailsByName(getGlobalDatabase(), metadataTable); - DatasetResponse datasetResponse = databaseRepository.findAll(fetchedMetadataTable, database); + DatasetResponse datasetResponse = databaseRepository.findAll(fetchedMetadataTable, getGlobalDatabase()); List> rows = datasetResponse.getData().getRows(); List programNames = new ArrayList<>(); @@ -112,76 +100,45 @@ public List getProgramAndEncounterNames() { return programNames; } - public void createQuestionsForSubjectTypes() throws Exception { - SyncStatus syncStatus = getInitialSyncStatus(); - if (syncStatus != SyncStatus.COMPLETE) { - throw new RuntimeException("Database sync is not complete. Cannot create questions."); - } - - Database database = getGlobalDatabase(); + ensureSyncComplete(); List subjectTypeNames = getSubjectTypeNames(); - TableDetails addressTableDetails = new TableDetails(); - addressTableDetails.setName("Address"); - - TableDetails fetchedAddressTableDetails = databaseRepository.findTableDetailsByName(database, addressTableDetails); + TableDetails addressTableDetails = new TableDetails(ADDRESS_TABLE); + TableDetails fetchedAddressTableDetails = databaseRepository.findTableDetailsByName(getGlobalDatabase(), addressTableDetails); - FieldDetails addressFieldDetails = new FieldDetails(); - addressFieldDetails.setName("id"); + FieldDetails addressFieldDetails = new FieldDetails("id"); + FieldDetails subjectFieldDetails = new FieldDetails("address_id"); - FieldDetails subjectFieldDetails = new FieldDetails(); - subjectFieldDetails.setName("address_id"); for (String subjectTypeName : subjectTypeNames) { TableDetails subjectTableDetails = new TableDetails(); subjectTableDetails.setName(subjectTypeName); - - TableDetails fetchedSubjectTableDetails = databaseRepository.findTableDetailsByName(database, subjectTableDetails); - + TableDetails fetchedSubjectTableDetails = databaseRepository.findTableDetailsByName(getGlobalDatabase(), subjectTableDetails); addressQuestionCreationService.createQuestionForTable(fetchedSubjectTableDetails, fetchedAddressTableDetails, addressFieldDetails, subjectFieldDetails); } } - - public void createQuestionsForProgramsAndEncounters() throws Exception { - SyncStatus syncStatus = getInitialSyncStatus(); - if (syncStatus != SyncStatus.COMPLETE) { - throw new RuntimeException("Database sync is not complete. Cannot create questions."); - } - - Database database = getGlobalDatabase(); + ensureSyncComplete(); List programAndEncounterNames = getProgramAndEncounterNames(); - TableDetails addressTableDetails = new TableDetails(); - addressTableDetails.setName("Address"); - - TableDetails fetchedAddressTableDetails = databaseRepository.findTableDetailsByName(database, addressTableDetails); - - FieldDetails addressFieldDetails = new FieldDetails(); - addressFieldDetails.setName("id"); + TableDetails addressTableDetails = new TableDetails(ADDRESS_TABLE); + TableDetails fetchedAddressTableDetails = databaseRepository.findTableDetailsByName(getGlobalDatabase(), addressTableDetails); - FieldDetails programFieldDetails = new FieldDetails(); - programFieldDetails.setName("address_id"); + FieldDetails addressFieldDetails = new FieldDetails("id"); + FieldDetails programFieldDetails = new FieldDetails("address_id"); for (String programName : programAndEncounterNames) { TableDetails programTableDetails = new TableDetails(); programTableDetails.setName(programName); - - TableDetails fetchedProgramTableDetails = databaseRepository.findTableDetailsByName(database, programTableDetails); - + TableDetails fetchedProgramTableDetails = databaseRepository.findTableDetailsByName(getGlobalDatabase(), programTableDetails); addressQuestionCreationService.createQuestionForTable(fetchedProgramTableDetails, fetchedAddressTableDetails, addressFieldDetails, programFieldDetails); } } - - public void createQuestionsForIndividualTables() { - SyncStatus syncStatus = getInitialSyncStatus(); - if (syncStatus != SyncStatus.COMPLETE) { - throw new RuntimeException("Database sync is not complete. Cannot create questions."); - } + ensureSyncComplete(); Database database = getGlobalDatabase(); diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java index 429e7ad3c..050ee873d 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/MetabaseService.java @@ -64,27 +64,19 @@ public void setupMetabase() { collectionPermissionsRepository.updateCollectionPermissions(collectionPermissions, metabaseGroup.getId(), metabaseCollection.getId()); } - public int getGlobalDatabaseId() { + public Database getGlobalDatabase() { if (globalDatabase == null) { Organisation currentOrganisation = organisationService.getCurrentOrganisation(); - Database database = new Database(); - database.setName(currentOrganisation.getName()); - globalDatabase = databaseRepository.getDatabaseByName(database); + globalDatabase = databaseRepository.getDatabaseByName(new Database(currentOrganisation.getName())); } - return globalDatabase.getId(); + return globalDatabase; } - public int getGlobalCollectionId() { + public CollectionInfoResponse getGlobalCollection() { if (globalCollection == null) { Organisation currentOrganisation = organisationService.getCurrentOrganisation(); - Database database = new Database(); - database.setName(currentOrganisation.getName()); - globalCollection = databaseRepository.getCollectionByName(database); + globalCollection = databaseRepository.getCollectionByName(new Database(currentOrganisation.getName())); } - return globalCollection.getIdAsInt(); - } - - public Database getGlobalDatabase() { - return globalDatabase; + return globalCollection; } } From dbef0752797de8bb5f0994911c2891a54874f7b2 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Mon, 9 Sep 2024 00:15:37 +0530 Subject: [PATCH 31/34] avniproject#762 | Fixed question creation implementation for individual tables : Its now using AddressQuestionCreationService --- .../dao/metabase/DatabaseRepository.java | 35 +++++++----- .../AddressQuestionCreationService.java | 27 ++------- .../service/metabase/DatabaseService.java | 57 ++++--------------- ...tUtilTest.java => CollectionUtilTest.java} | 2 +- 4 files changed, 36 insertions(+), 85 deletions(-) rename avni-server-api/src/test/java/org/avni/server/util/{CreateCollectionRequestUtilTest.java => CollectionUtilTest.java} (93%) diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java index 066860a6e..d8cd74566 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java @@ -25,18 +25,6 @@ public Database save(Database database) { return database; } - public Database getDatabaseById(Database database) { - int id = database.getId(); - String url = metabaseApiUrl + "/database/" + id; - try { - String jsonResponse = getForObject(url, String.class); - ObjectMapper objectMapper = new ObjectMapper(); - return objectMapper.readValue(jsonResponse, Database.class); - } catch (Exception e) { - throw new RuntimeException("Failed to retrieve database with ID " + id, e); - } - } - public Database getDatabaseByName(Database database) { String url = metabaseApiUrl + "/database"; @@ -78,9 +66,9 @@ public CollectionInfoResponse getCollectionByName(Database database) { } } - public void createQuestionForTable(Database database, TableDetails tableDetails, TableDetails addressTableDetails, FieldDetails addressField, FieldDetails tableField) { - FieldDetails joinField1 = getFieldDetailsByName(database, addressTableDetails, addressField); - FieldDetails joinField2 = getFieldDetailsByName(database, tableDetails, tableField); + public void createQuestionForTable(Database database, TableDetails tableDetails, TableDetails addressTableDetails, FieldDetails originField, FieldDetails destinationField) { + FieldDetails joinField1 = getFieldDetailsByName(database, addressTableDetails, originField); + FieldDetails joinField2 = getFieldDetailsByName(database, tableDetails, destinationField); ArrayNode joinsArray = objectMapper.createArrayNode(); MetabaseQuery query = new MetabaseQueryBuilder(database, joinsArray, objectMapper) @@ -100,6 +88,23 @@ public void createQuestionForTable(Database database, TableDetails tableDetails, postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); } + public void createQuestionForIndividualTable(Database database, TableDetails tableDetails) { + MetabaseQuery query = new MetabaseQueryBuilder(database, objectMapper.createArrayNode(), objectMapper) + .forTable(tableDetails) + .build(); + + MetabaseRequestBody requestBody = new MetabaseRequestBody( + tableDetails.getDisplayName(), + query, + VisualizationType.TABLE, + null, + objectMapper.createObjectNode(), + getCollectionByName(database).getIdAsInt() + ); + + postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); + } + public FieldDetails getFieldDetailsByName(Database database, TableDetails tableDetails, FieldDetails fieldDetails) { List fieldsList = getFields(database); String snakeCaseTableName = S.toSnakeCase(tableDetails.getName()); diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java index 306052614..676b3192c 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java @@ -1,11 +1,8 @@ package org.avni.server.service.metabase; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import org.avni.server.dao.metabase.DatabaseRepository; import org.avni.server.domain.metabase.*; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; @@ -14,43 +11,27 @@ public class AddressQuestionCreationService implements QuestionCreationService { private final DatabaseService databaseService; private final DatabaseRepository databaseRepository; - private final ObjectMapper objectMapper; - private final String metabaseApiUrl; @Autowired - public AddressQuestionCreationService(@Lazy DatabaseService databaseService, DatabaseRepository databaseRepository, ObjectMapper objectMapper, @Value("${metabase.api.url}") String metabaseApiUrl) { + public AddressQuestionCreationService(@Lazy DatabaseService databaseService, DatabaseRepository databaseRepository) { this.databaseService = databaseService; this.databaseRepository = databaseRepository; - this.objectMapper = objectMapper; - this.metabaseApiUrl = metabaseApiUrl; } @Override - public void createQuestionForTable(TableDetails tableDetails, TableDetails addressTableDetails, FieldDetails addressFieldDetails, FieldDetails tableFieldDetails) throws Exception { + public void createQuestionForTable(TableDetails tableDetails, TableDetails addressTableDetails, FieldDetails addressFieldDetails, FieldDetails tableFieldDetails) { Database database = databaseService.getGlobalDatabase(); databaseRepository.createQuestionForTable(database, tableDetails, addressTableDetails, addressFieldDetails, tableFieldDetails); } @Override - public void createQuestionForTable(String tableName, String schema) throws Exception { + public void createQuestionForTable(String tableName, String schema) { Database database = databaseService.getGlobalDatabase(); TableDetails tableDetails = new TableDetails(); tableDetails.setName(tableName); TableDetails fetchedTableDetails = databaseRepository.findTableDetailsByName(database, tableDetails); - MetabaseQuery query = new MetabaseQueryBuilder(database, objectMapper.createArrayNode(), objectMapper) - .forTable(fetchedTableDetails) - .build(); - - MetabaseRequestBody requestBody = new MetabaseRequestBody( - tableName, - query, - VisualizationType.TABLE, - null, - objectMapper.createObjectNode(), - databaseService.getCollectionId() - ); - databaseRepository.postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); + databaseRepository.createQuestionForIndividualTable(database, fetchedTableDetails); } } diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index f19dfdf00..784936d15 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -1,11 +1,8 @@ package org.avni.server.service.metabase; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import org.avni.server.dao.metabase.DatabaseRepository; import org.avni.server.domain.metabase.*; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.util.Arrays; import java.util.ArrayList; @@ -15,22 +12,14 @@ public class DatabaseService { private final DatabaseRepository databaseRepository; - private final ObjectMapper objectMapper; private final MetabaseService metabaseService; private final AddressQuestionCreationService addressQuestionCreationService; - @Value("${metabase.api.url}") - private String metabaseApiUrl; - - @Value("${metabase.api.key}") - private String apiKey; - private static final String ADDRESS_TABLE = "Address"; @Autowired - public DatabaseService(DatabaseRepository databaseRepository, ObjectMapper objectMapper, MetabaseService metabaseService, AddressQuestionCreationService addressQuestionCreationService) { + public DatabaseService(DatabaseRepository databaseRepository, MetabaseService metabaseService, AddressQuestionCreationService addressQuestionCreationService) { this.databaseRepository = databaseRepository; - this.objectMapper = objectMapper; this.metabaseService = metabaseService; this.addressQuestionCreationService = addressQuestionCreationService; } @@ -39,8 +28,8 @@ public Database getGlobalDatabase() { return metabaseService.getGlobalDatabase(); } - public int getCollectionId() { - return metabaseService.getGlobalCollection().getIdAsInt(); + public CollectionInfoResponse getGlobalCollection() { + return metabaseService.getGlobalCollection(); } public SyncStatus getInitialSyncStatus() { @@ -57,9 +46,7 @@ private void ensureSyncComplete() { } public List getSubjectTypeNames() { - TableDetails metadataTable = new TableDetails("table_metadata"); - - TableDetails fetchedMetadataTable = databaseRepository.findTableDetailsByName(getGlobalDatabase(), metadataTable); + TableDetails fetchedMetadataTable = databaseRepository.findTableDetailsByName(getGlobalDatabase(), new TableDetails("table_metadata")); DatasetResponse datasetResponse = databaseRepository.findAll(fetchedMetadataTable, getGlobalDatabase()); List> rows = datasetResponse.getData().getRows(); @@ -80,9 +67,7 @@ public List getSubjectTypeNames() { } public List getProgramAndEncounterNames() { - TableDetails metadataTable = new TableDetails("table_metadata"); - - TableDetails fetchedMetadataTable = databaseRepository.findTableDetailsByName(getGlobalDatabase(), metadataTable); + TableDetails fetchedMetadataTable = databaseRepository.findTableDetailsByName(getGlobalDatabase(), new TableDetails("table_metadata")); DatasetResponse datasetResponse = databaseRepository.findAll(fetchedMetadataTable, getGlobalDatabase()); List> rows = datasetResponse.getData().getRows(); @@ -100,7 +85,7 @@ public List getProgramAndEncounterNames() { return programNames; } - public void createQuestionsForSubjectTypes() throws Exception { + public void createQuestionsForSubjectTypes(){ ensureSyncComplete(); List subjectTypeNames = getSubjectTypeNames(); @@ -119,12 +104,11 @@ public void createQuestionsForSubjectTypes() throws Exception { } } - public void createQuestionsForProgramsAndEncounters() throws Exception { + public void createQuestionsForProgramsAndEncounters(){ ensureSyncComplete(); List programAndEncounterNames = getProgramAndEncounterNames(); - TableDetails addressTableDetails = new TableDetails(ADDRESS_TABLE); - TableDetails fetchedAddressTableDetails = databaseRepository.findTableDetailsByName(getGlobalDatabase(), addressTableDetails); + TableDetails fetchedAddressTableDetails = databaseRepository.findTableDetailsByName(getGlobalDatabase(), new TableDetails(ADDRESS_TABLE)); FieldDetails addressFieldDetails = new FieldDetails("id"); FieldDetails programFieldDetails = new FieldDetails("address_id"); @@ -137,36 +121,17 @@ public void createQuestionsForProgramsAndEncounters() throws Exception { } } - public void createQuestionsForIndividualTables() { + public void createQuestionsForIndividualTables(){ ensureSyncComplete(); - Database database = getGlobalDatabase(); - List individualTables = Arrays.asList("address", "media", "sync_telemetry"); for (String tableName : individualTables) { - TableDetails tableDetails = new TableDetails(); - tableDetails.setName(tableName); - TableDetails fetchedTableDetails = databaseRepository.findTableDetailsByName(database, tableDetails); - - MetabaseQuery query = new MetabaseQueryBuilder(database, objectMapper.createArrayNode(), objectMapper) - .forTable(fetchedTableDetails) - .build(); - - MetabaseRequestBody requestBody = new MetabaseRequestBody( - tableName, - query, - VisualizationType.TABLE, - null, - objectMapper.createObjectNode(), - databaseRepository.getCollectionByName(database).getIdAsInt() - ); - - databaseRepository.postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); + addressQuestionCreationService.createQuestionForTable(tableName, "!public"); } } - public void createQuestions() throws Exception { + public void createQuestions() { createQuestionsForSubjectTypes(); createQuestionsForProgramsAndEncounters(); diff --git a/avni-server-api/src/test/java/org/avni/server/util/CreateCollectionRequestUtilTest.java b/avni-server-api/src/test/java/org/avni/server/util/CollectionUtilTest.java similarity index 93% rename from avni-server-api/src/test/java/org/avni/server/util/CreateCollectionRequestUtilTest.java rename to avni-server-api/src/test/java/org/avni/server/util/CollectionUtilTest.java index b0928ecb8..d6219aa2f 100644 --- a/avni-server-api/src/test/java/org/avni/server/util/CreateCollectionRequestUtilTest.java +++ b/avni-server-api/src/test/java/org/avni/server/util/CollectionUtilTest.java @@ -5,7 +5,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -public class CreateCollectionRequestUtilTest { +public class CollectionUtilTest { @Test public void hasOnlyTrailingEmptyStrings() { assertTrue(CollectionUtil.hasOnlyTrailingEmptyStrings(Arrays.asList("a", "b", ""))); From 381d921733753cf85394e2b82030b4daa5e01440 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:03:54 +0530 Subject: [PATCH 32/34] avniproject#762 | Added FieldAttributes Enum , used it in MQB , Renamed createQuestionForIndividualTable method , Deleted MetabaseJoin --- .../dao/metabase/DatabaseRepository.java | 2 +- .../avni/server/domain/metabase/CardType.java | 1 + .../domain/metabase/FieldAttribute.java | 31 +++++++++++++++ .../{BaseType.java => FieldType.java} | 6 ++- .../server/domain/metabase/MetabaseJoin.java | 39 ------------------- .../domain/metabase/MetabaseQueryBuilder.java | 22 +++++------ .../domain/metabase/MetabaseRequestBody.java | 1 + .../domain/metabase/VisualizationType.java | 1 + .../AddressQuestionCreationService.java | 2 +- 9 files changed, 51 insertions(+), 54 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldAttribute.java rename avni-server-api/src/main/java/org/avni/server/domain/metabase/{BaseType.java => FieldType.java} (63%) delete mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java index d8cd74566..f57e0d05b 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java @@ -88,7 +88,7 @@ public void createQuestionForTable(Database database, TableDetails tableDetails, postForObject(metabaseApiUrl + "/card", requestBody.toJson(objectMapper), JsonNode.class); } - public void createQuestionForIndividualTable(Database database, TableDetails tableDetails) { + public void createQuestionForASingleTable(Database database, TableDetails tableDetails) { MetabaseQuery query = new MetabaseQueryBuilder(database, objectMapper.createArrayNode(), objectMapper) .forTable(tableDetails) .build(); diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/CardType.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/CardType.java index 6afc67c28..2c45436ee 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/CardType.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/CardType.java @@ -1,5 +1,6 @@ package org.avni.server.domain.metabase; +//Refer Documentation : https://www.metabase.com/docs/latest/api/card#post-apicard public enum CardType { QUESTION("question"); diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldAttribute.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldAttribute.java new file mode 100644 index 000000000..e74428016 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldAttribute.java @@ -0,0 +1,31 @@ +package org.avni.server.domain.metabase; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public enum FieldAttribute { + FIELD("field"), + FIELDS("fields"), + SOURCE_TABLE("source-table"), + ALIAS("alias"), + CONDITION("condition"), + JOINS("joins"), + JOIN_ALIAS("join-alias"), + BASE_TYPE("base-type"); + + private final String attributeName; + + FieldAttribute(String attributeName) { + this.attributeName = attributeName; + } + + public String getAttributeName() { + return attributeName; + } + + public ObjectNode toJson(ObjectMapper objectMapper, Object value) { + ObjectNode attributeNode = objectMapper.createObjectNode(); + attributeNode.put(attributeName, value.toString()); + return attributeNode; + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/BaseType.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldType.java similarity index 63% rename from avni-server-api/src/main/java/org/avni/server/domain/metabase/BaseType.java rename to avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldType.java index e9404dd72..f4f93b6d4 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/BaseType.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldType.java @@ -1,13 +1,15 @@ package org.avni.server.domain.metabase; -public enum BaseType { +// Refer Documentation here : https://www.metabase.com/docs/latest/data-modeling/field-types + +public enum FieldType { INTEGER("type/Integer"), TEXT("type/Text"), BOOLEAN("type/Boolean"); private final String typeName; - BaseType(String typeName) { + FieldType(String typeName) { this.typeName = typeName; } diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java deleted file mode 100644 index f3bde2044..000000000 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseJoin.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.avni.server.domain.metabase; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; - -public class MetabaseJoin { - private String fields; - private String alias; - private int sourceTable; - private JsonNode condition; - - public MetabaseJoin(String fields, String alias, int sourceTable, int joinFieldId1, int joinFieldId2, String tableName, ObjectMapper objectMapper) throws Exception { - this.fields = fields; - this.alias = alias; - this.sourceTable = sourceTable; - - this.condition = createConditionNode(joinFieldId1, joinFieldId2, tableName, BaseType.INTEGER, objectMapper); - } - - private JsonNode createConditionNode(int joinFieldId1, int joinFieldId2, String tableName, BaseType baseType, ObjectMapper objectMapper) throws Exception { - ArrayNode conditionNode = objectMapper.createArrayNode(); - conditionNode.add(ConditionType.EQUAL.getOperator()); - conditionNode.add(objectMapper.createArrayNode().add("field").add(joinFieldId1).add(objectMapper.createObjectNode().put("base-type", baseType.getTypeName()))); - conditionNode.add(objectMapper.createArrayNode().add("field").add(joinFieldId2).add(objectMapper.createObjectNode().put("base-type", baseType.getTypeName()).put("join-alias", tableName))); - return conditionNode; - } - - public ObjectNode toJson(ObjectMapper objectMapper) { - ObjectNode joinNode = objectMapper.createObjectNode(); - joinNode.put("fields", this.fields); - joinNode.put("alias", this.alias); - joinNode.put("source-table", this.sourceTable); - joinNode.set("condition", this.condition); - - return joinNode; - } -} diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java index d5a417bd1..2f9ed77d3 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java @@ -18,34 +18,34 @@ public MetabaseQueryBuilder(Database database, ArrayNode joinsArray, ObjectMappe } public MetabaseQueryBuilder forTable(TableDetails tableDetails) { - queryNode.put("source-table", tableDetails.getId()); + queryNode.put(FieldAttribute.SOURCE_TABLE.getAttributeName(), tableDetails.getId()); return this; } public MetabaseQueryBuilder joinWith(TableDetails addressTable, FieldDetails joinField1, FieldDetails joinField2) { ObjectNode joinNode = objectMapper.createObjectNode(); - joinNode.put("fields", "all"); - joinNode.put("alias", addressTable.getName()); - joinNode.put("source-table", addressTable.getId()); + joinNode.put(FieldAttribute.FIELDS.getAttributeName(), "all"); + joinNode.put(FieldAttribute.ALIAS.getAttributeName(), addressTable.getName()); ArrayNode conditionArray = objectMapper.createArrayNode(); - conditionArray.add("="); + conditionArray.add(ConditionType.EQUAL.getOperator()); ArrayNode leftField = objectMapper.createArrayNode(); - leftField.add("field"); + leftField.add(FieldAttribute.FIELD.getAttributeName()); leftField.add(joinField2.getId()); - leftField.add(objectMapper.createObjectNode().put("base-type", "type/Integer")); + leftField.add(objectMapper.createObjectNode().put(FieldAttribute.BASE_TYPE.getAttributeName(), FieldType.INTEGER.getTypeName())); conditionArray.add(leftField); ArrayNode rightField = objectMapper.createArrayNode(); - rightField.add("field"); + rightField.add(FieldAttribute.FIELD.getAttributeName()); rightField.add(joinField1.getId()); - rightField.add(objectMapper.createObjectNode().put("base-type", "type/Integer").put("join-alias", addressTable.getName())); + rightField.add(objectMapper.createObjectNode().put(FieldAttribute.BASE_TYPE.getAttributeName(), FieldType.INTEGER.getTypeName()).put(FieldAttribute.JOIN_ALIAS.getAttributeName(), addressTable.getName())); conditionArray.add(rightField); - joinNode.set("condition", conditionArray); + joinNode.set(FieldAttribute.CONDITION.getAttributeName(), conditionArray); + joinNode.put(FieldAttribute.SOURCE_TABLE.getAttributeName(), addressTable.getId()); joinsArray.add(joinNode); - queryNode.set("joins", joinsArray); + queryNode.set(FieldAttribute.JOINS.getAttributeName(), joinsArray); return this; } diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java index 6076c9ed2..0de5a6fe5 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseRequestBody.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +// Refer docs : https://www.metabase.com/docs/latest/api/card#post-apicard public class MetabaseRequestBody { private String name; private MetabaseQuery datasetQuery; diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/VisualizationType.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/VisualizationType.java index 94dc0611c..d44b50ac6 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/VisualizationType.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/VisualizationType.java @@ -1,5 +1,6 @@ package org.avni.server.domain.metabase; +//Refer documentation : https://www.metabase.com/docs/latest/questions/sharing/visualizing-results public enum VisualizationType { TABLE("table"), CHART("chart"); diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java index 676b3192c..1e2441dda 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java @@ -32,6 +32,6 @@ public void createQuestionForTable(String tableName, String schema) { tableDetails.setName(tableName); TableDetails fetchedTableDetails = databaseRepository.findTableDetailsByName(database, tableDetails); - databaseRepository.createQuestionForIndividualTable(database, fetchedTableDetails); + databaseRepository.createQuestionForASingleTable(database, fetchedTableDetails); } } From d63b14721212d73cdd16b8d8ba981ffe1a190027 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Wed, 11 Sep 2024 14:31:27 +0530 Subject: [PATCH 33/34] avniproject#762 | Encounter added , DatasetColumn added to get rid of int literals --- .../server/domain/metabase/DatasetColumn.java | 16 ++++++++++++++++ .../avni/server/domain/metabase/TableType.java | 1 + .../server/service/metabase/DatabaseService.java | 9 +++++---- 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/DatasetColumn.java diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatasetColumn.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatasetColumn.java new file mode 100644 index 000000000..48509b4c7 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/DatasetColumn.java @@ -0,0 +1,16 @@ +package org.avni.server.domain.metabase; + +public enum DatasetColumn { + NAME(1), + TYPE(2); + + private final int index; + + DatasetColumn(int index) { + this.index = index; + } + + public int getIndex() { + return index; + } +} \ No newline at end of file diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableType.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableType.java index f361ed671..d1b221c6e 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableType.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/TableType.java @@ -5,6 +5,7 @@ public enum TableType { HOUSEHOLD("Household"), GROUP("Group"), PERSON("Person"), + ENCOUNTER("Encounter"), PROGRAM_ENCOUNTER("ProgramEncounter"), PROGRAM_ENROLMENT("ProgramEnrolment"); diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index 784936d15..b3e7dca63 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -54,12 +54,12 @@ public List getSubjectTypeNames() { List subjectTypeNames = new ArrayList<>(); for (List row : rows) { - String type = row.get(2); + String type = row.get(DatasetColumn.TYPE.getIndex()); if (type.equalsIgnoreCase(TableType.INDIVIDUAL.getTypeName()) || type.equalsIgnoreCase(TableType.HOUSEHOLD.getTypeName()) || type.equalsIgnoreCase(TableType.GROUP.getTypeName()) || type.equalsIgnoreCase(TableType.PERSON.getTypeName())) { - subjectTypeNames.add(row.get(1)); + subjectTypeNames.add(row.get(DatasetColumn.NAME.getIndex())); } } @@ -75,10 +75,11 @@ public List getProgramAndEncounterNames() { List programNames = new ArrayList<>(); for (List row : rows) { - String type = row.get(2); + String type = row.get(DatasetColumn.TYPE.getIndex()); if (type.equalsIgnoreCase(TableType.PROGRAM_ENCOUNTER.getTypeName()) || + type.equalsIgnoreCase(TableType.ENCOUNTER.getTypeName()) || type.equalsIgnoreCase(TableType.PROGRAM_ENROLMENT.getTypeName())) { - programNames.add(row.get(1)); + programNames.add(row.get(DatasetColumn.NAME.getIndex())); } } From f621d61ee0928e2b231e57640e1a7720a9693647 Mon Sep 17 00:00:00 2001 From: Om Bhardwaj <115864495+ombhardwajj@users.noreply.github.com> Date: Wed, 11 Sep 2024 23:34:49 +0530 Subject: [PATCH 34/34] avniproject#762 | 1) Moved AddressQuestionCreationService logic to DatabaseService 2) Replaced getSubjectTypeNames usage of int literals with constants depictiing columnsNames 3)Created common method called by createQuestionsForSubjectTypes() and createQuestionsForPrograms(), createQuestionsForProgramsEncounters() and createQuestionsForEncounters() with List entityNames 4)Added CollectionItem , Now before creating questions, the existing ones are filtered out --- .../dao/metabase/DatabaseRepository.java | 23 +++++ .../domain/metabase/CollectionItem.java | 25 ++++++ .../domain/metabase/FieldAttribute.java | 1 + .../domain/metabase/MetabaseQueryBuilder.java | 2 +- .../AddressQuestionCreationService.java | 37 -------- .../service/metabase/DatabaseService.java | 86 ++++++++++++------- 6 files changed, 103 insertions(+), 71 deletions(-) create mode 100644 avni-server-api/src/main/java/org/avni/server/domain/metabase/CollectionItem.java delete mode 100644 avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java diff --git a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java index f57e0d05b..c419f822f 100644 --- a/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java +++ b/avni-server-api/src/main/java/org/avni/server/dao/metabase/DatabaseRepository.java @@ -8,6 +8,8 @@ import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.stereotype.Repository; import com.fasterxml.jackson.databind.ObjectMapper; + +import java.util.ArrayList; import java.util.List; @Repository @@ -47,6 +49,27 @@ public Database getDatabaseByName(Database database) { } } + public List getExistingCollectionItems(int collectionId) { + String url = metabaseApiUrl + "/collection/" + collectionId + "/items"; + String jsonResponse = getForObject(url, String.class); + + try { + JsonNode rootNode = objectMapper.readTree(jsonResponse); + JsonNode dataArray = rootNode.path("data"); + + List items = new ArrayList<>(); + for (JsonNode itemNode : dataArray) { + CollectionItem item = new CollectionItem(); + item.setName(itemNode.get("name").asText()); + item.setId(itemNode.get("id").asInt()); + items.add(item); + } + return items; + } catch (Exception e) { + throw new RuntimeException("Failed to fetch collection items", e); + } + } + public CollectionInfoResponse getCollectionByName(Database database) { String url = metabaseApiUrl + "/collection"; try { diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/CollectionItem.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/CollectionItem.java new file mode 100644 index 000000000..c19a70571 --- /dev/null +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/CollectionItem.java @@ -0,0 +1,25 @@ +package org.avni.server.domain.metabase; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class CollectionItem { + private String name; + private int id; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } +} diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldAttribute.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldAttribute.java index e74428016..c00910731 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldAttribute.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/FieldAttribute.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; public enum FieldAttribute { + ALL("all"), FIELD("field"), FIELDS("fields"), SOURCE_TABLE("source-table"), diff --git a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java index 2f9ed77d3..1997cb8ec 100644 --- a/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java +++ b/avni-server-api/src/main/java/org/avni/server/domain/metabase/MetabaseQueryBuilder.java @@ -24,7 +24,7 @@ public MetabaseQueryBuilder forTable(TableDetails tableDetails) { public MetabaseQueryBuilder joinWith(TableDetails addressTable, FieldDetails joinField1, FieldDetails joinField2) { ObjectNode joinNode = objectMapper.createObjectNode(); - joinNode.put(FieldAttribute.FIELDS.getAttributeName(), "all"); + joinNode.put(FieldAttribute.FIELDS.getAttributeName(), FieldAttribute.ALL.getAttributeName()); joinNode.put(FieldAttribute.ALIAS.getAttributeName(), addressTable.getName()); ArrayNode conditionArray = objectMapper.createArrayNode(); diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java deleted file mode 100644 index 1e2441dda..000000000 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/AddressQuestionCreationService.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.avni.server.service.metabase; - -import org.avni.server.dao.metabase.DatabaseRepository; -import org.avni.server.domain.metabase.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Service; - -@Service -public class AddressQuestionCreationService implements QuestionCreationService { - - private final DatabaseService databaseService; - private final DatabaseRepository databaseRepository; - - @Autowired - public AddressQuestionCreationService(@Lazy DatabaseService databaseService, DatabaseRepository databaseRepository) { - this.databaseService = databaseService; - this.databaseRepository = databaseRepository; - } - - @Override - public void createQuestionForTable(TableDetails tableDetails, TableDetails addressTableDetails, FieldDetails addressFieldDetails, FieldDetails tableFieldDetails) { - Database database = databaseService.getGlobalDatabase(); - databaseRepository.createQuestionForTable(database, tableDetails, addressTableDetails, addressFieldDetails, tableFieldDetails); - } - - @Override - public void createQuestionForTable(String tableName, String schema) { - Database database = databaseService.getGlobalDatabase(); - - TableDetails tableDetails = new TableDetails(); - tableDetails.setName(tableName); - TableDetails fetchedTableDetails = databaseRepository.findTableDetailsByName(database, tableDetails); - - databaseRepository.createQuestionForASingleTable(database, fetchedTableDetails); - } -} diff --git a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java index b3e7dca63..fd3cec2e6 100644 --- a/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java +++ b/avni-server-api/src/main/java/org/avni/server/service/metabase/DatabaseService.java @@ -7,21 +7,21 @@ import java.util.Arrays; import java.util.ArrayList; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; @Service -public class DatabaseService { +public class DatabaseService implements QuestionCreationService{ private final DatabaseRepository databaseRepository; private final MetabaseService metabaseService; - private final AddressQuestionCreationService addressQuestionCreationService; private static final String ADDRESS_TABLE = "Address"; @Autowired - public DatabaseService(DatabaseRepository databaseRepository, MetabaseService metabaseService, AddressQuestionCreationService addressQuestionCreationService) { + public DatabaseService(DatabaseRepository databaseRepository, MetabaseService metabaseService) { this.databaseRepository = databaseRepository; this.metabaseService = metabaseService; - this.addressQuestionCreationService = addressQuestionCreationService; } public Database getGlobalDatabase() { @@ -45,6 +45,33 @@ private void ensureSyncComplete() { } } + private List filterOutExistingQuestions(List entityNames) { + Set existingItemNames = databaseRepository.getExistingCollectionItems(getGlobalCollection().getIdAsInt()).stream() + .map(item -> item.getName().trim().toLowerCase().replace(" ", "_")) + .collect(Collectors.toSet()); + + return entityNames.stream() + .filter(entityName -> !existingItemNames.contains(entityName.toLowerCase())) + .collect(Collectors.toList()); + } + + @Override + public void createQuestionForTable(TableDetails tableDetails, TableDetails addressTableDetails, FieldDetails addressFieldDetails, FieldDetails tableFieldDetails) { + Database database = getGlobalDatabase(); + databaseRepository.createQuestionForTable(database, tableDetails, addressTableDetails, addressFieldDetails, tableFieldDetails); + } + + @Override + public void createQuestionForTable(String tableName, String schema) { + Database database = getGlobalDatabase(); + + TableDetails tableDetails = new TableDetails(); + tableDetails.setName(tableName); + TableDetails fetchedTableDetails = databaseRepository.findTableDetailsByName(database, tableDetails); + + databaseRepository.createQuestionForASingleTable(database, fetchedTableDetails); + } + public List getSubjectTypeNames() { TableDetails fetchedMetadataTable = databaseRepository.findTableDetailsByName(getGlobalDatabase(), new TableDetails("table_metadata")); @@ -86,49 +113,42 @@ public List getProgramAndEncounterNames() { return programNames; } - public void createQuestionsForSubjectTypes(){ + private void createQuestionsForEntities(List entityNames, FieldDetails addressFieldDetails, FieldDetails entityFieldDetails) { ensureSyncComplete(); - List subjectTypeNames = getSubjectTypeNames(); + TableDetails fetchedAddressTableDetails = databaseRepository.findTableDetailsByName(getGlobalDatabase(), new TableDetails(ADDRESS_TABLE)); + + List filteredEntities = filterOutExistingQuestions(entityNames); - TableDetails addressTableDetails = new TableDetails(ADDRESS_TABLE); - TableDetails fetchedAddressTableDetails = databaseRepository.findTableDetailsByName(getGlobalDatabase(), addressTableDetails); + for (String entityName : filteredEntities) { + TableDetails entityTableDetails = new TableDetails(); + entityTableDetails.setName(entityName); + TableDetails fetchedEntityTableDetails = databaseRepository.findTableDetailsByName(getGlobalDatabase(), entityTableDetails); + createQuestionForTable(fetchedEntityTableDetails, fetchedAddressTableDetails, addressFieldDetails, entityFieldDetails); + } + } + public void createQuestionsForSubjectTypes() { + List subjectTypeNames = getSubjectTypeNames(); FieldDetails addressFieldDetails = new FieldDetails("id"); FieldDetails subjectFieldDetails = new FieldDetails("address_id"); - - - for (String subjectTypeName : subjectTypeNames) { - TableDetails subjectTableDetails = new TableDetails(); - subjectTableDetails.setName(subjectTypeName); - TableDetails fetchedSubjectTableDetails = databaseRepository.findTableDetailsByName(getGlobalDatabase(), subjectTableDetails); - addressQuestionCreationService.createQuestionForTable(fetchedSubjectTableDetails, fetchedAddressTableDetails, addressFieldDetails, subjectFieldDetails); - } + createQuestionsForEntities(subjectTypeNames, addressFieldDetails, subjectFieldDetails); } - public void createQuestionsForProgramsAndEncounters(){ - ensureSyncComplete(); + public void createQuestionsForProgramsAndEncounters() { List programAndEncounterNames = getProgramAndEncounterNames(); - - TableDetails fetchedAddressTableDetails = databaseRepository.findTableDetailsByName(getGlobalDatabase(), new TableDetails(ADDRESS_TABLE)); - FieldDetails addressFieldDetails = new FieldDetails("id"); - FieldDetails programFieldDetails = new FieldDetails("address_id"); - - for (String programName : programAndEncounterNames) { - TableDetails programTableDetails = new TableDetails(); - programTableDetails.setName(programName); - TableDetails fetchedProgramTableDetails = databaseRepository.findTableDetailsByName(getGlobalDatabase(), programTableDetails); - addressQuestionCreationService.createQuestionForTable(fetchedProgramTableDetails, fetchedAddressTableDetails, addressFieldDetails, programFieldDetails); - } + FieldDetails programOrEncounterFieldDetails = new FieldDetails("address_id"); + createQuestionsForEntities(programAndEncounterNames, addressFieldDetails, programOrEncounterFieldDetails); } - public void createQuestionsForIndividualTables(){ + public void createQuestionsForIndividualTables() { ensureSyncComplete(); - List individualTables = Arrays.asList("address", "media", "sync_telemetry"); - for (String tableName : individualTables) { - addressQuestionCreationService.createQuestionForTable(tableName, "!public"); + List filteredTables = filterOutExistingQuestions(individualTables); + + for (String tableName : filteredTables) { + createQuestionForTable(tableName, "!public"); } }