Optimized SQL for fetching related standard concepts, eliminated vend…

…or-specific string aggregating function

src/main/java/org/ohdsi/webapi/service/VocabularyService.java

@@ -3,6 +3,8 @@
 import static org.ohdsi.webapi.service.cscompare.ConceptSetCompareService.CONCEPT_SET_COMPARISON_ROW_MAPPER;
 import static org.ohdsi.webapi.util.SecurityUtils.whitelist;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
@@ -97,7 +99,10 @@ public class VocabularyService extends AbstractDaoService {
 
   @Autowired
   private ConceptSetCompareService conceptSetCompareService;
-
+
+  @Autowired
+  private ObjectMapper objectMapper;
+
   @Value("${datasource.driverClassName}")
   private String driver;
 
@@ -787,63 +792,70 @@ public Collection<RelatedConcept> getRelatedConcepts(@PathParam("sourceKey") Str
    @POST
    @Path("{sourceKey}/related-standard")
    @Produces(MediaType.APPLICATION_JSON)
-   public Collection<MappedRelatedConcept> getRelatedStandardMappedConcepts(@PathParam("sourceKey") String sourceKey, List<Long> conceptIds) {
+   public Collection<MappedRelatedConcept> getRelatedStandardMappedConcepts(@PathParam("sourceKey") String sourceKey, List<Long> allConceptIds) {
      Source source = getSourceRepository().findBySourceKey(sourceKey);
-     String sqlPath = "/resources/vocabulary/sql/getRelatedStandardMappedConcepts.sql";
+     String relatedConceptsSQLPath = "/resources/vocabulary/sql/getRelatedStandardMappedConcepts.sql";
+     String relatedMappedFromIdsSQLPath = "/resources/vocabulary/sql/getRelatedStandardMappedConcepts_getMappedFromIds.sql";
      String tableQualifier = source.getTableQualifier(SourceDaimon.DaimonType.Vocabulary);
 
      String[] searchStrings = {"CDM_schema"};
      String[] replacementStrings = {tableQualifier};
 
      String[] varNames = {"conceptIdList"};
-     Object[] varValues = {conceptIds.toArray()};
-
-     PreparedStatementRenderer psr = new PreparedStatementRenderer(source, sqlPath, searchStrings, replacementStrings, varNames, varValues);
 
-     final Map<Long, MappedRelatedConcept> concepts = new HashMap<>();
-     getSourceJdbcTemplate(source).query(psr.getSql(), psr.getSetter(), (RowMapper<Void>) (resultSet, arg1) -> {
-       addMappedRelationships(concepts, resultSet);
-       return null;
-     });
-     return concepts.values();
+     final Map<Long, MappedRelatedConcept> resultCombinedMappedConcepts = new HashMap<>();
+     final Map<Long, RelatedConcept> relatedStandardConcepts = new HashMap<>();
+     for(final List<Long> conceptIdsBatch: Lists.partition(allConceptIds, PreparedSqlRender.getParameterLimit(source))) {
+       Object[] varValues = {conceptIdsBatch.toArray()};
+       PreparedStatementRenderer relatedConceptsRenderer = new PreparedStatementRenderer(source, relatedConceptsSQLPath, searchStrings, replacementStrings, varNames, varValues);
+       getSourceJdbcTemplate(source).query(relatedConceptsRenderer.getSql(), relatedConceptsRenderer.getSetter(), (RowMapper<Void>) (resultSet, arg1) -> {
+         addRelationships(relatedStandardConcepts, resultSet);
+         return null;
+       });
+
+       final Map<Long, Set<Long>> relatedNonStandardConceptIdsByStandardId = new HashMap<>();
+
+       PreparedStatementRenderer mappedFromConceptsRenderer = new PreparedStatementRenderer(source, relatedMappedFromIdsSQLPath, searchStrings, replacementStrings, varNames, varValues);
+       getSourceJdbcTemplate(source).query(mappedFromConceptsRenderer.getSql(), mappedFromConceptsRenderer.getSetter(), (RowMapper<Void>) (resultSet, arg1) -> {
+         populateRelatedConceptIds(relatedNonStandardConceptIdsByStandardId, resultSet);
+         return null;
+       });
+
+       enrichResultCombinedMappedConcepts(resultCombinedMappedConcepts, relatedStandardConcepts, relatedNonStandardConceptIdsByStandardId);
+      }
+     return resultCombinedMappedConcepts.values();
    }
-   private void addMappedRelationships(final Map<Long, MappedRelatedConcept> concepts, final ResultSet resultSet) throws SQLException {
+
+   private void populateRelatedConceptIds(final Map<Long, Set<Long>> mappedConceptsIds, final ResultSet resultSet) throws SQLException {
      final Long concept_id = resultSet.getLong("CONCEPT_ID");
-     if (!concepts.containsKey(concept_id)) {
-       final MappedRelatedConcept concept = new MappedRelatedConcept();
-       concept.conceptId = concept_id;
-       concept.conceptCode = resultSet.getString("CONCEPT_CODE");
-       concept.conceptName = resultSet.getString("CONCEPT_NAME");
-       concept.standardConcept = resultSet.getString("STANDARD_CONCEPT");
-       concept.invalidReason = resultSet.getString("INVALID_REASON");
-       concept.vocabularyId = resultSet.getString("VOCABULARY_ID");
-       concept.validStartDate = resultSet.getDate("VALID_START_DATE");
-       concept.validEndDate = resultSet.getDate("VALID_END_DATE");
-       concept.conceptClassId = resultSet.getString("CONCEPT_CLASS_ID");
-       concept.domainId = resultSet.getString("DOMAIN_ID");
-
-       final ConceptRelationship relationship = new ConceptRelationship();
-       relationship.relationshipName = resultSet.getString("RELATIONSHIP_NAME");
-       relationship.relationshipDistance = resultSet.getInt("RELATIONSHIP_DISTANCE");
-       concept.relationships.add(relationship);
-
-       concept.mappedFromIds = extractMappedFromIds(resultSet);
-       concepts.put(concept_id, concept);
+     if (!mappedConceptsIds.containsKey(concept_id)) {
+       Set<Long> mappedIds = new HashSet<>();
+       mappedIds.add(resultSet.getLong("MAPPED_FROM_ID"));
+       mappedConceptsIds.put(concept_id,mappedIds);
      } else {
-       final ConceptRelationship relationship = new ConceptRelationship();
-       relationship.relationshipName = resultSet.getString("RELATIONSHIP_NAME");
-       relationship.relationshipDistance = resultSet.getInt("RELATIONSHIP_DISTANCE");
-       concepts.get(concept_id).relationships.add(relationship);
-       concepts.get(concept_id).mappedFromIds.addAll(extractMappedFromIds(resultSet));
+       mappedConceptsIds.get(concept_id).add(resultSet.getLong("MAPPED_FROM_ID"));
      }
    }
-   private Set<Long> extractMappedFromIds(ResultSet resultSet) throws SQLException {
-     String mappedFromIds = resultSet.getString("mapped_from");
-     return Arrays.stream(mappedFromIds.split(","))
-             .map(Long::parseLong)
-             .collect(Collectors.toSet());
-   }
 
+   void enrichResultCombinedMappedConcepts(Map<Long, MappedRelatedConcept> resultCombinedMappedConcepts,
+                                           Map<Long, RelatedConcept> relatedStandardConcepts,
+                                           Map<Long, Set<Long>> relatedNonStandardConceptIdsByStandardId) {
+    relatedNonStandardConceptIdsByStandardId.forEach((standardConceptId, mappedFromIds)->{
+      if(resultCombinedMappedConcepts.containsKey(standardConceptId)){
+        resultCombinedMappedConcepts.get(standardConceptId).mappedFromIds.addAll(mappedFromIds);
+      } else {
+        MappedRelatedConcept mappedRelatedConcept;
+         try {
+           mappedRelatedConcept = objectMapper.readValue(objectMapper.writeValueAsString(relatedStandardConcepts.get(standardConceptId)), MappedRelatedConcept.class);
+           mappedRelatedConcept.mappedFromIds=mappedFromIds;
+           resultCombinedMappedConcepts.put(standardConceptId,mappedRelatedConcept);
+         } catch (JsonProcessingException e) {
+           log.error("Could not convert RelatedConcept to MappedRelatedConcept", e);
+           throw new WebApplicationException(e);
+         }
+      }
+    });
+   }
 
    /**
    * Get ancestor and descendant concepts for the selected concept identifier 

src/main/resources/resources/vocabulary/sql/getRelatedStandardMappedConcepts.sql

@@ -10,14 +10,12 @@ SELECT
    c.VALID_START_DATE,
    c.VALID_END_DATE,
    c.RELATIONSHIP_NAME,
-   c.RELATIONSHIP_DISTANCE,
-   STRING_AGG(DISTINCT c.mapped_from_id, ',') AS mapped_from
+   c.RELATIONSHIP_DISTANCE
 FROM (
     SELECT
         c.CONCEPT_ID, CONCEPT_NAME, COALESCE(c.STANDARD_CONCEPT, 'N') as STANDARD_CONCEPT, COALESCE(c.INVALID_REASON, 'V') as INVALID_REASON,
         c.CONCEPT_CODE, c.CONCEPT_CLASS_ID, c.DOMAIN_ID, c.VOCABULARY_ID, c.VALID_START_DATE, c.VALID_END_DATE,
-        r.RELATIONSHIP_NAME, 1 as RELATIONSHIP_DISTANCE,
-        CAST(cr.CONCEPT_ID_1 AS VARCHAR) as mapped_from_id
+        r.RELATIONSHIP_NAME, 1 as RELATIONSHIP_DISTANCE
     FROM
         @CDM_schema.concept_relationship cr
     JOIN
@@ -28,38 +26,6 @@ FROM (
         cr.CONCEPT_ID_1 IN (@conceptIdList)
         AND COALESCE(c.STANDARD_CONCEPT, 'N') IN ('S', 'C')
         AND cr.INVALID_REASON IS NULL
-
-    UNION ALL
-
-    SELECT
-        ca.ANCESTOR_CONCEPT_ID as CONCEPT_ID, c.CONCEPT_NAME, COALESCE(c.STANDARD_CONCEPT, 'N') as STANDARD_CONCEPT, COALESCE(c.INVALID_REASON, 'V') as INVALID_REASON,
-        c.CONCEPT_CODE, c.CONCEPT_CLASS_ID, c.DOMAIN_ID, c.VOCABULARY_ID, c.VALID_START_DATE, c.VALID_END_DATE,
-        'Has ancestor of' as RELATIONSHIP_NAME, MIN_LEVELS_OF_SEPARATION as RELATIONSHIP_DISTANCE,
-        CAST(ca.DESCENDANT_CONCEPT_ID AS VARCHAR) as mapped_from_id
-    FROM
-        @CDM_schema.concept_ancestor ca
-    JOIN
-        @CDM_schema.concept c ON c.CONCEPT_ID = ca.ANCESTOR_CONCEPT_ID
-    WHERE
-        ca.DESCENDANT_CONCEPT_ID IN (@conceptIdList)
-        AND COALESCE(c.STANDARD_CONCEPT, 'N') IN ('S', 'C')
-        AND ca.ANCESTOR_CONCEPT_ID NOT IN (@conceptIdList)
-
-    UNION ALL
-
-    SELECT
-        ca.DESCENDANT_CONCEPT_ID as CONCEPT_ID, c.CONCEPT_NAME, COALESCE(c.STANDARD_CONCEPT, 'N') as STANDARD_CONCEPT, COALESCE(c.INVALID_REASON, 'V') as INVALID_REASON,
-        c.CONCEPT_CODE, c.CONCEPT_CLASS_ID, c.DOMAIN_ID, c.VOCABULARY_ID, c.VALID_START_DATE, c.VALID_END_DATE,
-        'Has descendant of' as RELATIONSHIP_NAME, MIN_LEVELS_OF_SEPARATION as RELATIONSHIP_DISTANCE,
-        CAST(ca.ANCESTOR_CONCEPT_ID AS VARCHAR) as mapped_from_id
-    FROM
-        @CDM_schema.concept_ancestor ca
-    JOIN
-        @CDM_schema.concept c ON c.CONCEPT_ID = ca.DESCENDANT_CONCEPT_ID
-    WHERE
-        ca.ANCESTOR_CONCEPT_ID IN (@conceptIdList)
-        AND COALESCE(c.STANDARD_CONCEPT, 'N') IN ('S', 'C')
-        AND ca.DESCENDANT_CONCEPT_ID NOT IN (@conceptIdList)
 ) c
 GROUP BY
    c.CONCEPT_ID,

src/main/resources/resources/vocabulary/sql/getRelatedStandardMappedConcepts_getMappedFromIds.sql

@@ -0,0 +1,20 @@
+SELECT
+   c.CONCEPT_ID,
+   c.MAPPED_FROM_ID
+FROM (
+    SELECT DISTINCT
+        c.CONCEPT_ID,
+        CAST(cr.CONCEPT_ID_1 AS VARCHAR) as MAPPED_FROM_ID
+    FROM
+        @CDM_schema.concept_relationship cr
+    JOIN
+        @CDM_schema.concept c ON cr.CONCEPT_ID_2 = c.CONCEPT_ID
+    JOIN
+        @CDM_schema.relationship r ON cr.RELATIONSHIP_ID = r.RELATIONSHIP_ID
+    WHERE
+        cr.CONCEPT_ID_1 IN (@conceptIdList)
+        AND COALESCE(c.STANDARD_CONCEPT, 'N') IN ('S', 'C')
+        AND cr.INVALID_REASON IS NULL
+) c
+ORDER BY
+   c.CONCEPT_ID;