From 7c0b7df926286df02bafe4c941d64852caf129ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Andri=C4=87?= Date: Thu, 29 Oct 2020 13:34:20 -0400 Subject: [PATCH 1/7] Github packages --- Jenkinsfile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 39d8169..c011600 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,4 @@ -def dockerHubRepo = "icgcargo/song-search" +def dockerRepo = "ghcr.io/icgc-argo/song-search" def gitHubRepo = "icgc-argo/song-search" def chartVersion = "1.1.0" def commit = "UNKNOWN" @@ -69,15 +69,15 @@ spec: } steps { container('docker') { - withCredentials([usernamePassword(credentialsId:'argoDockerHub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh 'docker login -u $USERNAME -p $PASSWORD' + withCredentials([usernamePassword(credentialsId:'argoContainers', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh 'docker login ghcr.io -u $USERNAME -p $PASSWORD' } // DNS error if --network is default - sh "docker build --network=host . -t ${dockerHubRepo}:edge -t ${dockerHubRepo}:${commit}" + sh "docker build --network=host . -t ${dockerRepo}:edge -t ${dockerRepo}:${commit}" - sh "docker push ${dockerHubRepo}:${commit}" - sh "docker push ${dockerHubRepo}:edge" + sh "docker push ${dockerRepo}:${commit}" + sh "docker push ${dockerRepo}:edge" } } } @@ -110,15 +110,15 @@ spec: sh "git push https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${gitHubRepo} --tags" } - withCredentials([usernamePassword(credentialsId:'argoDockerHub', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh 'docker login -u $USERNAME -p $PASSWORD' + withCredentials([usernamePassword(credentialsId:'argoContainers', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh 'docker login ghcr.io -u $USERNAME -p $PASSWORD' } // DNS error if --network is default - sh "docker build --network=host . -t ${dockerHubRepo}:latest -t ${dockerHubRepo}:${version}" + sh "docker build --network=host . -t ${dockerRepo}:latest -t ${dockerRepo}:${version}" - sh "docker push ${dockerHubRepo}:${version}" - sh "docker push ${dockerHubRepo}:latest" + sh "docker push ${dockerRepo}:${version}" + sh "docker push ${dockerRepo}:latest" } } } From e02fc81321abd90f785a722825017e0a547b1e6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Andri=C4=87?= Date: Thu, 29 Oct 2020 14:13:19 -0400 Subject: [PATCH 2/7] Update Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c011600..e6d9be8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ def dockerRepo = "ghcr.io/icgc-argo/song-search" def gitHubRepo = "icgc-argo/song-search" -def chartVersion = "1.1.0" +def chartVersion = "1.2.0" def commit = "UNKNOWN" def version = "UNKNOWN" From d366a661f8ad0c26f3fcd33913bfcee2e47fa557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Andri=C4=87?= Date: Thu, 29 Oct 2020 14:31:40 -0400 Subject: [PATCH 3/7] disables gateway restart step --- Jenkinsfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c011600..ceb4e84 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -93,10 +93,10 @@ spec: [$class: 'StringParameterValue', name: 'AP_HELM_CHART_VERSION', value: "${chartVersion}"], [$class: 'StringParameterValue', name: 'AP_ARGS_LINE', value: "--set-string image.tag=${commit}" ] ]) - sleep(time:30,unit:"SECONDS") - build(job: "/provision/rdpc-gateway-restart", parameters: [ - [$class: 'StringParameterValue', name: 'AP_RDPC_ENV', value: 'dev' ], - ]) + // sleep(time:30,unit:"SECONDS") + // build(job: "/provision/rdpc-gateway-restart", parameters: [ + // [$class: 'StringParameterValue', name: 'AP_RDPC_ENV', value: 'dev' ], + // ]) } } stage('Build & Publish Release') { @@ -134,10 +134,10 @@ spec: [$class: 'StringParameterValue', name: 'AP_HELM_CHART_VERSION', value: "${chartVersion}"], [$class: 'StringParameterValue', name: 'AP_ARGS_LINE', value: "--set-string image.tag=${version}" ] ]) - sleep(time:30,unit:"SECONDS") - build(job: "/provision/rdpc-gateway-restart", parameters: [ - [$class: 'StringParameterValue', name: 'AP_RDPC_ENV', value: 'qa' ], - ]) + // sleep(time:30,unit:"SECONDS") + // build(job: "/provision/rdpc-gateway-restart", parameters: [ + // [$class: 'StringParameterValue', name: 'AP_RDPC_ENV', value: 'qa' ], + // ]) } } } From e8856d3d789e95ddb6fb872d9573d252e05b3df1 Mon Sep 17 00:00:00 2001 From: jaserud Date: Wed, 13 Jan 2021 12:39:43 -0500 Subject: [PATCH 4/7] analysis sort and searchresult (#44) * adds sorts and extend analysis query which has search info * better naming and update some comments * sorts * rename proberesult to searchresult and add aggregateAnalyses * clean up and some docs * fix default sort * revert to previous setup for argumentPathMap and use same in sortMap * add aggregate analysis query to gqlprovider --- .../config/constants/EsDefaults.java | 13 ++++++ .../config/{ => constants}/SearchFields.java | 5 ++- .../graphql/AnalysisDataFetcher.java | 33 +++++++++++++-- .../songsearch/graphql/EntityDataFetcher.java | 4 +- .../songsearch/graphql/GraphQLProvider.java | 3 ++ .../songsearch/model/AggregationResult.java | 8 ++++ .../overture/songsearch/model/Analysis.java | 6 +++ .../songsearch/model/SearchResult.java | 22 ++++++++++ .../bio/overture/songsearch/model/Sort.java | 9 ++++ .../repository/AnalysisRepository.java | 41 +++++++++++++++++-- .../songsearch/repository/FileRepository.java | 2 +- .../songsearch/service/AnalysisService.java | 32 +++++++++++++-- .../songsearch/service/FileService.java | 2 +- .../utils/ElasticsearchQueryUtils.java | 29 +++++++++++-- .../songsearch/utils/JacksonUtils.java | 31 ++++++++++++++ src/main/resources/schema.graphql | 39 +++++++++++++++++- 16 files changed, 258 insertions(+), 21 deletions(-) create mode 100644 src/main/java/bio/overture/songsearch/config/constants/EsDefaults.java rename src/main/java/bio/overture/songsearch/config/{ => constants}/SearchFields.java (90%) create mode 100644 src/main/java/bio/overture/songsearch/model/AggregationResult.java create mode 100644 src/main/java/bio/overture/songsearch/model/SearchResult.java create mode 100644 src/main/java/bio/overture/songsearch/model/Sort.java create mode 100644 src/main/java/bio/overture/songsearch/utils/JacksonUtils.java diff --git a/src/main/java/bio/overture/songsearch/config/constants/EsDefaults.java b/src/main/java/bio/overture/songsearch/config/constants/EsDefaults.java new file mode 100644 index 0000000..58f70c1 --- /dev/null +++ b/src/main/java/bio/overture/songsearch/config/constants/EsDefaults.java @@ -0,0 +1,13 @@ +package bio.overture.songsearch.config.constants; + +import static lombok.AccessLevel.PRIVATE; + +import lombok.NoArgsConstructor; + +@NoArgsConstructor(access = PRIVATE) +public class EsDefaults { + // Default values from ES pagination: + // https://www.elastic.co/guide/en/elasticsearch/reference/7.x/paginate-search-results.html + public static final Integer ES_PAGE_DEFAULT_SIZE = 10; + public static final Integer ES_PAGE_DEFAULT_FROM = 0; +} diff --git a/src/main/java/bio/overture/songsearch/config/SearchFields.java b/src/main/java/bio/overture/songsearch/config/constants/SearchFields.java similarity index 90% rename from src/main/java/bio/overture/songsearch/config/SearchFields.java rename to src/main/java/bio/overture/songsearch/config/constants/SearchFields.java index 6ce87e0..0ea258a 100644 --- a/src/main/java/bio/overture/songsearch/config/SearchFields.java +++ b/src/main/java/bio/overture/songsearch/config/constants/SearchFields.java @@ -16,7 +16,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package bio.overture.songsearch.config; +package bio.overture.songsearch.config.constants; import static lombok.AccessLevel.PRIVATE; @@ -39,4 +39,7 @@ public class SearchFields { public static final String SUBMITTER_SAMPLE_ID = "submitterSampleId"; public static final String MATCHED_NORMAL_SUBMITTER_SAMPLE_ID = "matchedNormalSubmitterSampleId"; public static final String RUN_ID = "runId"; + public static final String PUBLISHED_AT = "publishedAt"; + public static final String UPDATED_AT = "updatedAt"; + public static final String FIRST_PUBLISHED_AT = "firstPublishedAt"; } diff --git a/src/main/java/bio/overture/songsearch/graphql/AnalysisDataFetcher.java b/src/main/java/bio/overture/songsearch/graphql/AnalysisDataFetcher.java index 5f530f8..00ef8d7 100644 --- a/src/main/java/bio/overture/songsearch/graphql/AnalysisDataFetcher.java +++ b/src/main/java/bio/overture/songsearch/graphql/AnalysisDataFetcher.java @@ -18,9 +18,12 @@ package bio.overture.songsearch.graphql; -import bio.overture.songsearch.model.Analysis; -import bio.overture.songsearch.model.SampleMatchedAnalysisPair; +import static bio.overture.songsearch.utils.JacksonUtils.convertValue; +import static java.util.stream.Collectors.toUnmodifiableList; + +import bio.overture.songsearch.model.*; import bio.overture.songsearch.service.AnalysisService; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import graphql.schema.DataFetcher; import java.util.List; @@ -42,18 +45,40 @@ public AnalysisDataFetcher(AnalysisService analysisService) { } @SuppressWarnings("unchecked") - public DataFetcher> getAnalysesDataFetcher() { + public DataFetcher> getAnalysesDataFetcher() { return environment -> { val args = environment.getArguments(); val filter = ImmutableMap.builder(); val page = ImmutableMap.builder(); + val sorts = ImmutableList.builder(); if (args != null) { if (args.get("filter") != null) filter.putAll((Map) args.get("filter")); if (args.get("page") != null) page.putAll((Map) args.get("page")); + if (args.get("sorts") != null) { + val rawSorts = (List) args.get("sorts"); + sorts.addAll( + rawSorts.stream() + .map(sort -> convertValue(sort, Sort.class)) + .collect(toUnmodifiableList())); + } + } + return analysisService.searchAnalyses(filter.build(), page.build(), sorts.build()); + }; + } + + @SuppressWarnings("unchecked") + public DataFetcher getAggregateAnalysesDataFetcher() { + return environment -> { + val args = environment.getArguments(); + + val filter = ImmutableMap.builder(); + + if (args != null) { + if (args.get("filter") != null) filter.putAll((Map) args.get("filter")); } - return analysisService.getAnalyses(filter.build(), page.build()); + return analysisService.aggregateAnalyses(filter.build()); }; } diff --git a/src/main/java/bio/overture/songsearch/graphql/EntityDataFetcher.java b/src/main/java/bio/overture/songsearch/graphql/EntityDataFetcher.java index 44db995..4874ae8 100644 --- a/src/main/java/bio/overture/songsearch/graphql/EntityDataFetcher.java +++ b/src/main/java/bio/overture/songsearch/graphql/EntityDataFetcher.java @@ -18,8 +18,8 @@ package bio.overture.songsearch.graphql; -import static bio.overture.songsearch.config.SearchFields.ANALYSIS_ID; -import static bio.overture.songsearch.config.SearchFields.RUN_ID; +import static bio.overture.songsearch.config.constants.SearchFields.ANALYSIS_ID; +import static bio.overture.songsearch.config.constants.SearchFields.RUN_ID; import static bio.overture.songsearch.utils.CommonUtils.asImmutableMap; import static com.google.common.base.Strings.isNullOrEmpty; import static java.util.stream.Collectors.toList; diff --git a/src/main/java/bio/overture/songsearch/graphql/GraphQLProvider.java b/src/main/java/bio/overture/songsearch/graphql/GraphQLProvider.java index 5c865fb..e7b049c 100644 --- a/src/main/java/bio/overture/songsearch/graphql/GraphQLProvider.java +++ b/src/main/java/bio/overture/songsearch/graphql/GraphQLProvider.java @@ -127,6 +127,9 @@ private RuntimeWiring buildWiring() { .type( newTypeWiring("Query") .dataFetcher("analyses", analysisDataFetcher.getAnalysesDataFetcher())) + .type( + newTypeWiring("Query") + .dataFetcher("aggregateAnalyses", analysisDataFetcher.getAggregateAnalysesDataFetcher())) .type(newTypeWiring("Query").dataFetcher("files", fileDataFetcher.getFilesDataFetcher())) .type( newTypeWiring("Query") diff --git a/src/main/java/bio/overture/songsearch/model/AggregationResult.java b/src/main/java/bio/overture/songsearch/model/AggregationResult.java new file mode 100644 index 0000000..8d53656 --- /dev/null +++ b/src/main/java/bio/overture/songsearch/model/AggregationResult.java @@ -0,0 +1,8 @@ +package bio.overture.songsearch.model; + +import lombok.Value; + +@Value +public class AggregationResult { + Long totalHits; +} diff --git a/src/main/java/bio/overture/songsearch/model/Analysis.java b/src/main/java/bio/overture/songsearch/model/Analysis.java index 8ae43c3..6fcfbfe 100644 --- a/src/main/java/bio/overture/songsearch/model/Analysis.java +++ b/src/main/java/bio/overture/songsearch/model/Analysis.java @@ -55,6 +55,12 @@ public class Analysis { private Workflow workflow; + private String updatedAt; + + private String publishedAt; + + private String firstPublishedAt; + @SneakyThrows public static Analysis parse(@NonNull Map sourceMap) { return MAPPER.convertValue(sourceMap, Analysis.class); diff --git a/src/main/java/bio/overture/songsearch/model/SearchResult.java b/src/main/java/bio/overture/songsearch/model/SearchResult.java new file mode 100644 index 0000000..918f90a --- /dev/null +++ b/src/main/java/bio/overture/songsearch/model/SearchResult.java @@ -0,0 +1,22 @@ +package bio.overture.songsearch.model; + +import java.util.List; +import lombok.Value; + +@Value +public class SearchResult { + List content; + Info info; + + public SearchResult(List content, Boolean hasNextFrom, Long totalHits) { + this.content = content; + this.info = new Info(hasNextFrom, totalHits, content.size()); + } + + @Value + public static class Info { + Boolean hasNextFrom; + Long totalHits; + Integer contentCount; + } +} diff --git a/src/main/java/bio/overture/songsearch/model/Sort.java b/src/main/java/bio/overture/songsearch/model/Sort.java new file mode 100644 index 0000000..4a35f52 --- /dev/null +++ b/src/main/java/bio/overture/songsearch/model/Sort.java @@ -0,0 +1,9 @@ +package bio.overture.songsearch.model; + +import lombok.Data; + +@Data +public class Sort { + String fieldName; + String order; +} diff --git a/src/main/java/bio/overture/songsearch/repository/AnalysisRepository.java b/src/main/java/bio/overture/songsearch/repository/AnalysisRepository.java index 23b55f6..30a0ae4 100644 --- a/src/main/java/bio/overture/songsearch/repository/AnalysisRepository.java +++ b/src/main/java/bio/overture/songsearch/repository/AnalysisRepository.java @@ -18,11 +18,14 @@ package bio.overture.songsearch.repository; -import static bio.overture.songsearch.config.SearchFields.*; -import static bio.overture.songsearch.utils.ElasticsearchQueryUtils.queryFromArgs; +import static bio.overture.songsearch.config.constants.SearchFields.*; +import static bio.overture.songsearch.utils.ElasticsearchQueryUtils.*; +import static java.util.Collections.emptyList; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.search.sort.SortOrder.ASC; import bio.overture.songsearch.config.ElasticsearchProperties; +import bio.overture.songsearch.model.Sort; import com.google.common.collect.ImmutableMap; import java.util.*; import java.util.function.Function; @@ -42,6 +45,8 @@ import org.elasticsearch.index.query.NestedQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.SortBuilders; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -51,6 +56,8 @@ public class AnalysisRepository { private static final Map>> QUERY_RESOLVER = argumentPathMap(); + private static final Map SORT_BUILDER_RESOLVER = sortPathMap(); + private final RestHighLevelClient client; private final String analysisCentricIndex; @@ -107,13 +114,28 @@ private static Map>> argumentPa .build(); } + private static Map sortPathMap() { + return ImmutableMap.builder() + .put(ANALYSIS_ID, SortBuilders.fieldSort("analysis_id")) + .put(ANALYSIS_STATE, SortBuilders.fieldSort("analysis_state")) + .put(PUBLISHED_AT, SortBuilders.fieldSort("published_at")) + .put(UPDATED_AT, SortBuilders.fieldSort("updated_at")) + .put(FIRST_PUBLISHED_AT, SortBuilders.fieldSort("first_published_at")) + .build(); + } + public SearchResponse getAnalyses(Map filter, Map page) { + return getAnalyses(filter, page, emptyList()); + } + + public SearchResponse getAnalyses( + Map filter, Map page, List sorts) { final AbstractQueryBuilder query = (filter == null || filter.size() == 0) ? matchAllQuery() : queryFromArgs(QUERY_RESOLVER, filter); - val searchSourceBuilder = createSearchSourceBuilder(query, page); + val searchSourceBuilder = createSearchSourceBuilder(query, page, sorts); return execute(searchSourceBuilder); } @@ -135,7 +157,20 @@ public MultiSearchResponse getAnalyses( private SearchSourceBuilder createSearchSourceBuilder( AbstractQueryBuilder query, Map page) { + return createSearchSourceBuilder(query, page, emptyList()); + } + + private SearchSourceBuilder createSearchSourceBuilder( + AbstractQueryBuilder query, Map page, List sorts) { val searchSourceBuilder = new SearchSourceBuilder(); + + if (sorts.isEmpty()) { + searchSourceBuilder.sort(SORT_BUILDER_RESOLVER.get(UPDATED_AT).order(ASC)); + } else { + val sortBuilders = sortsToEsSortBuilders(SORT_BUILDER_RESOLVER, sorts); + sortBuilders.forEach(searchSourceBuilder::sort); + } + searchSourceBuilder.query(query); if (page != null && page.size() != 0) { diff --git a/src/main/java/bio/overture/songsearch/repository/FileRepository.java b/src/main/java/bio/overture/songsearch/repository/FileRepository.java index a2f8780..4bff923 100644 --- a/src/main/java/bio/overture/songsearch/repository/FileRepository.java +++ b/src/main/java/bio/overture/songsearch/repository/FileRepository.java @@ -18,7 +18,7 @@ package bio.overture.songsearch.repository; -import static bio.overture.songsearch.config.SearchFields.*; +import static bio.overture.songsearch.config.constants.SearchFields.*; import static bio.overture.songsearch.utils.ElasticsearchQueryUtils.queryFromArgs; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; diff --git a/src/main/java/bio/overture/songsearch/service/AnalysisService.java b/src/main/java/bio/overture/songsearch/service/AnalysisService.java index b42762c..c0a951c 100644 --- a/src/main/java/bio/overture/songsearch/service/AnalysisService.java +++ b/src/main/java/bio/overture/songsearch/service/AnalysisService.java @@ -18,15 +18,15 @@ package bio.overture.songsearch.service; -import static bio.overture.songsearch.config.SearchFields.*; +import static bio.overture.songsearch.config.constants.EsDefaults.ES_PAGE_DEFAULT_FROM; +import static bio.overture.songsearch.config.constants.EsDefaults.ES_PAGE_DEFAULT_SIZE; +import static bio.overture.songsearch.config.constants.SearchFields.*; import static bio.overture.songsearch.model.enums.SpecimenType.NORMAL; import static bio.overture.songsearch.model.enums.SpecimenType.TUMOUR; import static java.util.Collections.emptyList; import static java.util.stream.Collectors.toUnmodifiableList; -import bio.overture.songsearch.model.Analysis; -import bio.overture.songsearch.model.Sample; -import bio.overture.songsearch.model.SampleMatchedAnalysisPair; +import bio.overture.songsearch.model.*; import bio.overture.songsearch.repository.AnalysisRepository; import com.google.common.collect.ImmutableMap; import java.util.*; @@ -52,6 +52,30 @@ private static Analysis hitToAnalysis(SearchHit hit) { return Analysis.parse(sourceMap); } + public SearchResult searchAnalyses( + Map filter, Map page, List sorts) { + val response = analysisRepository.getAnalyses(filter, page, sorts); + val responseSearchHits = response.getHits(); + + val totalHits = responseSearchHits.getTotalHits().value; + val from = page.getOrDefault("from", ES_PAGE_DEFAULT_FROM); + val size = page.getOrDefault("size", ES_PAGE_DEFAULT_SIZE); + + val analyses = + Arrays.stream(responseSearchHits.getHits()) + .map(AnalysisService::hitToAnalysis) + .collect(toUnmodifiableList()); + val nextFrom = (totalHits - from) / size > 0; + return new SearchResult<>(analyses, nextFrom, totalHits); + } + + public AggregationResult aggregateAnalyses(Map filter) { + val response = analysisRepository.getAnalyses(filter, Map.of(), List.of()); + val responseSearchHits = response.getHits(); + val totalHits = responseSearchHits.getTotalHits().value; + return new AggregationResult(totalHits); + } + public List getAnalyses(Map filter, Map page) { val response = analysisRepository.getAnalyses(filter, page); val hitStream = Arrays.stream(response.getHits().getHits()); diff --git a/src/main/java/bio/overture/songsearch/service/FileService.java b/src/main/java/bio/overture/songsearch/service/FileService.java index 671de3d..4b9a7d1 100644 --- a/src/main/java/bio/overture/songsearch/service/FileService.java +++ b/src/main/java/bio/overture/songsearch/service/FileService.java @@ -18,7 +18,7 @@ package bio.overture.songsearch.service; -import static bio.overture.songsearch.config.SearchFields.FILE_OBJECT_ID; +import static bio.overture.songsearch.config.constants.SearchFields.FILE_OBJECT_ID; import static java.util.stream.Collectors.toUnmodifiableList; import bio.overture.songsearch.model.File; diff --git a/src/main/java/bio/overture/songsearch/utils/ElasticsearchQueryUtils.java b/src/main/java/bio/overture/songsearch/utils/ElasticsearchQueryUtils.java index ca4c4f4..3c4a24e 100644 --- a/src/main/java/bio/overture/songsearch/utils/ElasticsearchQueryUtils.java +++ b/src/main/java/bio/overture/songsearch/utils/ElasticsearchQueryUtils.java @@ -18,13 +18,16 @@ package bio.overture.songsearch.utils; +import static java.util.stream.Collectors.toUnmodifiableList; + +import bio.overture.songsearch.model.Sort; +import java.util.List; import java.util.Map; import java.util.function.Function; import lombok.val; -import org.elasticsearch.index.query.AbstractQueryBuilder; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.query.*; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.SortOrder; public class ElasticsearchQueryUtils { /** @@ -47,6 +50,24 @@ public static BoolQueryBuilder queryFromArgs( return bool; } + /** + * For each sorts, find its SortBuilder and add the sort order value then collect them all in a + * list + * + * @param sorts List of Sort objects + * @return List of FiledSortBuilder + */ + public static List sortsToEsSortBuilders( + Map SORT_BUILDER_RESOLVER, List sorts) { + return sorts.stream() + .map( + sort -> { + val sortBuilder = SORT_BUILDER_RESOLVER.get(sort.getFieldName()); + return sortBuilder.order(SortOrder.fromString(sort.getOrder())); + }) + .collect(toUnmodifiableList()); + } + private static Function> simpleTermQueryBuilderResolver( String key) { return v -> new TermQueryBuilder(key, v); diff --git a/src/main/java/bio/overture/songsearch/utils/JacksonUtils.java b/src/main/java/bio/overture/songsearch/utils/JacksonUtils.java new file mode 100644 index 0000000..d4ae613 --- /dev/null +++ b/src/main/java/bio/overture/songsearch/utils/JacksonUtils.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2020 The Ontario Institute for Cancer Research. All rights reserved + * + * This program and the accompanying materials are made available under the terms of the GNU Affero General Public License v3.0. + * You should have received a copy of the GNU Affero General Public License along with + * this program. If not, see . + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package bio.overture.songsearch.utils; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; + +public class JacksonUtils { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @SneakyThrows + public static T convertValue(Object fromValue, Class toValueType) { + return OBJECT_MAPPER.convertValue(fromValue, toValueType); + } +} diff --git a/src/main/resources/schema.graphql b/src/main/resources/schema.graphql index 329cd0c..f6ab8da 100644 --- a/src/main/resources/schema.graphql +++ b/src/main/resources/schema.graphql @@ -12,6 +12,9 @@ type Analysis @key(fields: "analysisId") { analysisType: String analysisVersion: Int analysisState: AnalysisState + publishedAt: String + updatedAt: String + firstPublishedAt: String studyId: String donors: [Donor] files: [AnalysisFile] @@ -149,8 +152,42 @@ type SampleMatchedAnalysisPair { tumourSampleAnalysis: Analysis } +enum AnalysisSortField { + analysisId, + analysisState, + publishedAt, + updatedAt, + firstPublishedAt +} + +enum SortOrder { + asc, + desc +} + +input AnalysisSort { + fieldName: AnalysisSortField! + order: SortOrder! +} + +type SearchResultInfo { + contentCount: String! + hasNextFrom: String! + totalHits: String! +} + +type AnalysesSearchResult { + content: [Analysis!] + info: SearchResultInfo! +} + +type AggregationResult { + totalHits: String! +} + extend type Query { - analyses(filter: AnalysisFilter, page: Page): [Analysis] + analyses(filter: AnalysisFilter, page: Page, sorts: [AnalysisSort!]): AnalysesSearchResult! + aggregateAnalyses(filter: AnalysisFilter): AggregationResult! files(filter: FileFilter, page: Page): [File] sampleMatchedAnalysisPairs(analysisId: String!): [SampleMatchedAnalysisPair] } From 18677c450b0c4f129b3eae913a524721130a3629 Mon Sep 17 00:00:00 2001 From: jaserud Date: Wed, 13 Jan 2021 15:54:52 -0500 Subject: [PATCH 5/7] apply new model to files' gql entity (#45) * apply new model to files * fix copy pasta error * run formatter --- .../songsearch/graphql/FileDataFetcher.java | 33 +++++++++++++++++-- .../songsearch/graphql/GraphQLProvider.java | 6 +++- .../songsearch/repository/FileRepository.java | 30 +++++++++++++++++ .../songsearch/service/FileService.java | 28 +++++++++++++++- src/main/resources/schema.graphql | 20 ++++++++++- 5 files changed, 112 insertions(+), 5 deletions(-) diff --git a/src/main/java/bio/overture/songsearch/graphql/FileDataFetcher.java b/src/main/java/bio/overture/songsearch/graphql/FileDataFetcher.java index ce420d9..5cadf55 100644 --- a/src/main/java/bio/overture/songsearch/graphql/FileDataFetcher.java +++ b/src/main/java/bio/overture/songsearch/graphql/FileDataFetcher.java @@ -18,8 +18,15 @@ package bio.overture.songsearch.graphql; +import static bio.overture.songsearch.utils.JacksonUtils.convertValue; +import static java.util.stream.Collectors.toUnmodifiableList; + +import bio.overture.songsearch.model.AggregationResult; import bio.overture.songsearch.model.File; +import bio.overture.songsearch.model.SearchResult; +import bio.overture.songsearch.model.Sort; import bio.overture.songsearch.service.FileService; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import graphql.schema.DataFetcher; import java.util.List; @@ -40,18 +47,40 @@ public FileDataFetcher(FileService fileService) { } @SuppressWarnings("unchecked") - public DataFetcher> getFilesDataFetcher() { + public DataFetcher> getFilesDataFetcher() { return environment -> { val args = environment.getArguments(); val filter = ImmutableMap.builder(); val page = ImmutableMap.builder(); + val sorts = ImmutableList.builder(); if (args != null) { if (args.get("filter") != null) filter.putAll((Map) args.get("filter")); if (args.get("page") != null) page.putAll((Map) args.get("page")); + if (args.get("sorts") != null) { + val rawSorts = (List) args.get("sorts"); + sorts.addAll( + rawSorts.stream() + .map(sort -> convertValue(sort, Sort.class)) + .collect(toUnmodifiableList())); + } + } + return fileService.searchFiles(filter.build(), page.build(), sorts.build()); + }; + } + + @SuppressWarnings("unchecked") + public DataFetcher getAggregateFilesDataFetcher() { + return environment -> { + val args = environment.getArguments(); + + val filter = ImmutableMap.builder(); + + if (args != null) { + if (args.get("filter") != null) filter.putAll((Map) args.get("filter")); } - return fileService.getFiles(filter.build(), page.build()); + return fileService.aggregateFiles(filter.build()); }; } } diff --git a/src/main/java/bio/overture/songsearch/graphql/GraphQLProvider.java b/src/main/java/bio/overture/songsearch/graphql/GraphQLProvider.java index e7b049c..3d5e798 100644 --- a/src/main/java/bio/overture/songsearch/graphql/GraphQLProvider.java +++ b/src/main/java/bio/overture/songsearch/graphql/GraphQLProvider.java @@ -129,8 +129,12 @@ private RuntimeWiring buildWiring() { .dataFetcher("analyses", analysisDataFetcher.getAnalysesDataFetcher())) .type( newTypeWiring("Query") - .dataFetcher("aggregateAnalyses", analysisDataFetcher.getAggregateAnalysesDataFetcher())) + .dataFetcher( + "aggregateAnalyses", analysisDataFetcher.getAggregateAnalysesDataFetcher())) .type(newTypeWiring("Query").dataFetcher("files", fileDataFetcher.getFilesDataFetcher())) + .type( + newTypeWiring("Query") + .dataFetcher("aggregateFiles", fileDataFetcher.getAggregateFilesDataFetcher())) .type( newTypeWiring("Query") .dataFetcher( diff --git a/src/main/java/bio/overture/songsearch/repository/FileRepository.java b/src/main/java/bio/overture/songsearch/repository/FileRepository.java index 4bff923..0dc920c 100644 --- a/src/main/java/bio/overture/songsearch/repository/FileRepository.java +++ b/src/main/java/bio/overture/songsearch/repository/FileRepository.java @@ -20,10 +20,14 @@ import static bio.overture.songsearch.config.constants.SearchFields.*; import static bio.overture.songsearch.utils.ElasticsearchQueryUtils.queryFromArgs; +import static bio.overture.songsearch.utils.ElasticsearchQueryUtils.sortsToEsSortBuilders; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.search.sort.SortOrder.ASC; import bio.overture.songsearch.config.ElasticsearchProperties; +import bio.overture.songsearch.model.Sort; import com.google.common.collect.ImmutableMap; +import java.util.List; import java.util.Map; import java.util.function.Function; import lombok.NonNull; @@ -39,6 +43,8 @@ import org.elasticsearch.index.query.NestedQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.SortBuilders; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -48,6 +54,8 @@ public class FileRepository { private static final Map>> QUERY_RESOLVER = argumentPathMap(); + private static final Map SORT_BUILDER_RESOLVER = sortPathMap(); + private final RestHighLevelClient client; private final String fileCentricIndex; @@ -75,13 +83,35 @@ private static Map>> argumentPa .build(); } + private static Map sortPathMap() { + return ImmutableMap.builder() + .put(FILE_OBJECT_ID, SortBuilders.fieldSort("object_id")) + .put(FILE_ACCESS, SortBuilders.fieldSort("file_access")) + .put(FILE_DATA_TYPE, SortBuilders.fieldSort("data_type")) + .put(FILE_NAME, SortBuilders.fieldSort("file.name")) + .build(); + } + public SearchResponse getFiles(Map filter, Map page) { + return getFiles(filter, page, List.of()); + } + + public SearchResponse getFiles( + Map filter, Map page, List sorts) { final AbstractQueryBuilder query = (filter == null || filter.size() == 0) ? matchAllQuery() : queryFromArgs(QUERY_RESOLVER, filter); val searchSourceBuilder = new SearchSourceBuilder(); + + if (sorts.isEmpty()) { + searchSourceBuilder.sort(SORT_BUILDER_RESOLVER.get(FILE_OBJECT_ID).order(ASC)); + } else { + val sortBuilders = sortsToEsSortBuilders(SORT_BUILDER_RESOLVER, sorts); + sortBuilders.forEach(searchSourceBuilder::sort); + } + searchSourceBuilder.query(query); if (page != null && page.size() != 0) { diff --git a/src/main/java/bio/overture/songsearch/service/FileService.java b/src/main/java/bio/overture/songsearch/service/FileService.java index 4b9a7d1..97da382 100644 --- a/src/main/java/bio/overture/songsearch/service/FileService.java +++ b/src/main/java/bio/overture/songsearch/service/FileService.java @@ -18,10 +18,12 @@ package bio.overture.songsearch.service; +import static bio.overture.songsearch.config.constants.EsDefaults.ES_PAGE_DEFAULT_FROM; +import static bio.overture.songsearch.config.constants.EsDefaults.ES_PAGE_DEFAULT_SIZE; import static bio.overture.songsearch.config.constants.SearchFields.FILE_OBJECT_ID; import static java.util.stream.Collectors.toUnmodifiableList; -import bio.overture.songsearch.model.File; +import bio.overture.songsearch.model.*; import bio.overture.songsearch.repository.FileRepository; import java.util.Arrays; import java.util.List; @@ -45,6 +47,30 @@ private static File hitToFile(SearchHit hit) { return File.parse(sourceMap); } + public SearchResult searchFiles( + Map filter, Map page, List sorts) { + val response = fileRepository.getFiles(filter, page, sorts); + val responseSearchHits = response.getHits(); + + val totalHits = responseSearchHits.getTotalHits().value; + val from = page.getOrDefault("from", ES_PAGE_DEFAULT_FROM); + val size = page.getOrDefault("size", ES_PAGE_DEFAULT_SIZE); + + val analyses = + Arrays.stream(responseSearchHits.getHits()) + .map(FileService::hitToFile) + .collect(toUnmodifiableList()); + val nextFrom = (totalHits - from) / size > 0; + return new SearchResult<>(analyses, nextFrom, totalHits); + } + + public AggregationResult aggregateFiles(Map filter) { + val response = fileRepository.getFiles(filter, Map.of(), List.of()); + val responseSearchHits = response.getHits(); + val totalHits = responseSearchHits.getTotalHits().value; + return new AggregationResult(totalHits); + } + public List getFiles(Map filter, Map page) { val response = fileRepository.getFiles(filter, page); val hitStream = Arrays.stream(response.getHits().getHits()); diff --git a/src/main/resources/schema.graphql b/src/main/resources/schema.graphql index f6ab8da..2d15e6f 100644 --- a/src/main/resources/schema.graphql +++ b/src/main/resources/schema.graphql @@ -160,6 +160,13 @@ enum AnalysisSortField { firstPublishedAt } +enum FileSortField { + objectId, + fileAccess, + dataType, + name +} + enum SortOrder { asc, desc @@ -170,6 +177,11 @@ input AnalysisSort { order: SortOrder! } +input FileSort { + fieldName: FileSortField! + order: SortOrder! +} + type SearchResultInfo { contentCount: String! hasNextFrom: String! @@ -181,6 +193,11 @@ type AnalysesSearchResult { info: SearchResultInfo! } +type FilesSearchResult { + content: [File!] + info: SearchResultInfo! +} + type AggregationResult { totalHits: String! } @@ -188,6 +205,7 @@ type AggregationResult { extend type Query { analyses(filter: AnalysisFilter, page: Page, sorts: [AnalysisSort!]): AnalysesSearchResult! aggregateAnalyses(filter: AnalysisFilter): AggregationResult! - files(filter: FileFilter, page: Page): [File] + files(filter: FileFilter, page: Page, sorts: [FileSort!]): FilesSearchResult! + aggregateFiles(filter: FileFilter): AggregationResult! sampleMatchedAnalysisPairs(analysisId: String!): [SampleMatchedAnalysisPair] } From 08d4793024863425f1c17d970ccbb8c79b9cca5f Mon Sep 17 00:00:00 2001 From: jaserud Date: Mon, 18 Jan 2021 10:06:26 -0500 Subject: [PATCH 6/7] update snapshot --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0f2f320..3cd4a0e 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ bio.overture song-search - 2.4.0 + 2.5.0-SNAPSHOT song-search GQL microservice for searching maestro generated song indexes From 83ce99250e11a7a2683e6c9af2c6b4695ab34cde Mon Sep 17 00:00:00 2001 From: jaserud Date: Mon, 18 Jan 2021 10:32:00 -0500 Subject: [PATCH 7/7] update to 2.5.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3cd4a0e..3592e9e 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ bio.overture song-search - 2.5.0-SNAPSHOT + 2.5.0 song-search GQL microservice for searching maestro generated song indexes