forked from opensearch-project/neural-search
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adding Reciprocal Rank Fusion (RRF) in hybrid query (opensearch-proje…
…ct#1086) * Reciprocal Rank Fusion (RRF) normalization technique in hybrid query (opensearch-project#874) --------- Signed-off-by: Isaac Johnson <[email protected]> Signed-off-by: Ryan Bogan <[email protected]> Signed-off-by: Martin Gaievski <[email protected]>
- Loading branch information
Showing
46 changed files
with
2,245 additions
and
260 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
65 changes: 65 additions & 0 deletions
65
src/main/java/org/opensearch/neuralsearch/processor/AbstractScoreHybridizationProcessor.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package org.opensearch.neuralsearch.processor; | ||
|
|
||
| import org.opensearch.action.search.SearchPhaseContext; | ||
| import org.opensearch.action.search.SearchPhaseResults; | ||
| import org.opensearch.search.SearchPhaseResult; | ||
| import org.opensearch.search.internal.SearchContext; | ||
| import org.opensearch.search.pipeline.PipelineProcessingContext; | ||
| import org.opensearch.search.pipeline.SearchPhaseResultsProcessor; | ||
|
|
||
| import java.util.Optional; | ||
|
|
||
| /** | ||
| * Base class for all score hybridization processors. This class is responsible for executing the score hybridization process. | ||
| * It is a pipeline processor that is executed after the query phase and before the fetch phase. | ||
| */ | ||
| public abstract class AbstractScoreHybridizationProcessor implements SearchPhaseResultsProcessor { | ||
| /** | ||
| * Method abstracts functional aspect of score normalization and score combination. Exact methods for each processing stage | ||
| * are set as part of class constructor. This method is called when there is no pipeline context | ||
| * @param searchPhaseResult {@link SearchPhaseResults} DTO that has query search results. Results will be mutated as part of this method execution | ||
| * @param searchPhaseContext {@link SearchContext} | ||
| */ | ||
| @Override | ||
| public <Result extends SearchPhaseResult> void process( | ||
| final SearchPhaseResults<Result> searchPhaseResult, | ||
| final SearchPhaseContext searchPhaseContext | ||
| ) { | ||
| hybridizeScores(searchPhaseResult, searchPhaseContext, Optional.empty()); | ||
| } | ||
|
|
||
| /** | ||
| * Method abstracts functional aspect of score normalization and score combination. Exact methods for each processing stage | ||
| * are set as part of class constructor. This method is called when there is pipeline context | ||
| * @param searchPhaseResult {@link SearchPhaseResults} DTO that has query search results. Results will be mutated as part of this method execution | ||
| * @param searchPhaseContext {@link SearchContext} | ||
| * @param requestContext {@link PipelineProcessingContext} processing context of search pipeline | ||
| * @param <Result> | ||
| */ | ||
| @Override | ||
| public <Result extends SearchPhaseResult> void process( | ||
| final SearchPhaseResults<Result> searchPhaseResult, | ||
| final SearchPhaseContext searchPhaseContext, | ||
| final PipelineProcessingContext requestContext | ||
| ) { | ||
| hybridizeScores(searchPhaseResult, searchPhaseContext, Optional.ofNullable(requestContext)); | ||
| } | ||
|
|
||
| /** | ||
| * Method abstracts functional aspect of score normalization and score combination. Exact methods for each processing stage | ||
| * are set as part of class constructor | ||
| * @param searchPhaseResult | ||
| * @param searchPhaseContext | ||
| * @param requestContextOptional | ||
| * @param <Result> | ||
| */ | ||
| abstract <Result extends SearchPhaseResult> void hybridizeScores( | ||
| SearchPhaseResults<Result> searchPhaseResult, | ||
| SearchPhaseContext searchPhaseContext, | ||
| Optional<PipelineProcessingContext> requestContextOptional | ||
| ); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
35 changes: 35 additions & 0 deletions
35
src/main/java/org/opensearch/neuralsearch/processor/NormalizationExecuteDTO.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package org.opensearch.neuralsearch.processor; | ||
|
|
||
| import lombok.AllArgsConstructor; | ||
| import lombok.Builder; | ||
| import lombok.Getter; | ||
| import lombok.NonNull; | ||
| import org.opensearch.neuralsearch.processor.combination.ScoreCombinationTechnique; | ||
| import org.opensearch.neuralsearch.processor.normalization.ScoreNormalizationTechnique; | ||
| import org.opensearch.search.fetch.FetchSearchResult; | ||
| import org.opensearch.search.query.QuerySearchResult; | ||
|
|
||
| import java.util.List; | ||
| import java.util.Optional; | ||
|
|
||
| /** | ||
| * DTO object to hold data in NormalizationProcessorWorkflow class | ||
| * in NormalizationProcessorWorkflow. | ||
| */ | ||
| @AllArgsConstructor | ||
| @Builder | ||
| @Getter | ||
| public class NormalizationExecuteDTO { | ||
| @NonNull | ||
| private List<QuerySearchResult> querySearchResults; | ||
| @NonNull | ||
| private Optional<FetchSearchResult> fetchSearchResultOptional; | ||
| @NonNull | ||
| private ScoreNormalizationTechnique normalizationTechnique; | ||
| @NonNull | ||
| private ScoreCombinationTechnique combinationTechnique; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
26 changes: 26 additions & 0 deletions
26
src/main/java/org/opensearch/neuralsearch/processor/NormalizeScoresDTO.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package org.opensearch.neuralsearch.processor; | ||
|
|
||
| import lombok.AllArgsConstructor; | ||
| import lombok.Builder; | ||
| import lombok.Getter; | ||
| import lombok.NonNull; | ||
| import org.opensearch.neuralsearch.processor.normalization.ScoreNormalizationTechnique; | ||
|
|
||
| import java.util.List; | ||
|
|
||
| /** | ||
| * DTO object to hold data required for score normalization. | ||
| */ | ||
| @AllArgsConstructor | ||
| @Builder | ||
| @Getter | ||
| public class NormalizeScoresDTO { | ||
| @NonNull | ||
| private List<CompoundTopDocs> queryTopDocs; | ||
| @NonNull | ||
| private ScoreNormalizationTechnique normalizationTechnique; | ||
| } |
Oops, something went wrong.