From c8ee7d7aba0c4a8cb93a7af56df8d82530c67962 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:30:15 -0700 Subject: [PATCH] Add README to async-query-core (#2766) (#2770) (cherry picked from commit fbff4a346f0b3bd909011939e9e826cb5ec91f4a) Signed-off-by: Tomoyuki Morita Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- async-query-core/README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 async-query-core/README.md diff --git a/async-query-core/README.md b/async-query-core/README.md new file mode 100644 index 0000000000..61b6057269 --- /dev/null +++ b/async-query-core/README.md @@ -0,0 +1,32 @@ +# async-query-core library + +This directory contains async-query-core library, which implements the core logic of async-query and provide extension points to allow plugin different implementation of data storage, etc. +`async-query` module provides implementations for OpenSearch index based implementation. + +## Type of queries +There are following types of queries, and the type is automatically identified by analysing the query. +- BatchQuery: Execute single query in Spark +- InteractiveQuery: Establish session and execute queries in single Spark session +- IndexDMLQuery: Handles DROP/ALTER/VACUUM operation for Flint indices +- RefreshQuery: One time query request to refresh(update) Flint index +- StreamingQuery: Continuously update flint index in single Spark session + +## Extension points +Following is the list of extension points where the consumer of the library needs to provide their own implementation. + +- Data store interface + - [AsyncQueryJobMetadataStorageService](src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java) + - [SessionStorageService](java/org/opensearch/sql/spark/execution/statestore/SessionStorageService.java) + - [StatementStorageService](src/main/java/org/opensearch/sql/spark/execution/statestore/StatementStorageService.java) + - [FlintIndexMetadataService](src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataService.java) + - [FlintIndexStateModelService](src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModelService.java) + - [IndexDMLResultStorageService](src/main/java/org/opensearch/sql/spark/flint/IndexDMLResultStorageService.java) +- Other + - [LeaseManager](src/main/java/org/opensearch/sql/spark/leasemanager/LeaseManager.java) + - [JobExecutionResponseReader](src/main/java/org/opensearch/sql/spark/response/JobExecutionResponseReader.java) + - [QueryIdProvider](src/main/java/org/opensearch/sql/spark/dispatcher/QueryIdProvider.java) + - [SessionIdProvider](src/main/java/org/opensearch/sql/spark/execution/session/SessionIdProvider.java) + - [SessionConfigSupplier](src/main/java/org/opensearch/sql/spark/execution/session/SessionConfigSupplier.java) + - [SparkExecutionEngineConfigSupplier](src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigSupplier.java) + - [SparkSubmitParameterModifier](src/main/java/org/opensearch/sql/spark/config/SparkSubmitParameterModifier.java) + - [EMRServerlessClientFactory](src/main/java/org/opensearch/sql/spark/client/EMRServerlessClientFactory.java)