From 5c0293515aab3fd0d426a1af93bebb7e912e1de0 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Mon, 11 Dec 2023 11:36:35 -0800 Subject: [PATCH] Update javadoc and user manual Signed-off-by: Chen Dai --- docs/index.md | 15 +++++++++++++++ .../core/storage/FlintOpenSearchMetadataLog.java | 2 +- .../FlintSparkMaterializedViewSqlITSuite.scala | 4 +++- .../spark/FlintSparkTransactionITSuite.scala | 1 + 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index ea6778f39..2c6db8576 100644 --- a/docs/index.md +++ b/docs/index.md @@ -110,6 +110,12 @@ writer.close() ``` +### Index State Transition + +Flint index state transition: + +![FlintCoreIndexState](https://private-user-images.githubusercontent.com/46505291/289648864-8033f981-e406-498d-a3cc-d49d38c6e539.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTEiLCJleHAiOjE3MDIzMjM0MTgsIm5iZiI6MTcwMjMyMzExOCwicGF0aCI6Ii80NjUwNTI5MS8yODk2NDg4NjQtODAzM2Y5ODEtZTQwNi00OThkLWEzY2MtZDQ5ZDM4YzZlNTM5LnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFJV05KWUFYNENTVkVINTNBJTJGMjAyMzEyMTElMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjMxMjExVDE5MzE1OFomWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPTE5MzU4MDZlNjM4NTgzN2Y2MjFmMjBlNjI0NjRkOTFhYWI3MzNkYTg1NzlmYmZjNDUzNGI4ZDU4ODlmOTA2MmQmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0JmFjdG9yX2lkPTAma2V5X2lkPTAmcmVwb19pZD0wIn0.bEQTiNMwsJWbpbzQ42QrYITcgpkn8yLsIBOfZubtXLI) + ### API High level API is dependent on query engine implementation. Please see Query Engine Integration section for details. @@ -435,8 +441,17 @@ flint.materializedView() .create() flint.refreshIndex("flint_spark_catalog_default_alb_logs_metrics") + +flint.deleteIndex("flint_spark_catalog_default_alb_logs_skipping_index") +flint.vacuumIndex("flint_spark_catalog_default_alb_logs_skipping_index") ``` +#### Index State Transition + +Flint Spark index state transition: + +![FlintSparkIndexState](https://private-user-images.githubusercontent.com/46505291/289648896-24306a61-ecf3-4969-9ce1-107dd4ce9d62.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTEiLCJleHAiOjE3MDIzMjM0MTgsIm5iZiI6MTcwMjMyMzExOCwicGF0aCI6Ii80NjUwNTI5MS8yODk2NDg4OTYtMjQzMDZhNjEtZWNmMy00OTY5LTljZTEtMTA3ZGQ0Y2U5ZDYyLnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFJV05KWUFYNENTVkVINTNBJTJGMjAyMzEyMTElMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjMxMjExVDE5MzE1OFomWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPWRiMGYzN2FmM2E0M2RhOTA5OTI3NGY2ZTA1MTAyNmIyYTI5Y2U2Mjg2OGE4NmViNGEzOWE0YjZiNTQxY2I2Y2EmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0JmFjdG9yX2lkPTAma2V5X2lkPTAmcmVwb19pZD0wIn0.vAnMt42tKlVI6612pXO6zWexk9djCXHZzRA9eZub57I) + #### Skipping Index Provider SPI ```scala diff --git a/flint-core/src/main/scala/org/opensearch/flint/core/storage/FlintOpenSearchMetadataLog.java b/flint-core/src/main/scala/org/opensearch/flint/core/storage/FlintOpenSearchMetadataLog.java index f7eed019f..ab38a5f60 100644 --- a/flint-core/src/main/scala/org/opensearch/flint/core/storage/FlintOpenSearchMetadataLog.java +++ b/flint-core/src/main/scala/org/opensearch/flint/core/storage/FlintOpenSearchMetadataLog.java @@ -109,7 +109,7 @@ public void purge() { LOG.info("Purged log entry with result " + response.getResult()); } catch (Exception e) { - throw new IllegalStateException("Failed to purge metadata log entry", e); + throw new IllegalStateException("Failed to purge log entry", e); } } diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewSqlITSuite.scala b/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewSqlITSuite.scala index bc3433d11..f9bd3967a 100644 --- a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewSqlITSuite.scala +++ b/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewSqlITSuite.scala @@ -182,7 +182,9 @@ class FlintSparkMaterializedViewSqlITSuite extends FlintSparkSuite { test("show all materialized views in catalog and database") { // Show in catalog flint.materializedView().name("spark_catalog.default.mv1").query(testQuery).create() - checkAnswer(sql(s"SHOW MATERIALIZED VIEW IN spark_catalog"), Seq(Row("mv1"))) + checkAnswer( + sql(s"SHOW MATERIALIZED VIEW IN spark_catalog"), + Seq(Row("mv1"))) // Show in catalog.database flint.materializedView().name("spark_catalog.default.mv2").query(testQuery).create() diff --git a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkTransactionITSuite.scala b/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkTransactionITSuite.scala index d32405c7d..643a35516 100644 --- a/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkTransactionITSuite.scala +++ b/integ-test/src/test/scala/org/opensearch/flint/spark/FlintSparkTransactionITSuite.scala @@ -128,6 +128,7 @@ class FlintSparkTransactionITSuite extends OpenSearchTransactionSuite with Match deleteLogically(testLatestId) flint.vacuumIndex(testFlintIndex) + // Both index data and metadata log should be vacuumed openSearchClient .indices() .exists(new GetIndexRequest(testFlintIndex), RequestOptions.DEFAULT) shouldBe false