Skip to content

Commit

Permalink
Add Java doc
Browse files Browse the repository at this point in the history
Signed-off-by: Chen Dai <[email protected]>
  • Loading branch information
dai-chen committed Mar 26, 2024
1 parent 1328586 commit 5b1a49a
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,16 @@ object FlintSparkIndexRefresh extends Logging {
}
}

/**
* Validate if source table(s) of the given Flint index are not Hive table.
*
* @param spark
* Spark session
* @param index
* Flint index
* @return
* true if all non Hive, otherwise false
*/
def isSourceTableNonHive(spark: SparkSession, index: FlintSparkIndex): Boolean = {
// Extract source table name (possibly more than 1 for MV source query)
val tableNames = index match {
Expand All @@ -102,6 +112,17 @@ object FlintSparkIndexRefresh extends Logging {
}
}

/**
* Validate if checkpoint location is accessible (the folder exists and current Spark session
* has permission to access).
*
* @param spark
* Spark session
* @param checkpointLocation
* checkpoint location
* @return
* true if accessible, otherwise false
*/
def isCheckpointLocationAccessible(spark: SparkSession, checkpointLocation: String): Boolean = {
val checkpointPath = new Path(checkpointLocation)
val checkpointManager =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ class IncrementalIndexRefresh(indexName: String, index: FlintSparkIndex)

override def validate(spark: SparkSession): Unit = {
// Non-Hive table is required for incremental refresh
require(!isSourceTableNonHive(spark, index), "Flint index incremental refresh doesn't support Hive table")
require(
!isSourceTableNonHive(spark, index),
"Flint index incremental refresh doesn't support Hive table")

// Checkpoint location is required regardless of mandatory option
val options = index.options
Expand Down

0 comments on commit 5b1a49a

Please sign in to comment.