Skip to content

Commit

Permalink
Remove unused EMR info in Flint metadata
Browse files Browse the repository at this point in the history
Signed-off-by: Chen Dai <[email protected]>
  • Loading branch information
dai-chen committed Nov 3, 2023
1 parent 53b2f72 commit 3dc438a
Showing 1 changed file with 0 additions and 22 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -90,22 +90,6 @@ object FlintSparkIndex {
def flintIndexNamePrefix(fullTableName: String): String =
s"flint_${fullTableName.replace(".", "_")}_"

/**
* Populate environment variables to persist in Flint metadata.
*
* @return
* env key value mapping to populate
*/
def populateEnvToMetadata: Map[String, String] = {
// TODO: avoid hardcoding env name below by providing another config
val envNames = Seq("SERVERLESS_EMR_VIRTUAL_CLUSTER_ID", "SERVERLESS_EMR_JOB_ID")
envNames
.flatMap(key =>
Option(System.getenv(key))
.map(value => key -> value))
.toMap
}

/**
* Create Flint metadata builder with common fields.
*
Expand All @@ -120,12 +104,6 @@ object FlintSparkIndex {
builder.kind(index.kind)
builder.options(index.options.optionsWithDefault.mapValues(_.asInstanceOf[AnyRef]).asJava)

// Index properties
val envs = populateEnvToMetadata
if (envs.nonEmpty) {
builder.addProperty("env", envs.asJava)
}

// Optional index settings
val settings = index.options.indexSettings()
if (settings.isDefined) {
Expand Down

0 comments on commit 3dc438a

Please sign in to comment.