Skip to content

Commit

Permalink
Revert emr env changes
Browse files Browse the repository at this point in the history
Signed-off-by: Chen Dai <[email protected]>
  • Loading branch information
dai-chen committed Nov 8, 2023
1 parent d1280cf commit c35ab43
Showing 1 changed file with 22 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,22 @@ object FlintSparkIndex {
def flintIndexNamePrefix(fullTableName: String): String =
s"flint_${fullTableName.replace(".", "_")}_"

/**
* Populate environment variables to persist in Flint metadata.
*
* @return
* env key value mapping to populate
*/
def populateEnvToMetadata: Map[String, String] = {
// TODO: avoid hardcoding env name below by providing another config
val envNames = Seq("SERVERLESS_EMR_VIRTUAL_CLUSTER_ID", "SERVERLESS_EMR_JOB_ID")
envNames
.flatMap(key =>
Option(System.getenv(key))
.map(value => key -> value))
.toMap
}

/**
* Create Flint metadata builder with common fields.
*
Expand All @@ -104,6 +120,12 @@ object FlintSparkIndex {
builder.kind(index.kind)
builder.options(index.options.optionsWithDefault.mapValues(_.asInstanceOf[AnyRef]).asJava)

// Index properties
val envs = populateEnvToMetadata
if (envs.nonEmpty) {
builder.addProperty("env", envs.asJava)
}

// Optional index settings
val settings = index.options.indexSettings()
if (settings.isDefined) {
Expand Down

0 comments on commit c35ab43

Please sign in to comment.