From 36e39dd1f6b6ecfdf5fef00728fd5e88ca352237 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Fri, 21 Apr 2023 15:51:50 -0700 Subject: [PATCH 1/3] Create scala project folder for Flint Signed-off-by: Chen Dai --- flint/.bsp/sbt.json | 1 + flint/.gitignore | 20 +++++++++++ flint/build.sbt | 34 +++++++++++++++++++ .../opensearch/flint/core/FlintIndex.scala | 10 ++++++ .../flint/spark/FlintSparkExtensions.scala | 14 ++++++++ flint/project/build.properties | 1 + 6 files changed, 80 insertions(+) create mode 100644 flint/.bsp/sbt.json create mode 100644 flint/.gitignore create mode 100644 flint/build.sbt create mode 100644 flint/flint-core/src/main/scala/org/opensearch/flint/core/FlintIndex.scala create mode 100644 flint/flint-spark-integration/src/main/scala/org/opensearch/flint/spark/FlintSparkExtensions.scala create mode 100644 flint/project/build.properties diff --git a/flint/.bsp/sbt.json b/flint/.bsp/sbt.json new file mode 100644 index 0000000000..654c8f42cd --- /dev/null +++ b/flint/.bsp/sbt.json @@ -0,0 +1 @@ +{"name":"sbt","version":"1.8.2","bspVersion":"2.1.0-M1","languages":["scala"],"argv":["/Users/daichen/Library/Java/JavaVirtualMachines/openjdk-17.0.2/Contents/Home/bin/java","-Xms100m","-Xmx100m","-classpath","/Users/daichen/Library/Application Support/JetBrains/IntelliJIdea2021.3/plugins/Scala/launcher/sbt-launch.jar","xsbt.boot.Boot","-bsp","--sbt-launch-jar=/Users/daichen/Library/Application%20Support/JetBrains/IntelliJIdea2021.3/plugins/Scala/launcher/sbt-launch.jar"]} \ No newline at end of file diff --git a/flint/.gitignore b/flint/.gitignore new file mode 100644 index 0000000000..96ac86edf0 --- /dev/null +++ b/flint/.gitignore @@ -0,0 +1,20 @@ +# IDE files +.idea/ +*.iml + +# Compiled output +target/ +project/target/ + +# Log files +logs/ + +# sbt-specific files +.sbtserver +.sbt/ + +# Miscellaneous +.DS_Store +*.class +*.log +*.zip diff --git a/flint/build.sbt b/flint/build.sbt new file mode 100644 index 0000000000..541d2d5734 --- /dev/null +++ b/flint/build.sbt @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +lazy val scala212 = "2.12.14" +lazy val sparkVersion = "3.3.1" + +ThisBuild / version := "0.1.0-SNAPSHOT" + +ThisBuild / scalaVersion := scala212 + +lazy val root = (project in file(".")) + .aggregate(flintCore, flintSparkIntegration) + .settings( + name := "flint" + ) + +lazy val flintCore = (project in file("flint-core")) + .settings( + name := "flint-core", + scalaVersion := scala212 + ) + +lazy val flintSparkIntegration = (project in file("flint-spark-integration")) + .dependsOn(flintCore) + .settings( + name := "flint-spark-integration", + scalaVersion := scala212, + libraryDependencies ++= Seq( + "org.apache.spark" %% "spark-core" % sparkVersion, + "org.apache.spark" %% "spark-sql" % sparkVersion + ) + ) diff --git a/flint/flint-core/src/main/scala/org/opensearch/flint/core/FlintIndex.scala b/flint/flint-core/src/main/scala/org/opensearch/flint/core/FlintIndex.scala new file mode 100644 index 0000000000..e00b9ad91f --- /dev/null +++ b/flint/flint-core/src/main/scala/org/opensearch/flint/core/FlintIndex.scala @@ -0,0 +1,10 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.flint.core + +class FlintIndex { + +} diff --git a/flint/flint-spark-integration/src/main/scala/org/opensearch/flint/spark/FlintSparkExtensions.scala b/flint/flint-spark-integration/src/main/scala/org/opensearch/flint/spark/FlintSparkExtensions.scala new file mode 100644 index 0000000000..4b3c7473f7 --- /dev/null +++ b/flint/flint-spark-integration/src/main/scala/org/opensearch/flint/spark/FlintSparkExtensions.scala @@ -0,0 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.flint.spark + +import org.apache.spark.sql.SparkSessionExtensions + +class FlintSparkExtensions extends (SparkSessionExtensions => Unit) { + + override def apply(v1: SparkSessionExtensions): Unit = { + } +} diff --git a/flint/project/build.properties b/flint/project/build.properties new file mode 100644 index 0000000000..f344c14837 --- /dev/null +++ b/flint/project/build.properties @@ -0,0 +1 @@ +sbt.version = 1.8.2 From 6a0dfb8224a758bb59de1245808379218a02c63c Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Fri, 21 Apr 2023 16:03:56 -0700 Subject: [PATCH 2/3] Ignore .bsp folder Signed-off-by: Chen Dai --- flint/.bsp/sbt.json | 1 - flint/.gitignore | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 flint/.bsp/sbt.json diff --git a/flint/.bsp/sbt.json b/flint/.bsp/sbt.json deleted file mode 100644 index 654c8f42cd..0000000000 --- a/flint/.bsp/sbt.json +++ /dev/null @@ -1 +0,0 @@ -{"name":"sbt","version":"1.8.2","bspVersion":"2.1.0-M1","languages":["scala"],"argv":["/Users/daichen/Library/Java/JavaVirtualMachines/openjdk-17.0.2/Contents/Home/bin/java","-Xms100m","-Xmx100m","-classpath","/Users/daichen/Library/Application Support/JetBrains/IntelliJIdea2021.3/plugins/Scala/launcher/sbt-launch.jar","xsbt.boot.Boot","-bsp","--sbt-launch-jar=/Users/daichen/Library/Application%20Support/JetBrains/IntelliJIdea2021.3/plugins/Scala/launcher/sbt-launch.jar"]} \ No newline at end of file diff --git a/flint/.gitignore b/flint/.gitignore index 96ac86edf0..34602292e0 100644 --- a/flint/.gitignore +++ b/flint/.gitignore @@ -12,6 +12,7 @@ logs/ # sbt-specific files .sbtserver .sbt/ +.bsp/ # Miscellaneous .DS_Store From dfcc89bad0a2bce1a0d03956df0d60d68bd8d268 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Fri, 21 Apr 2023 16:39:54 -0700 Subject: [PATCH 3/3] Add README Signed-off-by: Chen Dai --- flint/README.md | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 flint/README.md diff --git a/flint/README.md b/flint/README.md new file mode 100644 index 0000000000..ce56c50f47 --- /dev/null +++ b/flint/README.md @@ -0,0 +1,43 @@ +# OpenSearch Flint + +OpenSearch Flint is ... It consists of two modules: + +- `flint-core`: a module that contains Flint specification and client. +- `flint-spark-integration`: a module that provides Spark integration for Flint and derived dataset based on it. + +## Prerequisites + ++ Spark 3.3.1 ++ Scala 2.12.14 + +## Usage + +To use this application, you can run Spark with Flint extension: + +``` +spark-sql --conf "spark.sql.extensions=org.opensearch.flint.FlintSparkExtensions" +``` + +## Build + +To build and run this application with Spark, you can run: + +``` +sbt clean publishLocal +``` + +## Code of Conduct + +This project has adopted an [Open Source Code of Conduct](../CODE_OF_CONDUCT.md). + +## Security + +If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public GitHub issue. + +## License + +See the [LICENSE](../LICENSE.txt) file for our project's licensing. We will ask you to confirm the licensing of your contribution. + +## Copyright + +Copyright OpenSearch Contributors. See [NOTICE](../NOTICE) for details. \ No newline at end of file