diff --git a/flint/.gitignore b/flint/.gitignore new file mode 100644 index 0000000000..34602292e0 --- /dev/null +++ b/flint/.gitignore @@ -0,0 +1,21 @@ +# IDE files +.idea/ +*.iml + +# Compiled output +target/ +project/target/ + +# Log files +logs/ + +# sbt-specific files +.sbtserver +.sbt/ +.bsp/ + +# Miscellaneous +.DS_Store +*.class +*.log +*.zip diff --git a/flint/README.md b/flint/README.md new file mode 100644 index 0000000000..ce56c50f47 --- /dev/null +++ b/flint/README.md @@ -0,0 +1,43 @@ +# OpenSearch Flint + +OpenSearch Flint is ... It consists of two modules: + +- `flint-core`: a module that contains Flint specification and client. +- `flint-spark-integration`: a module that provides Spark integration for Flint and derived dataset based on it. + +## Prerequisites + ++ Spark 3.3.1 ++ Scala 2.12.14 + +## Usage + +To use this application, you can run Spark with Flint extension: + +``` +spark-sql --conf "spark.sql.extensions=org.opensearch.flint.FlintSparkExtensions" +``` + +## Build + +To build and run this application with Spark, you can run: + +``` +sbt clean publishLocal +``` + +## Code of Conduct + +This project has adopted an [Open Source Code of Conduct](../CODE_OF_CONDUCT.md). + +## Security + +If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public GitHub issue. + +## License + +See the [LICENSE](../LICENSE.txt) file for our project's licensing. We will ask you to confirm the licensing of your contribution. + +## Copyright + +Copyright OpenSearch Contributors. See [NOTICE](../NOTICE) for details. \ No newline at end of file diff --git a/flint/build.sbt b/flint/build.sbt new file mode 100644 index 0000000000..541d2d5734 --- /dev/null +++ b/flint/build.sbt @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +lazy val scala212 = "2.12.14" +lazy val sparkVersion = "3.3.1" + +ThisBuild / version := "0.1.0-SNAPSHOT" + +ThisBuild / scalaVersion := scala212 + +lazy val root = (project in file(".")) + .aggregate(flintCore, flintSparkIntegration) + .settings( + name := "flint" + ) + +lazy val flintCore = (project in file("flint-core")) + .settings( + name := "flint-core", + scalaVersion := scala212 + ) + +lazy val flintSparkIntegration = (project in file("flint-spark-integration")) + .dependsOn(flintCore) + .settings( + name := "flint-spark-integration", + scalaVersion := scala212, + libraryDependencies ++= Seq( + "org.apache.spark" %% "spark-core" % sparkVersion, + "org.apache.spark" %% "spark-sql" % sparkVersion + ) + ) diff --git a/flint/flint-core/src/main/scala/org/opensearch/flint/core/FlintIndex.scala b/flint/flint-core/src/main/scala/org/opensearch/flint/core/FlintIndex.scala new file mode 100644 index 0000000000..e00b9ad91f --- /dev/null +++ b/flint/flint-core/src/main/scala/org/opensearch/flint/core/FlintIndex.scala @@ -0,0 +1,10 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.flint.core + +class FlintIndex { + +} diff --git a/flint/flint-spark-integration/src/main/scala/org/opensearch/flint/spark/FlintSparkExtensions.scala b/flint/flint-spark-integration/src/main/scala/org/opensearch/flint/spark/FlintSparkExtensions.scala new file mode 100644 index 0000000000..4b3c7473f7 --- /dev/null +++ b/flint/flint-spark-integration/src/main/scala/org/opensearch/flint/spark/FlintSparkExtensions.scala @@ -0,0 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.flint.spark + +import org.apache.spark.sql.SparkSessionExtensions + +class FlintSparkExtensions extends (SparkSessionExtensions => Unit) { + + override def apply(v1: SparkSessionExtensions): Unit = { + } +} diff --git a/flint/project/build.properties b/flint/project/build.properties new file mode 100644 index 0000000000..f344c14837 --- /dev/null +++ b/flint/project/build.properties @@ -0,0 +1 @@ +sbt.version = 1.8.2