Merge pull request #60 from badrinathpatchikolla/master

fixed test cases issue and added csv option in target deserializer
music-of-the-ainur · Oct 4, 2022 · 329c632 · 329c632
2 parents fc58c7c + 8d39204
commit 329c632
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 51 deletions.
diff --git a/src/main/scala/com/github/music/of/the/ainur/almaren/builder/core/Deserializer.scala b/src/main/scala/com/github/music/of/the/ainur/almaren/builder/core/Deserializer.scala
@@ -22,6 +22,7 @@ private[almaren] trait Deserializer extends Core {
       case "JSON" => json
       case "XML" => xml
       case "AVRO" => avro
+      case "CSV" => csv
       case d => throw InvalidDecoder(d)
     }
   }

diff --git a/src/test/scala/com/github/music/of/the/ainur/almaren/Test.scala b/src/test/scala/com/github/music/of/the/ainur/almaren/Test.scala
@@ -251,57 +251,6 @@ class Test extends FunSuite with BeforeAndAfter {
     }
   }
 
-  def testSourceSql(tableName: String): DataFrame = {
-    almaren.builder
-      .sourceSql(s"select * from $tableName")
-      .batch
-
-  }
-
-  def testTargetFileTarget(format: String, path: String, saveMode: SaveMode, params: Map[String, String], partitionBy: List[String], bucketBy: (Int, List[String]), sortBy: List[String], tableName: Option[String]): DataFrame = {
-    almaren.builder
-      .sourceDataFrame(movies)
-      .targetFile(format, path, saveMode, params, partitionBy, bucketBy, sortBy, tableName)
-      .batch
-  }
-
-  def testTargetFileBucketPartition(path: String, partitionBy: List[String], bucketBy: (Int, List[String]),fileFormat: String) = {
-    val filesList = getListOfDirectories(path).map(_.toString)
-    if (partitionBy.nonEmpty) {
-      val extractFiles = filesList.map(a => a.substring(a.lastIndexOf("=") + 1))
-      val distinctValues = movies.select(partitionBy(0)).distinct.as[String].collect.toList
-      val checkLists = extractFiles.intersect(distinctValues)
-      test(s"partitionBy_$fileFormat") {
-        assert(checkLists.size == distinctValues.size)
-      }
-    }
-    if (bucketBy._2.nonEmpty) {
-      val check = filesList.map(f => getListOfFiles(f).size)
-      val bool = if (check.forall(_ == check.head)) check.head == 2 * bucketBy._1 else false
-      test(s"bucketBy_$fileFormat") {
-        assert(bool == true)
-      }
-    }
-  }
-
-  def getListOfDirectories(dir: String): List[File] = {
-    val d = new File(dir)
-    if (d.exists && d.isDirectory) {
-      d.listFiles.filter(_.isDirectory).toList
-    } else {
-      List[File]()
-    }
-  }
-
-  def getListOfFiles(dir: String): List[File] = {
-    val d = new File(dir)
-    if (d.exists && d.isDirectory) {
-      d.listFiles.filter(_.isFile).toList
-    } else {
-      List[File]()
-    }
-  }
-
   def repartitionAndColaeseTest(dataFrame: DataFrame) {
     val repartition_df = almaren.builder.sourceSql(s"select * from $testTable")
       .repartition(10).batch