From 166d52a254e3760516df83e79e2db4e3341849cc Mon Sep 17 00:00:00 2001 From: Na Lee Ha Date: Mon, 18 Dec 2023 13:53:00 +0000 Subject: [PATCH] #1052 using the new basket provider and deleting old csv static loader --- .../module/basket/csv/CsvStaticLoader.scala | 117 ------------------ .../provider/BasketConstituentProvider.scala | 7 +- .../basket/provider/BasketProvider.scala | 7 +- .../finos/vuu/csv/CsvStaticLoaderTests.scala | 42 ------- 4 files changed, 7 insertions(+), 166 deletions(-) delete mode 100644 example/basket/src/main/scala/org/finos/vuu/core/module/basket/csv/CsvStaticLoader.scala delete mode 100644 example/basket/src/test/scala/org/finos/vuu/csv/CsvStaticLoaderTests.scala diff --git a/example/basket/src/main/scala/org/finos/vuu/core/module/basket/csv/CsvStaticLoader.scala b/example/basket/src/main/scala/org/finos/vuu/core/module/basket/csv/CsvStaticLoader.scala deleted file mode 100644 index 9fef0d572..000000000 --- a/example/basket/src/main/scala/org/finos/vuu/core/module/basket/csv/CsvStaticLoader.scala +++ /dev/null @@ -1,117 +0,0 @@ -package org.finos.vuu.core.module.basket.csv - -import com.typesafe.scalalogging.StrictLogging - -import java.io.File -import scala.io.Source -import scala.util.control.NonFatal - -object CsvStaticLoader extends StrictLogging { - def loadConstituent(basketId: String, resourcePath: Option[String] = None): Array[Map[String, Any]] = { - try { - - val constituentsFilesDirectory = - if(resourcePath.isDefined) resourcePath.get - else getClass.getResource("/static").getPath - - val dir = new File(constituentsFilesDirectory) - val csvFiles = dir.listFiles.filter(_.isFile) - .filter(_.getName.endsWith(basketId.replace(".", "").toLowerCase + ".csv")) - - if (csvFiles.isEmpty) { - logger.error(s"Failed to find constituents file for $basketId") - Array.empty - } - else { - val csvFile = csvFiles(0) - logger.info("Loading basket static:" + basketId + "(" + csvFile + ")") - val rows = readFileContent(csvFile) - logger.info(s"Found ${rows.length} constituents for basket $basketId") - - val header = rows(0) - val symbolInd = header.indexOf("Symbol") - val nameInd = header.indexOf("Name") - val lastTradeInd = header.indexOf("Last Trade") - val volumeInd = header.indexOf("Volume") - val weightInd = header.indexOf("Weighting") - val changeInd = header.indexOf("Change") - - val constituents = rows.tail.map(e => { - val weighting = if (getValueFromIndex(weightInd, e) == null) 0.0D else getValueFromIndex(weightInd, e).toDouble - Map[String, Any]( - "Symbol" -> getValueFromIndex(symbolInd, e), - "Last Trade" -> getValueFromIndex(lastTradeInd, e), - "Name" -> getValueFromIndex(nameInd, e), - "Weighting" -> weighting, - "Volume" -> getValueFromIndex(volumeInd, e), - "Change" -> getValueFromIndex(changeInd, e) - ) - }) - constituents - - } - } - catch { - case NonFatal(t) => logger.error(s"Failed to parse constituents for $basketId", t) - Array.empty - } - } - - - private def readFileContent(csvFile: File): Array[Array[String]] = { - val bufferedSource = Source.fromFile(csvFile) - val csv = for (line <- bufferedSource.getLines) yield line.split(",").map(_.trim) - val array = csv.toArray - bufferedSource.close - array - } - - def load: Array[String] = { - val staticDirPath = getClass.getResource("/static").getPath - val dir = new java.io.File(staticDirPath) - val csvFiles = dir.listFiles.filter(_.isFile) - .filter(_.getName.endsWith(".csv")) - val ids = csvFiles.map(e => "." + e.getName.replace(".csv", "").toUpperCase) - ids - } - - def loadStatic: Map[String, Array[Map[String, String]]] = { - var map: Map[String, Array[Map[String, String]]] = Map() - val staticDirPath = getClass.getResource("/static").getPath - val dir = new java.io.File(staticDirPath) - val csvFiles = dir.listFiles.filter(_.isFile) - .filter(_.getName.endsWith(".csv")) - - csvFiles.foreach(csvFile => { - println(csvFile) - - val bufferedSource = Source.fromFile(csvFile) - val csv = for (line <- bufferedSource.getLines) yield line.split(",").map(_.trim) - val array = csv.toArray - bufferedSource.close - var data: Map[String, String] = Map() - val header = array(0) - val symbolInd = header.indexOf("Symbol") - val nameInd = header.indexOf("Name") - val lastTradeInd = header.indexOf("Last Trade") - val volumeInd = header.indexOf("Volume") - val weightInd = header.indexOf("Weight") - val changeInd = header.indexOf("Change") - val list = array.tail.map(e => Map( - "Symbol" -> getValueFromIndex(symbolInd, e), - "Last Trade" -> getValueFromIndex(lastTradeInd, e), - "Name" -> getValueFromIndex(nameInd, e), - "Weight" -> getValueFromIndex(weightInd, e), - "Volume" -> getValueFromIndex(volumeInd, e), - "Change" -> getValueFromIndex(changeInd, e) - )) - val fileName = csvFile.getName.replace(".csv", "") - map += ("." + fileName.toUpperCase -> list) - }) - map - } - - private def getValueFromIndex(index: Int, e: Array[String]) = { - if (e.length > index && index > -1) e(index) else null - } -} diff --git a/example/basket/src/main/scala/org/finos/vuu/core/module/basket/provider/BasketConstituentProvider.scala b/example/basket/src/main/scala/org/finos/vuu/core/module/basket/provider/BasketConstituentProvider.scala index a87c400ed..eac0cfbab 100644 --- a/example/basket/src/main/scala/org/finos/vuu/core/module/basket/provider/BasketConstituentProvider.scala +++ b/example/basket/src/main/scala/org/finos/vuu/core/module/basket/provider/BasketConstituentProvider.scala @@ -4,25 +4,26 @@ import org.finos.toolbox.lifecycle.LifecycleContainer import org.finos.toolbox.thread.RunOnceLifeCycleRunner import org.finos.toolbox.time.Clock import org.finos.vuu.core.module.basket.BasketConstants -import org.finos.vuu.core.module.basket.csv.CsvStaticLoader +import org.finos.vuu.core.module.basket.csv.BasketLoader import org.finos.vuu.core.table.{DataTable, RowWithData} import org.finos.vuu.provider.DefaultProvider class BasketConstituentProvider(val table: DataTable)(implicit lifecycle: LifecycleContainer, clock: Clock) extends DefaultProvider { private val runner = new RunOnceLifeCycleRunner("BasketConstituentProvider", runOnce) + private val basketLoader = new BasketLoader() lifecycle(this).dependsOn(runner) import org.finos.vuu.core.module.basket.BasketModule.BasketConstituentColumnNames._ def runOnce(): Unit = { - val baskets = CsvStaticLoader.load + val baskets = basketLoader.loadBasketIds() baskets.foreach(basketId => updateBasketConstituents(basketId)) } def updateBasketConstituents(basketId: String): Unit = { - val list = CsvStaticLoader.loadConstituent(basketId) + val list = basketLoader.loadConstituents(basketId) list.foreach(row => { if (row.nonEmpty) { diff --git a/example/basket/src/main/scala/org/finos/vuu/core/module/basket/provider/BasketProvider.scala b/example/basket/src/main/scala/org/finos/vuu/core/module/basket/provider/BasketProvider.scala index 9f087222d..2708b45ac 100644 --- a/example/basket/src/main/scala/org/finos/vuu/core/module/basket/provider/BasketProvider.scala +++ b/example/basket/src/main/scala/org/finos/vuu/core/module/basket/provider/BasketProvider.scala @@ -4,17 +4,18 @@ import org.finos.toolbox.lifecycle.LifecycleContainer import org.finos.toolbox.thread.RunOnceLifeCycleRunner import org.finos.toolbox.time.Clock import org.finos.vuu.core.module.basket.BasketModule.BasketColumnNames._ -import org.finos.vuu.core.module.basket.csv.CsvStaticLoader +import org.finos.vuu.core.module.basket.csv.BasketLoader import org.finos.vuu.core.table.{DataTable, RowWithData} import org.finos.vuu.provider.DefaultProvider class BasketProvider(val table: DataTable)(implicit lifecycle: LifecycleContainer, clock: Clock) extends DefaultProvider { private val runner = new RunOnceLifeCycleRunner("BasketProvider", runOnce) + private val basketLoader = new BasketLoader() lifecycle(this).dependsOn(runner) def runOnce(): Unit = { - val data = CsvStaticLoader.load + val data = basketLoader.loadBasketIds() data.foreach(id => { table.processUpdate(id, RowWithData(id, Map( @@ -23,7 +24,5 @@ class BasketProvider(val table: DataTable)(implicit lifecycle: LifecycleContaine )), clock.now()) }) } - - override val lifecycleId: String = "org.finos.vuu.core.module.basket.provider.BasketProvider" } diff --git a/example/basket/src/test/scala/org/finos/vuu/csv/CsvStaticLoaderTests.scala b/example/basket/src/test/scala/org/finos/vuu/csv/CsvStaticLoaderTests.scala deleted file mode 100644 index e2d6ed5d0..000000000 --- a/example/basket/src/test/scala/org/finos/vuu/csv/CsvStaticLoaderTests.scala +++ /dev/null @@ -1,42 +0,0 @@ -package org.finos.vuu.csv - -import org.finos.vuu.core.module.basket.csv.CsvStaticLoader -import org.scalatest.featurespec.AnyFeatureSpec - -import scala.io.Source - -class CsvStaticLoaderTests extends AnyFeatureSpec{ - - Feature("CSV loading Test Case") { - - Scenario("Can successfully load and parse basket constituents") { - val testResourcePath = this.getClass.getResource("/constituents").getPath - val constituents = CsvStaticLoader.loadConstituent(".FTSE100", Some(testResourcePath)) - - assert(constituents.length == 3) - val firstRow = constituents.head - assert(firstRow("Symbol") == "AAL.L") - assert(firstRow("Last Trade") == "436.35") - assert(firstRow("Name") == "Anglo American PLC") - assert(firstRow("Weighting") == 0.0278736825813547) - assert(firstRow("Volume") == "5799089") - assert(firstRow("Change") == "5.35") - } - - Scenario("When parsing basket constituents fails return empty") { - - val constituents = CsvStaticLoader.loadConstituent(".FTSEWithError") - - assert(constituents.length == 0) - } - - - Scenario("When no matching basket constituents file return empty") { - - val constituents = CsvStaticLoader.loadConstituent(".NoSuchFile") - - assert(constituents.length == 0) - } - - } -}