#1167 add improvements to SchemaMapper

- a new apply method without a field->column map param, helpful when there are too many fields and a user wants a simple one-one mapping. - add documentation where required for improved DevEx. - also now we use Array DS for SchemaMapper.toInternalRowMap for constant time access.
finos · Mar 19, 2024 · 85375ee · 85375ee
1 parent 6497216
commit 85375ee
Show file tree

Hide file tree

Showing 2 changed files with 158 additions and 41 deletions.
diff --git a/vuu/src/main/scala/org/finos/vuu/util/schema/SchemaMapper.scala b/vuu/src/main/scala/org/finos/vuu/util/schema/SchemaMapper.scala
@@ -2,14 +2,30 @@ package org.finos.vuu.util.schema
 
 import org.finos.vuu.core.table.Column
 
+
+/**
+ * This class provides utility methods related to mapping external fields to internal columns
+ * and vice versa.
+ *
+ * @note For now converter functions i.e. `toInternalRowMap` doesn't perform any type-checks and/or
+ * type conversions. That feature is part of our roadmap and will be introduced in near
+ * future.
+ * */
 trait SchemaMapper {
   def tableColumn(extFieldName: String): Option[Column]
   def externalSchemaField(columnName: String): Option[SchemaField]
-  def toInternalRowMap(values: List[_]): Map[String, Any]
-  def toInternalRowMap(dto: Product): Map[String, Any]
+  def toInternalRowMap(externalValues: List[_]): Map[String, Any]
+  def toInternalRowMap(externalDto: Product): Map[String, Any]
 }
 
 object SchemaMapper {
+  /**
+   * Builds a schema mapper from the following:
+   *
+   * @param externalSchema schema representing external fields.
+   * @param internalColumns an array of internal Vuu columns.
+   * @param columnNameByExternalField a map from external field names to internal column names.
+   * */
   def apply(externalSchema: ExternalEntitySchema,
             internalColumns: Array[Column],
             columnNameByExternalField: Map[String, String]): SchemaMapper = {
@@ -19,6 +35,27 @@ object SchemaMapper {
     new SchemaMapperImpl(externalSchema, internalColumns, columnNameByExternalField)
   }
 
+  /**
+   * Builds a schema mapper from the following:
+   *
+   * @param externalSchema schema representing external fields.
+   * @param internalColumns an array of internal Vuu columns.
+   *
+   * @note Similar to `apply(ExternalEntitySchema, Array[Column], Map[String, String])`
+   * except that this method builds the `field->column` map from the passed fields
+   * and columns matching them by their indexes (`Column.index` and `SchemaField.index`).
+   *
+   * @see [[SchemaMapper.apply]]
+   * */
+  def apply(externalSchema: ExternalEntitySchema, internalColumns: Array[Column]): SchemaMapper = {
+    val columnNameByExternalField = mapFieldsToColumns(externalSchema.fields, internalColumns)
+    SchemaMapper(externalSchema, internalColumns, columnNameByExternalField)
+  }
+
+  private def mapFieldsToColumns(fields: List[SchemaField], columns: Array[Column]): Map[String, String] = {
+    fields.flatMap(f => columns.find(_.index == f.index).map(col => (f.name, col.name))).toMap
+  }
+
   private type ValidationError = Option[String]
   private def validateSchema(externalSchema: ExternalEntitySchema,
                              internalColumns: Array[Column],
@@ -52,21 +89,23 @@ object SchemaMapper {
 }
 
 private class SchemaMapperImpl(private val externalSchema: ExternalEntitySchema,
-                               private val tableColumns: Array[Column],
+                               private val internalColumns: Array[Column],
                                private val columnNameByExternalField: Map[String, String]) extends SchemaMapper {
-  private val externalSchemaFieldsByColumnName: Map[String, SchemaField] = getExternalSchemaFieldsByColumnName
-  private val tableColumnByExternalField: Map[String, Column] = getTableColumnByExternalField
-
-  override def tableColumn(extFieldName: String): Option[Column] = tableColumnByExternalField.get(extFieldName)
-  override def externalSchemaField(columnName: String): Option[SchemaField] = externalSchemaFieldsByColumnName.get(columnName)
-  override def toInternalRowMap(values: List[_]): Map[String, Any] = {
-    tableColumns.map(column => {
-      val f = externalSchemaField(column.name).get
-      val columnValue = values(f.index) // @todo add type conversion conforming to the passed schema
-      (column.name, columnValue)
+  private val externalFieldByColumnName: Map[String, SchemaField] = getExternalSchemaFieldsByColumnName
+  private val internalColumnByExtFieldName: Map[String, Column] = getTableColumnByExternalField
+
+  override def tableColumn(extFieldName: String): Option[Column] = internalColumnByExtFieldName.get(extFieldName)
+  override def externalSchemaField(columnName: String): Option[SchemaField] = externalFieldByColumnName.get(columnName)
+  override def toInternalRowMap(externalValues: List[_]): Map[String, Any] = toInternalRowMap(externalValues.toArray)
+  override def toInternalRowMap(externalDto: Product): Map[String, Any] = toInternalRowMap(externalDto.productIterator.toArray)
+
+  private def toInternalRowMap(externalValues: Array[_]): Map[String, Any] = {
+    externalFieldByColumnName.keys.map(columnName => {
+      val field = externalSchemaField(columnName).get
+      val columnValue = externalValues(field.index) // @todo add type conversion conforming to the passed schema
+      (columnName, columnValue)
     }).toMap
   }
-  override def toInternalRowMap(dto: Product): Map[String, Any] = toInternalRowMap(dto.productIterator.toList)
 
   private def getExternalSchemaFieldsByColumnName =
     externalSchema.fields.flatMap(f =>
@@ -75,6 +114,6 @@ private class SchemaMapperImpl(private val externalSchema: ExternalEntitySchema,
 
   private def getTableColumnByExternalField =
     columnNameByExternalField.flatMap({
-      case (extFieldName, columnName) => tableColumns.find(_.name == columnName).map((extFieldName, _))
+      case (extFieldName, columnName) => internalColumns.find(_.name == columnName).map((extFieldName, _))
     })
 }
diff --git a/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperTest.scala b/vuu/src/test/scala/org/finos/vuu/util/schema/SchemaMapperTest.scala
@@ -3,18 +3,18 @@ package org.finos.vuu.util.schema
 import org.finos.vuu.core.module.vui.VuiStateModule.stringToFieldDef
 import org.finos.vuu.core.table.{Column, Columns, SimpleColumn}
 import org.finos.vuu.util.schema.SchemaMapper.InvalidSchemaMapException
-import org.finos.vuu.util.schema.SchemaMapperTest.{fieldsMap, tableColumns}
+import org.finos.vuu.util.schema.SchemaMapperTest.{externalFields, externalSchema, fieldsMap, fieldsMapWithoutAssetClass, internalColumns}
 import org.scalatest.featurespec.AnyFeatureSpec
 import org.scalatest.matchers.should.Matchers
 
 class SchemaMapperTest extends AnyFeatureSpec with Matchers {
 
-  private val testExternalSchema = new TestEntitySchema
-  private val schemaMapper = SchemaMapper(testExternalSchema, tableColumns, fieldsMap)
-
   Feature("toInternalRowMap") {
-    Scenario("can convert an ordered list of external values to a map conforming to internal schema") {
-      val rowData = schemaMapper.toInternalRowMap(List(3, "ric", "assetClass", 10.5))
+    Scenario("can convert an ordered list of external values") {
+      val mapper = SchemaMapper(externalSchema, internalColumns, fieldsMap)
+
+      val rowData = mapper.toInternalRowMap(List(3, "ric", "assetClass", 10.5))
+
       rowData shouldEqual Map(
         "id" -> 3,
         "ric" -> "ric",
@@ -23,86 +23,164 @@ class SchemaMapperTest extends AnyFeatureSpec with Matchers {
       )
     }
 
-    Scenario("can convert a case class object containing external values to a map conforming to internal schema") {
-      val rowData = schemaMapper.toInternalRowMap(TestDto(3, "ric", "assetClass", 10.5))
+    Scenario("can convert ordered list excluding any values not present in the `field->column` map") {
+      val mapper = SchemaMapper(externalSchema, internalColumns, fieldsMapWithoutAssetClass)
+
+      val rowData = mapper.toInternalRowMap(List(3, "ric", "assetClass", 10.5))
+
+      rowData shouldEqual Map(
+        "id" -> 3,
+        "ric" -> "ric",
+        "price" -> 10.5
+      )
+    }
+
+    Scenario("can convert a case class object containing external values") {
+      val mapper = SchemaMapper(externalSchema, internalColumns, fieldsMap)
+
+      val rowData = mapper.toInternalRowMap(TestDto(3, "ric", "assetClass", 10.5))
+
       rowData shouldEqual Map(
         "id" -> 3,
         "ric" -> "ric",
         "assetClass" -> "assetClass",
         "price" -> 10.5
       )
     }
+
+    Scenario("can convert a case class object excluding any fields not present in `field->column` map") {
+      val mapper = SchemaMapper(externalSchema, internalColumns, fieldsMapWithoutAssetClass)
+
+      val rowData = mapper.toInternalRowMap(TestDto(3, "ric", "assetClass", 10.5))
+
+      rowData shouldEqual Map(
+        "id" -> 3,
+        "ric" -> "ric",
+        "price" -> 10.5
+      )
+    }
   }
 
   Feature("externalSchemaField") {
+    val mapper = SchemaMapper(externalSchema, internalColumns, fieldsMapWithoutAssetClass)
+
     Scenario("can get external schema field from internal column name") {
-      val f = schemaMapper.externalSchemaField("ric")
-      f.get shouldEqual SchemaField("externalRic", classOf[String], 1)
+      val field = mapper.externalSchemaField("ric")
+      field.get shouldEqual SchemaField("externalRic", classOf[String], 1)
+    }
+
+    Scenario("returns None if column not present in the mapped fields") {
+      val field = mapper.externalSchemaField("assetClass")
+      field shouldEqual None
     }
   }
 
   Feature("tableColumn") {
+    val mapper = SchemaMapper(externalSchema, internalColumns, fieldsMapWithoutAssetClass)
+
     Scenario("can get internal column from external field name") {
-      val column = schemaMapper.tableColumn("externalRic")
+      val column = mapper.tableColumn("externalRic")
       column.get shouldEqual SimpleColumn("ric", 1, classOf[String])
     }
+
+    Scenario("returns None if external field not present in the mapped fields") {
+      val column = mapper.tableColumn("assetClass")
+      column shouldEqual None
+    }
   }
 
 
   Feature("validation on instantiation") {
     Scenario("fails when mapped external field not found in external schema") {
       val exception = intercept[InvalidSchemaMapException](
-        SchemaMapper(testExternalSchema, Columns.fromNames("ric".int()), Map("non-existent" -> "ric"))
+        SchemaMapper(externalSchema, Columns.fromNames("ric".int()), Map("non-existent" -> "ric"))
       )
       exception shouldBe a[RuntimeException]
       exception.getMessage should include regex s"[Ff]ield `non-existent` not found"
     }
 
     Scenario("fails when mapped internal field not found in internal columns") {
       val exception = intercept[InvalidSchemaMapException](
-        SchemaMapper(testExternalSchema, Columns.fromNames("id".int()), Map("externalId" -> "absent-col"))
+        SchemaMapper(externalSchema, Columns.fromNames("id".int()), Map("externalId" -> "absent-col"))
       )
       exception shouldBe a[RuntimeException]
       exception.getMessage should include regex "[Cc]olumn `absent-col` not found"
     }
 
-    Scenario("fails when external->internal map contains duplicated internal field") {
+    Scenario("fails when external->internal map contains duplicated internal fields") {
       val exception = intercept[InvalidSchemaMapException](SchemaMapper(
-        testExternalSchema,
+        externalSchema,
         Columns.fromNames("id".int(), "ric".string()),
         Map("externalId" -> "id", "externalRic" -> "id")
       ))
       exception shouldBe a[RuntimeException]
       exception.getMessage should include("duplicated column names")
     }
   }
+
+  Feature("SchemaMapper.apply without user-defined fields map") {
+    Scenario("can generate mapper with exact fields matched by index") {
+      val mapper = SchemaMapper(externalSchema, internalColumns)
+
+      mapper.tableColumn("externalId").get.name shouldEqual "id"
+      mapper.tableColumn("externalRic").get.name shouldEqual "ric"
+      mapper.tableColumn("assetClass").get.name shouldEqual "assetClass"
+      mapper.tableColumn("price").get.name shouldEqual "price"
+    }
+
+    Scenario("can generate mapper when an external field has no matched column") {
+      val mapper = SchemaMapper(externalSchema, internalColumns.slice(0, 3))
+
+      mapper.tableColumn("externalId") shouldBe empty
+      mapper.tableColumn("externalRic").get.name shouldEqual "ric"
+      mapper.tableColumn("assetClass").get.name shouldEqual "assetClass"
+      mapper.tableColumn("price").get.name shouldEqual "price"
+    }
+
+    Scenario("can generate mapper when a column has no matched external field") {
+      val mapper = SchemaMapper(TestEntitySchema(externalFields.slice(0, 3)), internalColumns)
+
+      mapper.tableColumn("externalId").get.name shouldEqual "id"
+      mapper.tableColumn("externalRic") shouldBe empty
+      mapper.tableColumn("assetClass").get.name shouldBe  "assetClass"
+      mapper.tableColumn("price").get.name shouldEqual "price"
+    }
+  }
 }
 
-private class TestEntitySchema extends ExternalEntitySchema {
-  override val fields: List[SchemaField] = List(
+private case class TestEntitySchema(override val fields: List[SchemaField]) extends ExternalEntitySchema
+
+private case class TestDto(externalId: Int, externalRic: String, assetClass: String, price: Double)
+
+private object SchemaMapperTest {
+
+  // no need to be sorted by their index
+  val externalFields: List[SchemaField] = List(
     SchemaField("externalId", classOf[Int], 0),
-    SchemaField("externalRic", classOf[String], 1),
     SchemaField("assetClass", classOf[String], 2),
     SchemaField("price", classOf[Double], 3),
+    SchemaField("externalRic", classOf[String], 1),
   )
-}
-
-private case class TestDto(externalId: Int, externalRic: String, assetClass: String, price: Double)
 
-private object SchemaMapperTest {
+  val externalSchema: TestEntitySchema = TestEntitySchema(externalFields)
 
-  val tableColumns: Array[Column] = Columns.fromNames(
+  val internalColumns: Array[Column] = {
+    val columns = Columns.fromNames(
       "id".int(),
       "ric".string(),
       "assetClass".string(),
       "price".double(),
-  )
+    )
+    // no need to be sorted by their index
+    columns.tail.appended(columns.head)
+  }
 
   val fieldsMap: Map[String, String] = Map(
     "externalId" -> "id",
     "externalRic" -> "ric",
-    "assetClass" -> "assetClass",
-    "price" -> "price"
+    "price" -> "price",
+    "assetClass" -> "assetClass"
   )
 
+  val fieldsMapWithoutAssetClass: Map[String, String] = fieldsMap.slice(0, 3)
 }