Merge pull request #5 from QuanTemplate/task/QUAN-8655-use-yml

Parse yml for the revenueReport command, take Capital IQ ids from the QT

README.md

@@ -1,8 +1,8 @@
-# data-ingress
+# api-integrations
 
-Integrations with third-party data providers such as [Capital IQ](https://www.capitaliq.com) leveraging the [Quantemplate Data Ingress API](https://quantemplate.readme.io/docs/getting-started#-data-ingress).
+Integrations with third-party data providers such as [Capital IQ](https://www.capitaliq.com) leveraging the [Quantemplate API](https://quantemplate.readme.io/docs/getting-started).
 
-If you need help please contact as at [email protected]
+If you need help please contact us at [email protected]
 
 
 # Capital IQ
@@ -26,9 +26,18 @@ The diagram below describes a potential integration pattern where CapitalIQ data
 
 - Generating a total revenue report from the CapitalIQ data and uploading it to the Quantemplate dataset
 
-    ```sh
-    cat ./data/capitaliq-identifiers.txt | java -jar ./capitaliq/target/scala-3.0.0-RC1/capitaliq-assembly-1.0.jar generateRevenueReport --orgId c-my-small-insuranc-ltdzfd --datasetId d-e4tf3yyxerabcvicidv5oyey --currency USD --from 1988-12-31 --to 2018-12-31
-    ```
+    - with yaml config:
+        ```sh
+        java -jar ./capitaliq/target/scala-3.0.0-RC2/capitaliq-assembly-0.1.0.jar apply ./data/revReport.yml
+        ```
+
+        Check out the [config file](./data/revReport.yml)
+
+
+    - with CLI args:
+        ```sh
+        cat ./data/capitaliq-identifiers.txt | java -jar ./capitaliq/target/scala-3.0.0-RC2/capitaliq-assembly-0.1.0.jar generateRevenueReport --orgId c-my-small-insuranc-ltdzfd --datasetId d-e4tf3yyxerabcvicidv5oyey --currency USD --from 1988-12-31 --to 2018-12-31
+        ```
 
 
 
@@ -75,6 +84,7 @@ An example set of identifiers could be found in the `./data/capitaliq-identifier
     ```
     CAPITALIQ_API_USERNAME=<api username you got with Capital IQ API license>
     CAPITALIQ_API_PASSWORD=<password for the corresponding Capital IQ API user>
+    CAPITALIQ_DEMO_ACCOUNT=<indicates whether the Capital IQ demo account is used>
     QT_ENV=<name of the Quantemplate environment, should be `prod` for any consumers>
     QT_AUTH_REALM=<name of the Quantemplate auth ream, should be `qt` for any consumers>
     QT_CLIENT_ID=<id of the api-client user generated by the Quantemplate team>
@@ -85,6 +95,7 @@ An example set of identifiers could be found in the `./data/capitaliq-identifier
     ```
     [email protected]
     CAPITALIQ_API_PASSWORD=<password from dev credentials>
+    CAPITALIQ_DEMO_ACCOUNT=true
     QT_ENV=dev
     QT_AUTH_REALM=test
     QT_CLIENT_ID=<id of the api-client user>

build.sbt

@@ -1,8 +1,8 @@
-name := "data-ingress"
+name := "api-integrations"
 
 ThisBuild / organization := "com.quantemplate"
-ThisBuild / scalaVersion := "3.0.0-RC1"
-ThisBuild / version := "1.0"
+ThisBuild / scalaVersion := "3.0.0-RC2"
+ThisBuild / version := "0.1.0"
 
 val AkkaVersion = "2.6.12"
 val AkkaHttpVersion = "10.2.4"
@@ -25,15 +25,26 @@ lazy val capitaliq = (project in file("capitaliq"))
       // circe
       "io.circe" %% "circe-core" % CirceVersion,
       "io.circe" %% "circe-parser" % CirceVersion,
+      "io.circe" %% "circe-yaml" % "0.13.1",
       // misc
-      "org.scalatest" %% "scalatest" % "3.1.0" % Test,
-      "ch.qos.logback" % "logback-classic" % "1.2.3",
-      "com.typesafe" % "config" % "1.4.1",
       "org.typelevel" %% "cats-core" % "2.4.2",
       "com.norbitltd" %% "spoiwo" % "1.7.0",
-      "com.github.scopt" %% "scopt" % "4.0.1"
-    ).map(_.withDottyCompat(scalaVersion.value)),
-    assembly / mainClass := Some("com.quantemplate.capitaliq.Main")
+      "com.github.scopt" %% "scopt" % "4.0.1",
+    ).map(_.cross(CrossVersion.for3Use2_13)),
+
+    // java deps
+    libraryDependencies ++= Seq(
+      "ch.qos.logback" % "logback-classic" % "1.2.3",
+      "com.typesafe" % "config" % "1.4.1",
+      "org.mockito" % "mockito-core" % "3.9.0" % Test
+    ),
+
+    // native scala 3 deps
+    libraryDependencies ++= Seq(
+      "org.scalameta" %% "munit" % "0.7.23" % Test
+    ),
+
+    assembly / mainClass := Some("com.quantemplate.capitaliq.Main"),
   )
 
 Compile / run / fork := true

capitaliq/src/main/resources/application.conf

@@ -2,9 +2,10 @@ capitaliq {
   endpoint = "https://api-ciq.marketintelligence.spglobal.com/gdsapi/rest/v3/clientservice.json"
 
   # number of mnemonics / identifiers used in a single HTTP request
-  # 500 for prod account
-  # 100 for trial account
-  mnemonicsPerRequest = 100
+  mnemonicsPerRequestInDemoAccount = 100
+  mnemonicsPerRequestInProdAccount = 500
+
+  demoAccount = ${CAPITALIQ_DEMO_ACCOUNT}
 
   credentials {
     username = ${CAPITALIQ_API_USERNAME}

capitaliq/src/main/scala/com/quantemplate/capitaliq/Main.scala

@@ -3,12 +3,14 @@ package com.quantemplate.capitaliq
 import org.slf4j.LoggerFactory
 
 import com.quantemplate.capitaliq.commands.*
+import com.quantemplate.capitaliq.commands.revenuereport.*
 
 object Main:
   lazy val logger = LoggerFactory.getLogger(getClass)
 
   def main(args: Array[String]): Unit = args match
-    case Array("generateRevenueReport", _*) => RevenueReportCmd.run(args)
+    case Array(`configDefInterpreterCmdName`, _*) => ConfigDefInterpreterCmd.fromCli(args)
+    case Array(`revenueReportCmdName`, _*)        => RevenueReportCmd().fromCli(args)
     case _ => 
       logger.error("Unsupported command")
       System.exit(1)

capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDef.scala

@@ -0,0 +1,3 @@
+package com.quantemplate.capitaliq.commands
+
+trait ConfigDef

capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala

@@ -0,0 +1,65 @@
+package com.quantemplate.capitaliq.commands
+
+import java.time.LocalDate
+import java.nio.file.Path
+import io.circe.{ Encoder, Decoder, Json, DecodingFailure }
+import io.circe.yaml.{parser as ymlParser}
+import org.slf4j.LoggerFactory
+import scopt.OParser
+import cats.syntax.bifunctor.given
+
+import com.quantemplate.capitaliq.common.*
+import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier
+import com.quantemplate.capitaliq.domain.Identifiers
+import com.quantemplate.capitaliq.commands.revenuereport.*
+
+object ConfigDefInterpreterCmd:
+  lazy val logger = LoggerFactory.getLogger(getClass)
+
+  def fromCli(args: Array[String]) = ConfigArgsParser.parse(args).map { cliConfig =>
+    val configPath = IO.absolutePath(cliConfig.path)
+
+    loadConfig(configPath).bimap(
+      err => logger.error("Could not parse the config file", err),
+      {
+        case config: RevenueReportConfigDef => RevenueReportCmd().fromConfigFile(config, configPath)
+      }
+    )
+  }
+
+private def loadConfig(path: Path) =
+  IO.readAll(path)
+    .toEither
+    .flatMap(ymlParser.parse(_))
+    .flatMap(_.as[ConfigDef])
+
+given Decoder[ConfigDef] = Decoder.instance[ConfigDef] { c => 
+  c.get[String]("command").flatMap { 
+    case configDefInterpreterCmdName => c.get[RevenueReportConfigDef]("params")
+  }
+}
+
+val configDefInterpreterCmdName = "apply"
+
+object ConfigArgsParser:
+  case class CliConfig(path: String = "")
+
+  def parse(args: Array[String]) = OParser.parse(parser, args, CliConfig())
+
+  private lazy val builder = OParser.builder[CliConfig]
+  private lazy val parser =
+    import builder.*
+
+    OParser.sequence(
+      programName("capitaliq-qt integration"),
+      head("capitaliq-qt", "0.0.1"),
+      cmd(configDefInterpreterCmdName)
+        .text("Runs the command described in a file at a given path")
+        .children(
+          arg[String]("<config file>...")
+            .action((path, c) => c.copy(path = path))
+            .unbounded
+            .required
+        )
+    )
+

capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala

@@ -0,0 +1,14 @@
+package com.quantemplate.capitaliq.commands.revenuereport
+
+import java.time.LocalDate
+
+import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier
+
+case class CmdConfig(
+  orgId: String,
+  datasetId: String,
+  from: LocalDate,
+  to: LocalDate,
+  currency: String,
+  identifiers: Vector[Identifier]
+)

capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReport.scala → capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReport.scala

@@ -1,4 +1,4 @@
-package com.quantemplate.capitaliq.commands
+package com.quantemplate.capitaliq.commands.revenuereport
 
 import java.time.*
 import java.time.format.DateTimeFormatter
@@ -51,7 +51,7 @@ class RevenueReport(capitalIqService: CapitalIQService, qtService: QTService)(us
     range: (LocalDate, LocalDate),
     currency: String
   ): Future[ReportRows] =
-    import range.{ _1 => start, _2 => end }
+    import range.{ _1 as start, _2 as end }
 
     val periodType = "IQ_FY" back (end.getYear - start.getYear)
     val asOfDate = end.format(DateTimeFormatter.ofPattern("MM/dd/yyyy")).some

capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReportArgsParser.scala → capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportArgsParser.scala

@@ -1,29 +1,44 @@
-package com.quantemplate.capitaliq.commands
+package com.quantemplate.capitaliq.commands.revenuereport
 
 import java.util.Calendar
+import java.time.ZoneId
 import scopt.OParser
 
+import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier
+import com.quantemplate.capitaliq.domain.Identifiers
+
 object RevenueReportArgsParser:
-  // OParser requires default args
-  case class Config(
+  case class CliConfig(
     orgId: String = "",
     datasetId: String = "",
     from: Calendar = Calendar.getInstance,
     to: Calendar = Calendar.getInstance,
     currency: String = "",
     identifiers: Option[Vector[String]] = None
-  )
-
-  def parse(args: Array[String]) = OParser.parse(parser, args, Config()) 
+  ):
+    def toCmdConfig(fallbackIds: => Vector[Identifier]) = CmdConfig(
+      orgId = orgId,
+      datasetId = datasetId,
+      from = fromCalendar(from),
+      to = fromCalendar(to),
+      currency = currency,
+      identifiers = identifiers.map(Identifiers(_*)).getOrElse(fallbackIds)
+    ) 
+
+    private def fromCalendar(cal: Calendar) =
+      cal.getTime.toInstant.atZone(ZoneId.systemDefault).toLocalDate
+
+  def parse(args: Array[String]) = OParser.parse(parser, args, CliConfig()) 
 
-  private lazy val builder = OParser.builder[Config]
+  private lazy val builder = OParser.builder[CliConfig]
   private lazy val parser = 
     import builder.*
 
     OParser.sequence(
       programName("capitaliq-qt integration"),
       head("capitaliq-qt", "0.0.1"),
-      cmd("generateRevenueReport")
+      cmd(revenueReportCmdName)
+        .text("Generates a revenue report from CapitalIQ data and uploads it to the Quantemplate dataset")
         .children(
           opt[String]("orgId")
             .action((id, c) => c.copy(orgId = id))