From 2bdaa2485ca9158e3a1f480d8f16c4d95638a188 Mon Sep 17 00:00:00 2001
From: Grzegorz Bielski <gbielski@quantemplate.com>
Date: Thu, 15 Apr 2021 17:44:40 +0200
Subject: [PATCH 1/9] add yml parser for revenue report

---
 build.sbt                                     |  4 +-
 .../com/quantemplate/capitaliq/Main.scala     |  4 +-
 .../capitaliq/commands/ConfigDef.scala        |  3 +
 .../commands/ConfigDefInterpreterCmd.scala    | 55 ++++++++++++++++
 .../capitaliq/commands/RevenueReportCmd.scala | 52 ----------------
 .../commands/revenuereport/Config.scala       | 14 +++++
 .../{ => revenuereport}/RevenueReport.scala   |  4 +-
 .../RevenueReportArgsParser.scala             | 30 ++++++---
 .../revenuereport/RevenueReportCmd.scala      | 62 +++++++++++++++++++
 .../RevenueReportConfigDef.scala              | 58 +++++++++++++++++
 .../commands/revenuereport/cmdName.scala      |  3 +
 .../capitaliq/common/HttpService.scala        | 19 ++++++
 .../quantemplate/capitaliq/common/IO.scala    |  8 +++
 .../capitaliq/domain/CapitalIQ.scala          |  8 +--
 .../quantemplate/capitaliq/qt/QTService.scala | 19 +++++-
 data/revReport.yml                            | 17 +++++
 16 files changed, 292 insertions(+), 68 deletions(-)
 create mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDef.scala
 create mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala
 delete mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReportCmd.scala
 create mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala
 rename capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/{ => revenuereport}/RevenueReport.scala (97%)
 rename capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/{ => revenuereport}/RevenueReportArgsParser.scala (70%)
 create mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala
 create mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala
 create mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/cmdName.scala
 create mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala
 create mode 100644 data/revReport.yml

diff --git a/build.sbt b/build.sbt
index 30fe7d7..89bed78 100644
--- a/build.sbt
+++ b/build.sbt
@@ -25,13 +25,15 @@ lazy val capitaliq = (project in file("capitaliq"))
       // circe
       "io.circe" %% "circe-core" % CirceVersion,
       "io.circe" %% "circe-parser" % CirceVersion,
+      "io.circe" %% "circe-yaml" % "0.13.1",
       // misc
       "org.scalatest" %% "scalatest" % "3.1.0" % Test,
       "ch.qos.logback" % "logback-classic" % "1.2.3",
       "com.typesafe" % "config" % "1.4.1",
       "org.typelevel" %% "cats-core" % "2.4.2",
       "com.norbitltd" %% "spoiwo" % "1.7.0",
-      "com.github.scopt" %% "scopt" % "4.0.1"
+      "com.github.scopt" %% "scopt" % "4.0.1",
+      "com.lihaoyi" %% "os-lib" % "0.7.4"
     ).map(_.withDottyCompat(scalaVersion.value)),
     assembly / mainClass := Some("com.quantemplate.capitaliq.Main")
   )
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/Main.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/Main.scala
index 4a9bbfa..2a1c646 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/Main.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/Main.scala
@@ -3,12 +3,14 @@ package com.quantemplate.capitaliq
 import org.slf4j.LoggerFactory
 
 import com.quantemplate.capitaliq.commands.*
+import com.quantemplate.capitaliq.commands.revenuereport.*
 
 object Main:
   lazy val logger = LoggerFactory.getLogger(getClass)
 
   def main(args: Array[String]): Unit = args match
-    case Array("generateRevenueReport", _*) => RevenueReportCmd.run(args)
+    case Array(`configDefInterpreterCmdName`, _*) => ConfigDefInterpreterCmd.fromCli(args)
+    case Array(`revenueReportCmdName`, _*)        => RevenueReportCmd().fromCli(args)
     case _ => 
       logger.error("Unsupported command")
       System.exit(1)
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDef.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDef.scala
new file mode 100644
index 0000000..6e3b2b8
--- /dev/null
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDef.scala
@@ -0,0 +1,3 @@
+package com.quantemplate.capitaliq.commands
+
+trait ConfigDef
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala
new file mode 100644
index 0000000..5ce3e92
--- /dev/null
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala
@@ -0,0 +1,55 @@
+package com.quantemplate.capitaliq.commands
+
+import java.time.LocalDate
+import io.circe.{ Encoder, Decoder, Json, DecodingFailure }
+import io.circe.yaml.{parser as ymlParser}
+import io.circe.syntax.given
+import cats.syntax.apply.given
+import cats.instances.either.given
+import scopt.OParser
+
+import com.quantemplate.capitaliq.common.*
+import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier
+import com.quantemplate.capitaliq.domain.Identifiers
+import com.quantemplate.capitaliq.commands.revenuereport.*
+
+object ConfigDefInterpreterCmd:
+  def fromCli(args: Array[String]) = ConfigArgsParser.parse(args).map { cliConfig =>
+
+    val ymlPath = getPath(cliConfig.path)
+    val yml = os.read(ymlPath)
+
+    parseYml(yml).map {
+      case config: RevenueReportConfigDef => RevenueReportCmd().fromConfigFile(config, ymlPath)
+    }
+  }
+
+private def parseYml(yml: String) =
+  ymlParser.parse(yml).flatMap(_.as[ConfigDef]).toOption
+
+given Decoder[ConfigDef] = Decoder.instance[ConfigDef] { c => 
+  c.get[String]("command").flatMap { 
+    case configDefInterpreterCmdName => c.get[RevenueReportConfigDef]("params")
+  }
+}
+
+val configDefInterpreterCmdName = "apply"
+
+object ConfigArgsParser:
+  case class CliConfig(path: String = "")
+
+  def parse(args: Array[String]) = OParser.parse(parser, args, CliConfig())
+
+  private lazy val builder = OParser.builder[CliConfig]
+  private lazy val parser =
+    import builder.*
+
+    OParser.sequence(
+      programName("capitaliq-qt integration"),
+      head("capitaliq-qt", "0.0.1"),
+      opt[String](configDefInterpreterCmdName)
+        .action((path, c) => c.copy(path = path))
+        .required
+        .text("Runs the command described in a file at a given path")
+    )
+
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReportCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReportCmd.scala
deleted file mode 100644
index a3616c3..0000000
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReportCmd.scala
+++ /dev/null
@@ -1,52 +0,0 @@
-package com.quantemplate.capitaliq.commands
-
-import java.util.Calendar
-import java.time.ZoneId
-import akka.actor.typed.ActorSystem
-import akka.actor.typed.scaladsl.Behaviors
-import scala.concurrent.ExecutionContext
-import scala.util.{Failure, Success}
-
-import com.quantemplate.capitaliq.common.*
-import com.quantemplate.capitaliq.domain.*
-import com.quantemplate.capitaliq.qt.*
-
-object RevenueReportCmd:
-  def run(args: Array[String]) = 
-    RevenueReportArgsParser.parse(args).map { config => 
-      given Config = Config.load()
-      given sys: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "capitaliq")
-      given ExecutionContext = sys.executionContext
-
-      val logger = sys.log
-      val httpService = HttpService()
-      val revenueReport = RevenueReport(CapitalIQService(httpService), QTService(httpService))
-
-      revenueReport
-        .generateSpreadSheet(
-          ids = config.identifiers
-            .map(Identifiers(_*))
-            .getOrElse {
-              logger.info("Loading the Capital IQ identifiers from the STDIN")
-
-              Identifiers.loadFromStdin()
-            },
-          range = (
-            fromCalendar(config.from),
-            fromCalendar(config.to)
-          ),
-          currency = config.currency,
-          orgId = config.orgId,
-          datasetId = config.datasetId
-        ).onComplete { 
-          case Failure(e) => 
-            logger.error("Failed to generate the revenue report: {}", e.toString)
-            Runtime.getRuntime.halt(1)
-
-          case Success(_) =>
-            Runtime.getRuntime.halt(0)
-        }
-    }
-
-  private def fromCalendar(cal: Calendar) =
-    cal.getTime.toInstant.atZone(ZoneId.systemDefault).toLocalDate
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala
new file mode 100644
index 0000000..0caef0b
--- /dev/null
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala
@@ -0,0 +1,14 @@
+package com.quantemplate.capitaliq.commands.revenuereport
+
+import java.time.LocalDate
+
+import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier
+
+case class CmdConfig(
+  orgId: String,
+  datasetId: String,
+  from: LocalDate,
+  to: LocalDate,
+  currency: String,
+  identifiers: Option[Vector[Identifier]]
+)
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReport.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReport.scala
similarity index 97%
rename from capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReport.scala
rename to capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReport.scala
index 6bd98ec..6a7cbd8 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReport.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReport.scala
@@ -1,4 +1,4 @@
-package com.quantemplate.capitaliq.commands
+package com.quantemplate.capitaliq.commands.revenuereport
 
 import java.time.*
 import java.time.format.DateTimeFormatter
@@ -51,7 +51,7 @@ class RevenueReport(capitalIqService: CapitalIQService, qtService: QTService)(us
     range: (LocalDate, LocalDate),
     currency: String
   ): Future[ReportRows] =
-    import range.{ _1 => start, _2 => end }
+    import range.{ _1 as start, _2 as end }
 
     val periodType = "IQ_FY" back (end.getYear - start.getYear)
     val asOfDate = end.format(DateTimeFormatter.ofPattern("MM/dd/yyyy")).some
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReportArgsParser.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportArgsParser.scala
similarity index 70%
rename from capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReportArgsParser.scala
rename to capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportArgsParser.scala
index 46b3b72..858b7d2 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReportArgsParser.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportArgsParser.scala
@@ -1,29 +1,45 @@
-package com.quantemplate.capitaliq.commands
+package com.quantemplate.capitaliq.commands.revenuereport
 
 import java.util.Calendar
+import java.time.ZoneId
 import scopt.OParser
 
+import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier
+import com.quantemplate.capitaliq.domain.Identifiers
+
 object RevenueReportArgsParser:
   // OParser requires default args
-  case class Config(
+  case class CliConfig(
     orgId: String = "",
     datasetId: String = "",
     from: Calendar = Calendar.getInstance,
     to: Calendar = Calendar.getInstance,
     currency: String = "",
     identifiers: Option[Vector[String]] = None
-  )
-  
-  def parse(args: Array[String]) = OParser.parse(parser, args, Config()) 
+  ):
+    def toCmdConfig = CmdConfig(
+      orgId = orgId,
+      datasetId = datasetId,
+      from = fromCalendar(from),
+      to = fromCalendar(to),
+      currency = currency,
+      identifiers = identifiers.map(Identifiers(_*))
+    ) 
+
+    private def fromCalendar(cal: Calendar) =
+      cal.getTime.toInstant.atZone(ZoneId.systemDefault).toLocalDate
+
+
+  def parse(args: Array[String]) = OParser.parse(parser, args, CliConfig()) 
 
-  private lazy val builder = OParser.builder[Config]
+  private lazy val builder = OParser.builder[CliConfig]
   private lazy val parser = 
     import builder.*
 
     OParser.sequence(
       programName("capitaliq-qt integration"),
       head("capitaliq-qt", "0.0.1"),
-      cmd("generateRevenueReport")
+      cmd(revenueReportCmdName)
         .children(
           opt[String]("orgId")
             .action((id, c) => c.copy(orgId = id))
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala
new file mode 100644
index 0000000..9723b98
--- /dev/null
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala
@@ -0,0 +1,62 @@
+package com.quantemplate.capitaliq.commands.revenuereport
+
+import akka.actor.typed.ActorSystem
+import akka.actor.typed.scaladsl.Behaviors
+import scala.concurrent.ExecutionContext
+import scala.util.{Failure, Success}
+
+import com.quantemplate.capitaliq.common.*
+import com.quantemplate.capitaliq.domain.*
+import com.quantemplate.capitaliq.qt.*
+
+
+class RevenueReportCmd:
+  given Config = Config.load()
+  given sys: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "capitaliq")
+  given ExecutionContext = sys.executionContext
+
+  val logger = sys.log
+  val httpService = HttpService()
+  val qtService = QTService(httpService)
+  val revenueReport = RevenueReport(CapitalIQService(httpService), qtService)
+
+  def fromCli(args: Array[String]) =
+    RevenueReportArgsParser.parse(args).map(_.toCmdConfig).map(run)
+
+  def fromConfigFile(config: RevenueReportConfigDef, configPath: os.Path) =
+    val inlineIds = config.identifiers.flatMap(_.inline)
+    val localIds = config.identifiers.flatMap(_.local).map { rawPath => 
+      Identifiers(os.read.lines(getPath(rawPath, configPath)): _*)
+    }
+
+    // todo: load the ids from QT dataset
+    val remoteIds: Option[Vector[CapitalIQ.Identifier]] = ???
+
+    val identifiers = (inlineIds ++ localIds ++ remoteIds).reduceOption(_ ++ _)
+
+    run(config.toCmdConfig(identifiers))
+
+  private def run(config: CmdConfig) = 
+    revenueReport
+      .generateSpreadSheet(
+        ids = config.identifiers.getOrElse {
+          logger.info("Loading the Capital IQ identifiers from the STDIN")
+
+          Identifiers.loadFromStdin()
+          },
+        range = (
+          config.from,
+          config.to
+        ),
+        currency = config.currency,
+        orgId = config.orgId,
+        datasetId = config.datasetId
+      ).onComplete { 
+        case Failure(e) => 
+          logger.error("Failed to generate the revenue report: {}", e.toString)
+          Runtime.getRuntime.halt(1)
+
+        case Success(_) =>
+          Runtime.getRuntime.halt(0)
+      }
+
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala
new file mode 100644
index 0000000..5595972
--- /dev/null
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala
@@ -0,0 +1,58 @@
+package com.quantemplate.capitaliq.commands.revenuereport
+
+import java.nio.file.Paths
+import java.time.LocalDate
+import io.circe.{ Encoder, Decoder, Json, DecodingFailure }
+import io.circe.yaml.{parser as ymlParser}
+import io.circe.syntax.given
+import cats.syntax.apply.given
+import cats.instances.either.given
+import scopt.OParser
+
+import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier
+import com.quantemplate.capitaliq.commands.ConfigDef
+
+case class RevenueReportConfigDef(
+  orgId: String,
+  datasetId: String,
+  currency: String,
+  from: LocalDate,
+  to: LocalDate,
+  identifiers: Option[RevenueReportConfigDef.Identifiers]
+) extends ConfigDef:
+  def toCmdConfig(loadedIds: Option[Vector[Identifier]]) = CmdConfig(
+    orgId = orgId,
+    datasetId = datasetId,
+    from = from,
+    to = to,
+    currency = currency,
+    identifiers = loadedIds
+  )
+
+object RevenueReportConfigDef:
+  given Decoder[RevenueReportConfigDef] = Decoder.instance[RevenueReportConfigDef] { c => 
+    (
+      c.get[String]("orgId"),
+      c.get[String]("datasetId"),
+      c.get[String]("currency"),
+      c.get[LocalDate]("from"),
+      c.get[LocalDate]("to"),
+      c.get[Option[RevenueReportConfigDef.Identifiers]]("identifiers")
+    ).mapN(RevenueReportConfigDef.apply)
+  }
+
+  case class Identifiers(
+    local: Option[String],
+    dataset: Option[String],
+    inline: Option[Vector[Identifier]]
+  )
+
+  object Identifiers:
+    given Decoder[Identifiers] = Decoder.instance[Identifiers] { c => 
+      (
+        c.get[Option[String]]("local"),
+        c.get[Option[String]]("dataset"),
+        c.get[Option[Vector[Identifier]]]("inline")
+      ).mapN(Identifiers.apply)
+    }
+  
\ No newline at end of file
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/cmdName.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/cmdName.scala
new file mode 100644
index 0000000..53f4b2b
--- /dev/null
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/cmdName.scala
@@ -0,0 +1,3 @@
+package com.quantemplate.capitaliq.commands.revenuereport
+
+val revenueReportCmdName = "generateRevenueReport"
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala
index f91e7a2..2fe542d 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala
@@ -17,6 +17,16 @@ class HttpService(using system: ActorSystem[_]):
 
   given ExecutionContext = system.executionContext
 
+  def get[B: Decoder](
+    endpoint: String,
+    auth: Option[Authorization] 
+  ): Future[B] =
+    for
+      res <- GET(endpoint, auth)
+      body <- getResponseBody(res)
+      result <- Unmarshal(body).to[B]
+    yield result
+
   def post[B: Decoder](
     endpoint: String, 
     req: RequestEntity, 
@@ -63,5 +73,14 @@ class HttpService(using system: ActorSystem[_]):
       )
     )
 
+  private def GET(endpoint: String, auth: Option[Authorization]) = 
+    Http().singleRequest(
+      HttpRequest(
+        method = HttpMethods.GET,
+        uri = endpoint,
+        headers = auth.map(Seq(_)).getOrElse(Seq.empty)
+      )
+    )
+
 object HttpService:
   case class UploadResponse(statusCode: Int, body: String)
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala
new file mode 100644
index 0000000..8072c38
--- /dev/null
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala
@@ -0,0 +1,8 @@
+package com.quantemplate.capitaliq.common
+
+import java.nio.file.Paths
+
+def getPath(rawPath: String, base: os.Path = os.pwd) =
+  if Paths.get(rawPath).isAbsolute
+    then os.Path(rawPath)
+    else os.Path(rawPath, base = os.pwd)
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala
index 09bd288..56102c3 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala
@@ -1,14 +1,14 @@
 package com.quantemplate.capitaliq.domain
 
-import io.circe.{ Encoder, Decoder, Json, DecodingFailure }
-import io.circe.syntax.*
-import cats.syntax.traverse.{*, given}
-import cats.instances.vector.{*, given}
+import io.circe.{ Encoder, Decoder, Json }
+import io.circe.syntax.given
+import cats.syntax.traverse.given
 
 object CapitalIQ:
   opaque type Identifier = String
   object Identifier:
     def apply(s: String): Identifier = s
+    given Decoder[Identifier] = Decoder.decodeString.map(apply(_))
   extension (i: Identifier)
     def unwrap: String = i
 
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala
index dfafab1..73d2213 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala
@@ -15,8 +15,12 @@ class QTService(httpService: HttpService)(using ec: ExecutionContext, conf: Conf
   
   lazy val logger = LoggerFactory.getLogger(getClass)
 
+  def datasetEndpoint(orgId: String, datasetId: String) = 
+    s"${conf.quantemplate.api.baseUrl}/v1/organisations/$orgId/datasets/$datasetId"
+
+
   def uploadDataset(view: View, orgId: String, datasetId: String) =
-    val endpoint = s"${conf.quantemplate.api.baseUrl}/v1/organisations/$orgId/datasets/$datasetId"
+    val endpoint = datasetEndpoint(orgId, datasetId)
 
     for
       tokenRes <- getToken()
@@ -33,6 +37,18 @@ class QTService(httpService: HttpService)(using ec: ExecutionContext, conf: Conf
       case other => 
         throw DatasetUploadError(s"${other.statusCode}: ${other.body}")
 
+  def downloadADataset(orgId: String, datasetId: String) =
+    val endpoint = datasetEndpoint(orgId, datasetId)
+
+    for
+      tokenRes <- getToken()
+      res <- httpService.get[String](
+        endpoint,
+        Authorization(OAuth2BearerToken(tokenRes.accessToken)).some
+      )
+    yield ()
+
+
   def getToken(): Future[TokenResponse] = 
     httpService.post[TokenResponse](
       conf.quantemplate.auth.endpoint,
@@ -43,6 +59,7 @@ class QTService(httpService: HttpService)(using ec: ExecutionContext, conf: Conf
       ).toEntity,
       None
     )
+    
 
 object QTService:
   case class TokenResponse(accessToken: String)
diff --git a/data/revReport.yml b/data/revReport.yml
new file mode 100644
index 0000000..2493532
--- /dev/null
+++ b/data/revReport.yml
@@ -0,0 +1,17 @@
+command: generateRevenueReport
+params:
+  orgId: c-my-small-insuranc-ltdzfd
+  datasetId: d-e4tf3yyxerabcvicidv5oyey
+  currency: USD
+  from: 1988-12-31
+  to: 2018-12-31
+
+  # should handle loading identifiers from local file system, QT dataset, inline ones, STDIN
+  identifiers:
+    local: ../capitaliq-identifiers.txt
+    dataset: d-e4tf3yyxerabcvicidv5oyey
+    inline:
+      - IQ121238
+      - IQ598807
+      - IQ723344
+      - IQ956057

From 4f783de94ca151ee75cabedff0b31f55b5bb9070 Mon Sep 17 00:00:00 2001
From: Grzegorz Bielski <gbielski@quantemplate.com>
Date: Fri, 16 Apr 2021 16:13:07 +0200
Subject: [PATCH 2/9] load local, remote and inline identifiers

---
 .../commands/ConfigDefInterpreterCmd.scala    |  36 +-
 .../commands/revenuereport/Config.scala       |   2 +-
 .../RevenueReportArgsParser.scala             |   7 +-
 .../revenuereport/RevenueReportCmd.scala      |  63 +-
 .../RevenueReportConfigDef.scala              |   7 +-
 .../capitaliq/common/HttpService.scala        |  19 +-
 .../quantemplate/capitaliq/common/IO.scala    |   2 +-
 .../capitaliq/domain/CapitalIQ.scala          |  12 +-
 .../capitaliq/domain/Errors.scala             |   2 +-
 .../capitaliq/domain/Identifiers.scala        |  24 +-
 .../quantemplate/capitaliq/qt/QTService.scala |  14 +-
 data/capitaliq-identifiers.csv                | 850 ++++++++++++++++++
 data/revReport.yml                            |   7 +-
 13 files changed, 974 insertions(+), 71 deletions(-)
 create mode 100644 data/capitaliq-identifiers.csv

diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala
index 5ce3e92..dc6ba3f 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala
@@ -3,10 +3,9 @@ package com.quantemplate.capitaliq.commands
 import java.time.LocalDate
 import io.circe.{ Encoder, Decoder, Json, DecodingFailure }
 import io.circe.yaml.{parser as ymlParser}
-import io.circe.syntax.given
-import cats.syntax.apply.given
-import cats.instances.either.given
+import org.slf4j.LoggerFactory
 import scopt.OParser
+import cats.syntax.bifunctor.given
 
 import com.quantemplate.capitaliq.common.*
 import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier
@@ -14,18 +13,23 @@ import com.quantemplate.capitaliq.domain.Identifiers
 import com.quantemplate.capitaliq.commands.revenuereport.*
 
 object ConfigDefInterpreterCmd:
-  def fromCli(args: Array[String]) = ConfigArgsParser.parse(args).map { cliConfig =>
+  lazy val logger = LoggerFactory.getLogger(getClass)
 
-    val ymlPath = getPath(cliConfig.path)
-    val yml = os.read(ymlPath)
+  def fromCli(args: Array[String]) = ConfigArgsParser.parse(args).map { cliConfig =>
+    val configPath = getPath(cliConfig.path)
 
-    parseYml(yml).map {
-      case config: RevenueReportConfigDef => RevenueReportCmd().fromConfigFile(config, ymlPath)
-    }
+    loadConfig(configPath).bimap(
+      err => logger.error("Could not parse the config file", err),
+      {
+        case config: RevenueReportConfigDef => RevenueReportCmd().fromConfigFile(config, configPath)
+      }
+    )
   }
 
-private def parseYml(yml: String) =
-  ymlParser.parse(yml).flatMap(_.as[ConfigDef]).toOption
+private def loadConfig(path: os.Path) =
+  val yml = os.read(path)
+
+  ymlParser.parse(yml).flatMap(_.as[ConfigDef])
 
 given Decoder[ConfigDef] = Decoder.instance[ConfigDef] { c => 
   c.get[String]("command").flatMap { 
@@ -47,9 +51,13 @@ object ConfigArgsParser:
     OParser.sequence(
       programName("capitaliq-qt integration"),
       head("capitaliq-qt", "0.0.1"),
-      opt[String](configDefInterpreterCmdName)
-        .action((path, c) => c.copy(path = path))
-        .required
+      cmd(configDefInterpreterCmdName)
         .text("Runs the command described in a file at a given path")
+        .children(
+          arg[String]("<config file>...")
+            .action((path, c) => c.copy(path = path))
+            .unbounded
+            .required
+        )
     )
 
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala
index 0caef0b..36fd06c 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala
@@ -10,5 +10,5 @@ case class CmdConfig(
   from: LocalDate,
   to: LocalDate,
   currency: String,
-  identifiers: Option[Vector[Identifier]]
+  identifiers: Vector[Identifier]
 )
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportArgsParser.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportArgsParser.scala
index 858b7d2..609a1a4 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportArgsParser.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportArgsParser.scala
@@ -8,7 +8,6 @@ import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier
 import com.quantemplate.capitaliq.domain.Identifiers
 
 object RevenueReportArgsParser:
-  // OParser requires default args
   case class CliConfig(
     orgId: String = "",
     datasetId: String = "",
@@ -17,19 +16,18 @@ object RevenueReportArgsParser:
     currency: String = "",
     identifiers: Option[Vector[String]] = None
   ):
-    def toCmdConfig = CmdConfig(
+    def toCmdConfig(fallbackIds: => Vector[Identifier]) = CmdConfig(
       orgId = orgId,
       datasetId = datasetId,
       from = fromCalendar(from),
       to = fromCalendar(to),
       currency = currency,
-      identifiers = identifiers.map(Identifiers(_*))
+      identifiers = identifiers.map(Identifiers(_*)).getOrElse(fallbackIds)
     ) 
 
     private def fromCalendar(cal: Calendar) =
       cal.getTime.toInstant.atZone(ZoneId.systemDefault).toLocalDate
 
-
   def parse(args: Array[String]) = OParser.parse(parser, args, CliConfig()) 
 
   private lazy val builder = OParser.builder[CliConfig]
@@ -40,6 +38,7 @@ object RevenueReportArgsParser:
       programName("capitaliq-qt integration"),
       head("capitaliq-qt", "0.0.1"),
       cmd(revenueReportCmdName)
+        .text("Generates a revenue report from CapitalIQ data and uploads it to the Quantemplate dataset")
         .children(
           opt[String]("orgId")
             .action((id, c) => c.copy(orgId = id))
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala
index 9723b98..7a05f43 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala
@@ -4,6 +4,9 @@ import akka.actor.typed.ActorSystem
 import akka.actor.typed.scaladsl.Behaviors
 import scala.concurrent.ExecutionContext
 import scala.util.{Failure, Success}
+import scala.concurrent.Future
+import org.slf4j.LoggerFactory
+import cats.syntax.traverse.given
 
 import com.quantemplate.capitaliq.common.*
 import com.quantemplate.capitaliq.domain.*
@@ -15,35 +18,61 @@ class RevenueReportCmd:
   given sys: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "capitaliq")
   given ExecutionContext = sys.executionContext
 
-  val logger = sys.log
+  lazy val logger = LoggerFactory.getLogger(getClass)
+
   val httpService = HttpService()
   val qtService = QTService(httpService)
   val revenueReport = RevenueReport(CapitalIQService(httpService), qtService)
 
   def fromCli(args: Array[String]) =
-    RevenueReportArgsParser.parse(args).map(_.toCmdConfig).map(run)
+    RevenueReportArgsParser.parse(args)
+      .map(_.toCmdConfig(loadIdentifiersFromStdin()))
+      .map(run)
+
+  def fromConfigFile(config: RevenueReportConfigDef, configPath: os.Path) = 
+    loadIdentifiersFromConfig(config, configPath)
+      .map(_.getOrElse(loadIdentifiersFromStdin()))
+      .map(config.toCmdConfig(_))
+      .map(run)
 
-  def fromConfigFile(config: RevenueReportConfigDef, configPath: os.Path) =
-    val inlineIds = config.identifiers.flatMap(_.inline)
-    val localIds = config.identifiers.flatMap(_.local).map { rawPath => 
-      Identifiers(os.read.lines(getPath(rawPath, configPath)): _*)
-    }
+  private def loadIdentifiersFromStdin() = {
+    logger.info("Loading the Capital IQ identifiers from the STDIN")
 
-    // todo: load the ids from QT dataset
-    val remoteIds: Option[Vector[CapitalIQ.Identifier]] = ???
+    Identifiers.loadFromStdin()
+  }
 
-    val identifiers = (inlineIds ++ localIds ++ remoteIds).reduceOption(_ ++ _)
+  private def loadIdentifiersFromConfig(config: RevenueReportConfigDef, configPath: os.Path) =
+    config.identifiers
+      .flatMap(_.dataset)
+      .map(loadIdentifiersFromDataset(config.orgId, _))
+      .sequence
+      .map { remoteIds => 
+        val inlineIds = config.identifiers.flatMap(_.inline)
+        val localIds = config.identifiers.flatMap(_.local).map { rawPath =>
+          Identifiers(
+            os.read.lines(getPath(rawPath, configPath / os.up)): _*
+          )
+        }
+
+        (inlineIds ++ localIds ++ remoteIds).reduceOption(_ ++ _)
+      }
 
-    run(config.toCmdConfig(identifiers))
+  private def loadIdentifiersFromDataset(orgId: String, datasetId: String) =
+    qtService.downloadDataset(orgId, datasetId)
+      .map(Identifiers.loadFromCsvString)
+      .recover { 
+        case e: Throwable => 
+          logger.warn("Could not load the identifiers from the remote dataset")
+          println(("err", e))
+          e.printStackTrace()
 
-  private def run(config: CmdConfig) = 
+          Vector.empty[CapitalIQ.Identifier]
+      }
+
+  private def run(config: CmdConfig) =
     revenueReport
       .generateSpreadSheet(
-        ids = config.identifiers.getOrElse {
-          logger.info("Loading the Capital IQ identifiers from the STDIN")
-
-          Identifiers.loadFromStdin()
-          },
+        ids = config.identifiers.distinct,
         range = (
           config.from,
           config.to
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala
index 5595972..03f11f4 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala
@@ -1,13 +1,10 @@
 package com.quantemplate.capitaliq.commands.revenuereport
 
-import java.nio.file.Paths
 import java.time.LocalDate
-import io.circe.{ Encoder, Decoder, Json, DecodingFailure }
+import io.circe.Decoder
 import io.circe.yaml.{parser as ymlParser}
 import io.circe.syntax.given
 import cats.syntax.apply.given
-import cats.instances.either.given
-import scopt.OParser
 
 import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier
 import com.quantemplate.capitaliq.commands.ConfigDef
@@ -20,7 +17,7 @@ case class RevenueReportConfigDef(
   to: LocalDate,
   identifiers: Option[RevenueReportConfigDef.Identifiers]
 ) extends ConfigDef:
-  def toCmdConfig(loadedIds: Option[Vector[Identifier]]) = CmdConfig(
+  def toCmdConfig(loadedIds: => Vector[Identifier]) = CmdConfig(
     orgId = orgId,
     datasetId = datasetId,
     from = from,
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala
index 2fe542d..70fb808 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala
@@ -12,20 +12,21 @@ import akka.util.ByteString
 import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.*
 import io.circe.{Encoder, Decoder}
 
+// TODO: improve error handling by wrapping responses in Either / Try
 class HttpService(using system: ActorSystem[_]):
   import HttpService.*
 
   given ExecutionContext = system.executionContext
 
-  def get[B: Decoder](
+  def get(
     endpoint: String,
     auth: Option[Authorization] 
-  ): Future[B] =
+  ): Future[String] =
     for
       res <- GET(endpoint, auth)
       body <- getResponseBody(res)
-      result <- Unmarshal(body).to[B]
-    yield result
+      // result <- Unmarshal(body).to[B]
+    yield body.utf8String
 
   def post[B: Decoder](
     endpoint: String, 
@@ -50,15 +51,15 @@ class HttpService(using system: ActorSystem[_]):
       result <- Unmarshal(body).to[B]
     yield result
 
-  def upload(
+  def post(
     endpoint: String, 
     bytes: Array[Byte], 
-    auth: Option[Authorization] = None
-  ): Future[UploadResponse] = 
+    auth: Option[Authorization]
+  ): Future[Response] = 
     for
       res <- POST(endpoint, HttpEntity(bytes), auth)
       body <- getResponseBody(res)
-    yield HttpService.UploadResponse(res.status.intValue, body.utf8String)
+    yield HttpService.Response(res.status.intValue, body.utf8String)
 
   private def getResponseBody(res: HttpResponse) = 
     res.entity.dataBytes.runFold(ByteString.empty)(_ ++ _)
@@ -83,4 +84,4 @@ class HttpService(using system: ActorSystem[_]):
     )
 
 object HttpService:
-  case class UploadResponse(statusCode: Int, body: String)
+  case class Response(statusCode: Int, body: String)
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala
index 8072c38..ab47388 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala
@@ -5,4 +5,4 @@ import java.nio.file.Paths
 def getPath(rawPath: String, base: os.Path = os.pwd) =
   if Paths.get(rawPath).isAbsolute
     then os.Path(rawPath)
-    else os.Path(rawPath, base = os.pwd)
+    else os.Path(rawPath, base = base)
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala
index 56102c3..73faa5a 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala
@@ -3,12 +3,22 @@ package com.quantemplate.capitaliq.domain
 import io.circe.{ Encoder, Decoder, Json }
 import io.circe.syntax.given
 import cats.syntax.traverse.given
+import cats.syntax.applicative.given
 
 object CapitalIQ:
   opaque type Identifier = String
   object Identifier:
+    val prefix = "IQ"
+
     def apply(s: String): Identifier = s
-    given Decoder[Identifier] = Decoder.decodeString.map(apply(_))
+
+    // this might use some improved validation
+    def isValid(s: String) = s.startsWith(prefix)
+
+    given Decoder[Identifier] = Decoder.decodeString
+      .ensure(isValid, "Not a valid CapitalIQ identifier")
+      .map(Identifier(_))
+
   extension (i: Identifier)
     def unwrap: String = i
 
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Errors.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Errors.scala
index e1eabca..c94b4b6 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Errors.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Errors.scala
@@ -12,7 +12,7 @@ case class UnrecognizedServiceError(msg: String) extends DomainError:
   override def toString = s"Unrecognized Capital IQ API error: $msg"
 
 case class MnemonicsError(errors: Seq[DomainError]) extends DomainError:
-  override def toString = errors.mkString
+  override def toString = errors.mkString(",")
 
 case object DailyMnemonicLimitReachedError extends DomainError:
   override def toString = s"Reached daily limit of 24000 requested mnemonics"
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala
index 7f89195..2901348 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala
@@ -2,14 +2,24 @@ package com.quantemplate.capitaliq.domain
 
 import scala.io.Source
 
-object Identifiers:
-  val commentToken = "//"
+import CapitalIQ.Identifier
 
+object Identifiers:
   def apply(ids: String*) = 
-    ids.map(CapitalIQ.Identifier(_)).toVector
+    ids
+      .filter(Identifier.isValid)
+      .map(Identifier(_))
+      .toVector
+
+  // (!) assuming the dataset uses ',' as a separator
+  //  and the identifiers are in the first column
+  def loadFromCsvString(str: String) =
+    Identifiers(
+      str
+        .split('\n')
+        .toVector
+        .map(_.split(',').headOption.getOrElse("")): _*
+    )
     
   def loadFromStdin() = 
-    Source.stdin.getLines
-      .filter(!_.startsWith(commentToken))
-      .map(CapitalIQ.Identifier(_))
-      .toVector
+    Identifiers(Source.stdin.getLines.toVector: _*)
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala
index 73d2213..d0f7ddc 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala
@@ -15,7 +15,7 @@ class QTService(httpService: HttpService)(using ec: ExecutionContext, conf: Conf
   
   lazy val logger = LoggerFactory.getLogger(getClass)
 
-  def datasetEndpoint(orgId: String, datasetId: String) = 
+  private def datasetEndpoint(orgId: String, datasetId: String) = 
     s"${conf.quantemplate.api.baseUrl}/v1/organisations/$orgId/datasets/$datasetId"
 
 
@@ -24,29 +24,29 @@ class QTService(httpService: HttpService)(using ec: ExecutionContext, conf: Conf
 
     for
       tokenRes <- getToken()
-      res <- httpService.upload(
+      res <- httpService.post(
         endpoint, 
         view.toBytes, 
         Authorization(OAuth2BearerToken(tokenRes.accessToken)).some
       )
 
     yield res match 
-      case HttpService.UploadResponse(200, _) => ()
-      case HttpService.UploadResponse(403, _) => 
+      case HttpService.Response(200, _) => ()
+      case HttpService.Response(403, _) => 
         throw DatasetUploadError("403: Did you share the dataset with whole org?")
       case other => 
         throw DatasetUploadError(s"${other.statusCode}: ${other.body}")
 
-  def downloadADataset(orgId: String, datasetId: String) =
+  def downloadDataset(orgId: String, datasetId: String) =
     val endpoint = datasetEndpoint(orgId, datasetId)
 
     for
       tokenRes <- getToken()
-      res <- httpService.get[String](
+      res <- httpService.get(
         endpoint,
         Authorization(OAuth2BearerToken(tokenRes.accessToken)).some
       )
-    yield ()
+    yield res
 
 
   def getToken(): Future[TokenResponse] = 
diff --git a/data/capitaliq-identifiers.csv b/data/capitaliq-identifiers.csv
new file mode 100644
index 0000000..6d5b48c
--- /dev/null
+++ b/data/capitaliq-identifiers.csv
@@ -0,0 +1,850 @@
+identifiers,
+IQ121238,
+IQ598807,
+IQ723344,
+IQ956057,
+IQ250388,
+IQ5471113,
+IQ876215,
+IQ162456,
+IQ8201008,
+IQ22517019,
+IQ22663157,
+IQ875826,
+IQ992287,
+IQ8248315,
+IQ20551569,
+IQ874488,
+IQ341546,
+IQ876731,
+IQ8210142,
+IQ1082017,
+IQ698550,
+IQ876955,
+IQ5467549,
+IQ8475394,
+IQ882901,
+IQ7605761,
+IQ875328,
+IQ36739404,
+IQ878160,
+IQ2969167,
+IQ2159401,
+IQ877802,
+IQ3680446,
+IQ5477675,
+IQ303566889,
+IQ8125089,
+IQ882957,
+IQ4512131,
+IQ883670,
+IQ295539,
+IQ22510313,
+IQ4026111,
+IQ22729379,
+IQ26471214,
+IQ8475467,
+IQ37612389,
+IQ4481454,
+IQ8474885,
+IQ215098026,
+IQ2482281,
+IQ5491822,
+IQ27515847,
+IQ5478545,
+IQ883047,
+IQ12732661,
+IQ876952,
+IQ3561426,
+IQ874863,
+IQ240824094,
+IQ1051073,
+IQ199668,
+IQ20742105,
+IQ5470931,
+IQ874600,
+IQ41356562,
+IQ22533749,
+IQ875917,
+IQ250307,
+IQ4497069,
+IQ9235665,
+IQ966841,
+IQ883901,
+IQ20970264,
+IQ965915,
+IQ8221280,
+IQ5495596,
+IQ8105207,
+IQ1542889,
+IQ882762,
+IQ1882861,
+IQ13335251,
+IQ13483916,
+IQ12933390,
+IQ22039549,
+IQ8952000,
+IQ8237245,
+IQ8642193,
+IQ26972175,
+IQ259879432,
+IQ8292712,
+IQ225687232,
+IQ877323,
+IQ5476832,
+IQ22547601,
+IQ7884005,
+IQ26326530,
+IQ22134956,
+IQ5525514,
+IQ5566600,
+IQ10245092,
+IQ318898,
+IQ22548905,
+IQ877527,
+IQ307183867,
+IQ878130,
+IQ5569241,
+IQ12205230,
+IQ20527977,
+IQ5473193,
+IQ8982543,
+IQ26934166,
+IQ5604059,
+IQ5686144,
+IQ30921662,
+IQ10639513,
+IQ875073,
+IQ53474567,
+IQ32671961,
+IQ873776,
+IQ5533523,
+IQ615970073,
+IQ9514473,
+IQ59372417,
+IQ32777643,
+IQ261155221,
+IQ11519044,
+IQ26841815,
+IQ5569256,
+IQ11461103,
+IQ20394473,
+IQ20457561,
+IQ138540308,
+IQ20357747,
+IQ7683448,
+IQ310010418,
+IQ245593332,
+IQ9450216,
+IQ38010476,
+IQ8751533,
+IQ224448138,
+IQ20929816,
+IQ28526399,
+IQ21790041,
+IQ879281,
+IQ22709629,
+IQ233447435,
+IQ9003966,
+IQ22830143,
+IQ53474451,
+IQ20644228,
+IQ6462568,
+IQ128394122,
+IQ5554951,
+IQ5525485,
+IQ10580089,
+IQ20406261,
+IQ20334327,
+IQ32879143,
+IQ59290204,
+IQ35622138,
+IQ22530726,
+IQ2395875,
+IQ9285238,
+IQ20968703,
+IQ20411372,
+IQ36066453,
+IQ20405257,
+IQ41542793,
+IQ6499589,
+IQ24351707,
+IQ254768850,
+IQ35045462,
+IQ876389,
+IQ252124,
+IQ699678,
+IQ20376386,
+IQ30777099,
+IQ36070234,
+IQ20442970,
+IQ5705453,
+IQ20357911,
+IQ8951913,
+IQ20875991,
+IQ34360884,
+IQ45364909,
+IQ20439959,
+IQ419608830,
+IQ20388152,
+IQ142687475,
+IQ25603857,
+IQ20401153,
+IQ44731152,
+IQ9638335,
+IQ880319,
+IQ37530273,
+IQ8237517,
+IQ45364901,
+IQ9104794,
+IQ118758025,
+IQ23136532,
+IQ20385336,
+IQ47019555,
+IQ280102994,
+IQ6182739,
+IQ24389424,
+IQ98911479,
+IQ118529610,
+IQ34360875,
+IQ6096740,
+IQ206594311,
+IQ20351433,
+IQ20387499,
+IQ20357133,
+IQ25853564,
+IQ884289,
+IQ26840491,
+IQ9446689,
+IQ52657639,
+IQ22817884,
+IQ105309130,
+IQ36824712,
+IQ20342548,
+IQ41542810,
+IQ8863946,
+IQ26425108,
+IQ9317591,
+IQ9648039,
+IQ5435672,
+IQ9898706,
+IQ4509080,
+IQ20392649,
+IQ25601530,
+IQ11251263,
+IQ13338102,
+IQ20333219,
+IQ23956851,
+IQ9383541,
+IQ20928564,
+IQ33059622,
+IQ10837576,
+IQ39420520,
+IQ49756433,
+IQ20386271,
+IQ20387422,
+IQ22519534,
+IQ13389347,
+IQ9616853,
+IQ582343023,
+IQ47286427,
+IQ212327509,
+IQ30585041,
+IQ11354999,
+IQ26539734,
+IQ129791710,
+IQ54412624,
+IQ98910795,
+IQ11459490,
+IQ30746047,
+IQ4493353,
+IQ36070207,
+IQ36218630,
+IQ20356961,
+IQ233903518,
+IQ142039532,
+IQ47938242,
+IQ20388408,
+IQ105289664,
+IQ30612714,
+IQ13483021,
+IQ5449547,
+IQ5514776,
+IQ53474509,
+IQ34430293,
+IQ109965608,
+IQ30468440,
+IQ9648367,
+IQ59676977,
+IQ28506082,
+IQ29785842,
+IQ9648084,
+IQ38928712,
+IQ49943055,
+IQ20385355,
+IQ51446644,
+IQ47285586,
+IQ250801799,
+IQ20406066,
+IQ30746764,
+IQ20440724,
+IQ20406489,
+IQ46949286,
+IQ49273432,
+IQ9648044,
+IQ208360255,
+IQ36308773,
+IQ20376809,
+IQ20384809,
+IQ10167110,
+IQ47733696,
+IQ20379938,
+IQ34360936,
+IQ9265738,
+IQ20387321,
+IQ9264708,
+IQ20405282,
+IQ30446153,
+IQ7958370,
+IQ30589185,
+IQ12004130,
+IQ107792708,
+IQ20387682,
+IQ30864618,
+IQ20384756,
+IQ142681075,
+IQ9663375,
+IQ29226678,
+IQ104919719,
+IQ20357359,
+IQ20406865,
+IQ879972,
+IQ117920629,
+IQ83672799,
+IQ3448881,
+IQ9648086,
+IQ11311924,
+IQ23019707,
+IQ217249500,
+IQ20399470,
+IQ225000580,
+IQ20333227,
+IQ236534039,
+IQ28093936,
+IQ321937608,
+IQ30445681,
+IQ13199091,
+IQ552051031,
+IQ22249494,
+IQ9648296,
+IQ21927946,
+IQ317398442,
+IQ22453364,
+IQ11900969,
+IQ83586150,
+IQ20405278,
+IQ34579314,
+IQ138470937,
+IQ4462958,
+IQ112208069,
+IQ46950019,
+IQ20823232,
+IQ20376805,
+IQ30607633,
+IQ20356260,
+IQ27142992,
+IQ22929324,
+IQ46910621,
+IQ416203171,
+IQ12913444,
+IQ20407950,
+IQ42114936,
+IQ29383141,
+IQ106886296,
+IQ114576425,
+IQ84209473,
+IQ31093,
+IQ169008,
+IQ250309,
+IQ310818,
+IQ632483,
+IQ678651,
+IQ722875,
+IQ846304,
+IQ872445,
+IQ874020,
+IQ875292,
+IQ879079,
+IQ880335,
+IQ881280,
+IQ881579,
+IQ918690,
+IQ973824,
+IQ1028109,
+IQ1056331,
+IQ1511109,
+IQ1515993,
+IQ1529888,
+IQ1549703,
+IQ2745726,
+IQ2759623,
+IQ2897241,
+IQ2994821,
+IQ3649650,
+IQ4090862,
+IQ4096555,
+IQ4163031,
+IQ4165794,
+IQ4168338,
+IQ4183677,
+IQ4205936,
+IQ4210427,
+IQ4217488,
+IQ4222324,
+IQ4224615,
+IQ4249211,
+IQ4251241,
+IQ4279638,
+IQ4303310,
+IQ4332750,
+//IQ4347511 invalid,
+IQ4460048,
+IQ4472553,
+IQ4481550,
+IQ4494248,
+IQ4497162,
+IQ4559043,
+IQ4561438,
+IQ4586814,
+IQ4662943,
+IQ4717183,
+IQ4730735,
+IQ4762761,
+IQ4772278,
+IQ5068910,
+IQ5375495,
+IQ5383311,
+IQ5410293,
+IQ5423523,
+IQ5466287,
+IQ5477916,
+IQ5484075,
+IQ5491169,
+IQ5496125,
+IQ5507600,
+IQ5510525,
+IQ5525297,
+IQ5538187,
+IQ5538747,
+IQ5542158,
+IQ5547143,
+IQ5547613,
+IQ5554373,
+IQ5569286,
+IQ5579426,
+IQ5647001,
+IQ5704985,
+IQ5718690,
+IQ5723278,
+IQ5726675,
+IQ5774517,
+IQ5885638,
+IQ5901538,
+IQ5924809,
+IQ6139502,
+IQ6439926,
+IQ6463183,
+IQ6469667,
+IQ6475664,
+IQ6491233,
+IQ6564312,
+IQ6565183,
+IQ6613221,
+IQ6688088,
+IQ6699370,
+IQ6700017,
+IQ6812156,
+IQ6819792,
+IQ6921281,
+IQ6948694,
+IQ6949267,
+IQ7603981,
+IQ7918016,
+IQ7960416,
+IQ8024164,
+IQ8100801,
+IQ8101330,
+IQ8142882,
+IQ8190935,
+IQ8264274,
+IQ8359879,
+IQ9377948,
+IQ9450267,
+IQ9460690,
+IQ9461042,
+IQ9550032,
+IQ9560354,
+IQ9587298,
+IQ9638878,
+IQ9794004,
+IQ9899048,
+IQ9916947,
+IQ10052670,
+IQ11212063,
+IQ11323005,
+IQ11712739,
+IQ12058656,
+IQ12455886,
+IQ12456448,
+IQ12466795,
+IQ12784598,
+IQ12785981,
+IQ12868386,
+IQ12913479,
+IQ12932470,
+IQ13011212,
+IQ13154195,
+IQ13408154,
+IQ13610052,
+IQ20417852,
+IQ20475401,
+IQ20551258,
+IQ20670757,
+IQ20703772,
+IQ20724600,
+IQ20757724,
+IQ20784162,
+IQ20882165,
+IQ20986395,
+IQ21813186,
+IQ21854837,
+IQ21935579,
+IQ22095148,
+IQ22239048,
+IQ22383525,
+IQ22383606,
+IQ22431794,
+IQ22498272,
+IQ22511361,
+IQ22515611,
+IQ22515844,
+IQ22515964,
+IQ22516125,
+IQ22517048,
+IQ22534319,
+IQ22534512,
+IQ22547744,
+IQ22580222,
+IQ22581143,
+IQ22581472,
+IQ22631649,
+IQ22638720,
+IQ22728611,
+IQ22783832,
+IQ22798804,
+IQ22799264,
+IQ22905959,
+IQ22926403,
+IQ22931810,
+IQ23018346,
+IQ23021162,
+IQ23021452,
+IQ23035756,
+IQ23048633,
+IQ23063990,
+IQ23076171,
+IQ23185132,
+IQ23221640,
+IQ23294481,
+IQ23313835,
+IQ23335868,
+IQ23336901,
+IQ23631262,
+IQ23751882,
+IQ23756872,
+IQ23780053,
+IQ23958224,
+IQ24393771,
+IQ24861410,
+IQ25473977,
+IQ25525075,
+IQ25746444,
+IQ25813840,
+IQ25833297,
+IQ25835239,
+IQ25870800,
+IQ25878705,
+IQ25912594,
+IQ25913389,
+IQ25913460,
+IQ25915583,
+IQ25935856,
+IQ26023649,
+IQ26079324,
+IQ26307837,
+IQ26321017,
+IQ26442394,
+IQ26502798,
+IQ26554716,
+IQ26671175,
+IQ26763923,
+IQ26935576,
+IQ26935957,
+IQ26937924,
+IQ26954558,
+IQ26954965,
+IQ26955671,
+IQ26956943,
+IQ26962223,
+IQ26971203,
+IQ26971884,
+IQ26971963,
+IQ26971979,
+IQ26972138,
+IQ26972384,
+IQ26972531,
+IQ26972634,
+IQ26972671,
+IQ27129412,
+IQ27204765,
+IQ27400044,
+IQ27484055,
+IQ27490077,
+IQ27580696,
+IQ27580703,
+IQ27580715,
+IQ27606887,
+IQ27717292,
+IQ27868361,
+IQ27903956,
+IQ28100928,
+IQ28368577,
+IQ28387463,
+IQ28389647,
+IQ28942515,
+IQ29037395,
+IQ29183054,
+IQ29291881,
+IQ29601835,
+IQ30343267,
+IQ30441904,
+IQ30456609,
+IQ30465921,
+IQ30490816,
+IQ30512591,
+IQ30515896,
+IQ30601911,
+IQ30613257,
+IQ30637238,
+IQ30683814,
+IQ30705424,
+IQ30739286,
+IQ30789560,
+IQ30795690,
+IQ30884638,
+IQ30915913,
+IQ30964685,
+IQ31047931,
+IQ31065816,
+IQ31108425,
+IQ31131565,
+IQ31134863,
+IQ31136645,
+IQ31136770,
+IQ32589022,
+IQ32907389,
+IQ33350617,
+IQ33404477,
+IQ33597720,
+IQ34032519,
+IQ34360580,
+IQ34360903,
+IQ34399111,
+IQ34594700,
+IQ34648845,
+IQ34791159,
+IQ34791294,
+IQ35101458,
+IQ35134316,
+IQ35144642,
+IQ35392019,
+IQ35412230,
+IQ35424160,
+IQ35674202,
+IQ35749702,
+IQ35960436,
+IQ36621656,
+IQ36837622,
+IQ37262203,
+IQ37507853,
+IQ37782812,
+IQ38919390,
+IQ38991075,
+IQ39313436,
+IQ39601472,
+IQ39636404,
+IQ39637638,
+IQ39642860,
+IQ40153601,
+IQ40254139,
+IQ40311435,
+IQ40311458,
+IQ40332961,
+IQ40453340,
+IQ41087044,
+IQ42434220,
+IQ42439675,
+IQ42680296,
+IQ42746052,
+IQ43188556,
+IQ44459504,
+IQ44519103,
+IQ45978451,
+IQ46239777,
+IQ46367675,
+IQ46511782,
+IQ46638310,
+IQ46692499,
+IQ46771171,
+IQ46855213,
+IQ46865535,
+IQ46883648,
+IQ46884369,
+IQ46894216,
+IQ46895330,
+IQ46904183,
+IQ46954509,
+IQ46959086,
+IQ47014971,
+IQ47017185,
+IQ47064062,
+IQ47149593,
+IQ47439243,
+IQ47468586,
+IQ47475952,
+IQ47691832,
+IQ47838075,
+IQ47938173,
+IQ48970871,
+IQ49060929,
+IQ49380472,
+IQ49410153,
+IQ49411434,
+IQ49444150,
+IQ49516174,
+IQ49518798,
+IQ49545850,
+IQ49548668,
+IQ49650376,
+IQ49685718,
+IQ50015159,
+IQ50842667,
+IQ50876655,
+IQ51614649,
+IQ52034375,
+IQ52037418,
+IQ52139112,
+IQ52292994,
+IQ52324618,
+IQ52350790,
+IQ52414484,
+IQ52514348,
+IQ52715887,
+IQ53039417,
+IQ54075352,
+IQ54192726,
+IQ54408712,
+IQ54697432,
+IQ54838173,
+IQ54914944,
+IQ58106132,
+IQ58138359,
+IQ58258316,
+IQ58258443,
+IQ58859461,
+IQ59060203,
+IQ59322303,
+IQ59404618,
+IQ59404714,
+IQ59677595,
+IQ60229152,
+IQ60422576,
+IQ61350252,
+IQ62202991,
+IQ62315508,
+IQ79692443,
+IQ83478970,
+IQ83970849,
+IQ83984401,
+IQ95096703,
+IQ98741050,
+IQ99084162,
+IQ99937902,
+IQ100600351,
+IQ105289685,
+IQ106531222,
+IQ106631150,
+IQ109106005,
+IQ109134014,
+IQ109530569,
+IQ111222612,
+IQ112943576,
+IQ113534309,
+IQ113589908,
+IQ115789121,
+IQ116848128,
+IQ116959809,
+IQ117217007,
+IQ118031862,
+IQ118799766,
+IQ127913018,
+IQ129566475,
+IQ129642709,
+IQ132919045,
+IQ133764203,
+IQ134031773,
+IQ138304877,
+IQ139949085,
+IQ140722841,
+IQ140731306,
+IQ140786833,
+IQ141069742,
+IQ143995662,
+IQ144213439,
+IQ144398172,
+IQ144934211,
+IQ145637033,
+IQ166398410,
+IQ171518824,
+IQ207031891,
+IQ214891946,
+IQ216123401,
+IQ216542411,
+IQ216716050,
+IQ222661302,
+IQ225107913,
+IQ225463127,
+IQ232486210,
+IQ232732478,
+IQ241498873,
+IQ244663058,
+IQ245183346,
+IQ248535154,
+IQ248701560,
+IQ254045238,
+IQ256361384,
+IQ264279640,
+IQ264545174,
+IQ266784727,
+IQ267454529,
+IQ275369484,
+IQ279664808,
+IQ281069021,
+IQ306614602,
+IQ318869095,
+IQ320329041,
+IQ383989486,
+IQ420215945,
+IQ530308476,
+IQ47732041,
+IQ545930207,
diff --git a/data/revReport.yml b/data/revReport.yml
index 2493532..efa1eb2 100644
--- a/data/revReport.yml
+++ b/data/revReport.yml
@@ -5,13 +5,12 @@ params:
   currency: USD
   from: 1988-12-31
   to: 2018-12-31
-
-  # should handle loading identifiers from local file system, QT dataset, inline ones, STDIN
   identifiers:
-    local: ../capitaliq-identifiers.txt
-    dataset: d-e4tf3yyxerabcvicidv5oyey
+    # local: ./capitaliq-identifiers.txt
+    # dataset: d-pjuq6lofrnbszq5zntfy4rvv
     inline:
       - IQ121238
       - IQ598807
       - IQ723344
       - IQ956057
+      # - InvalidID

From be3ab3c7af8d171ca05a0981a01771aed0743b28 Mon Sep 17 00:00:00 2001
From: Grzegorz Bielski <gbielski@quantemplate.com>
Date: Mon, 19 Apr 2021 11:46:54 +0200
Subject: [PATCH 3/9] use java api instead of os lib

---
 build.sbt                                     |  1 -
 .../commands/ConfigDefInterpreterCmd.scala    | 12 +++--
 .../revenuereport/RevenueReportCmd.scala      | 50 +++++++++++++------
 .../RevenueReportConfigDef.scala              |  4 +-
 .../quantemplate/capitaliq/common/IO.scala    | 28 +++++++++--
 .../capitaliq/domain/CapitalIQ.scala          |  8 +--
 .../capitaliq/domain/Identifiers.scala        |  3 --
 data/revReport.yml                            | 11 ++--
 8 files changed, 76 insertions(+), 41 deletions(-)

diff --git a/build.sbt b/build.sbt
index 89bed78..83c5165 100644
--- a/build.sbt
+++ b/build.sbt
@@ -33,7 +33,6 @@ lazy val capitaliq = (project in file("capitaliq"))
       "org.typelevel" %% "cats-core" % "2.4.2",
       "com.norbitltd" %% "spoiwo" % "1.7.0",
       "com.github.scopt" %% "scopt" % "4.0.1",
-      "com.lihaoyi" %% "os-lib" % "0.7.4"
     ).map(_.withDottyCompat(scalaVersion.value)),
     assembly / mainClass := Some("com.quantemplate.capitaliq.Main")
   )
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala
index dc6ba3f..8319915 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala
@@ -1,6 +1,7 @@
 package com.quantemplate.capitaliq.commands
 
 import java.time.LocalDate
+import java.nio.file.Path
 import io.circe.{ Encoder, Decoder, Json, DecodingFailure }
 import io.circe.yaml.{parser as ymlParser}
 import org.slf4j.LoggerFactory
@@ -16,7 +17,7 @@ object ConfigDefInterpreterCmd:
   lazy val logger = LoggerFactory.getLogger(getClass)
 
   def fromCli(args: Array[String]) = ConfigArgsParser.parse(args).map { cliConfig =>
-    val configPath = getPath(cliConfig.path)
+    val configPath = IO.absolutePath(cliConfig.path)
 
     loadConfig(configPath).bimap(
       err => logger.error("Could not parse the config file", err),
@@ -26,10 +27,11 @@ object ConfigDefInterpreterCmd:
     )
   }
 
-private def loadConfig(path: os.Path) =
-  val yml = os.read(path)
-
-  ymlParser.parse(yml).flatMap(_.as[ConfigDef])
+private def loadConfig(path: Path) =
+  IO.readAll(path)
+    .toEither
+    .flatMap(ymlParser.parse(_))
+    .flatMap(_.as[ConfigDef])
 
 given Decoder[ConfigDef] = Decoder.instance[ConfigDef] { c => 
   c.get[String]("command").flatMap { 
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala
index 7a05f43..13981e8 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala
@@ -1,5 +1,6 @@
 package com.quantemplate.capitaliq.commands.revenuereport
 
+import java.nio.file.Path
 import akka.actor.typed.ActorSystem
 import akka.actor.typed.scaladsl.Behaviors
 import scala.concurrent.ExecutionContext
@@ -15,8 +16,8 @@ import com.quantemplate.capitaliq.qt.*
 
 class RevenueReportCmd:
   given Config = Config.load()
-  given sys: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "capitaliq")
-  given ExecutionContext = sys.executionContext
+  given system: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "capitaliq")
+  given ExecutionContext = system.executionContext
 
   lazy val logger = LoggerFactory.getLogger(getClass)
 
@@ -29,7 +30,7 @@ class RevenueReportCmd:
       .map(_.toCmdConfig(loadIdentifiersFromStdin()))
       .map(run)
 
-  def fromConfigFile(config: RevenueReportConfigDef, configPath: os.Path) = 
+  def fromConfigFile(config: RevenueReportConfigDef, configPath: Path) = 
     loadIdentifiersFromConfig(config, configPath)
       .map(_.getOrElse(loadIdentifiersFromStdin()))
       .map(config.toCmdConfig(_))
@@ -38,34 +39,51 @@ class RevenueReportCmd:
   private def loadIdentifiersFromStdin() = {
     logger.info("Loading the Capital IQ identifiers from the STDIN")
 
-    Identifiers.loadFromStdin()
+    IO.stdin(_.getLines.toVector) match
+      case Success(ids) => Identifiers(ids: _*)
+      case Failure(err) =>
+        logger.error("Could not load the Capital IQ identifiers from the STDIN. Aborting.") 
+        throw err
   }
 
-  private def loadIdentifiersFromConfig(config: RevenueReportConfigDef, configPath: os.Path) =
+  private def loadIdentifiersFromConfig(config: RevenueReportConfigDef, configPath: Path) =
     config.identifiers
       .flatMap(_.dataset)
-      .map(loadIdentifiersFromDataset(config.orgId, _))
+      .map(loadIdentifiersFromDataset(config.orgId))
       .sequence
       .map { remoteIds => 
         val inlineIds = config.identifiers.flatMap(_.inline)
-        val localIds = config.identifiers.flatMap(_.local).map { rawPath =>
-          Identifiers(
-            os.read.lines(getPath(rawPath, configPath / os.up)): _*
-          )
-        }
+        val localIds = config.identifiers
+          .flatMap(_.local)
+          .flatMap(loadIdentifiersFromLocalFile(configPath))
 
         (inlineIds ++ localIds ++ remoteIds).reduceOption(_ ++ _)
       }
 
-  private def loadIdentifiersFromDataset(orgId: String, datasetId: String) =
+  private def loadIdentifiersFromLocalFile(configPath: Path)(rawPath: String) =
+    // assuming the config path is a base for resolving the file path of identifiers
+    val idsPath = configPath.getParent.resolve(IO.toPath(rawPath))
+
+    IO.readLines(idsPath)
+      .recover {
+        case err: Throwable =>
+          logger.warn("Could not load the Capital IQ identifiers from the local file {}", err)
+          Vector.empty[String]
+      }
+      .toOption
+      .map(Identifiers(_: _*))
+      
+
+  private def loadIdentifiersFromDataset(datasetId: String)(orgId: String) =
     qtService.downloadDataset(orgId, datasetId)
       .map(Identifiers.loadFromCsvString)
+      .map { ids => 
+        logger.info("Loaded CapitalIQ identifiers from remote dataset")
+        ids
+      }
       .recover { 
         case e: Throwable => 
-          logger.warn("Could not load the identifiers from the remote dataset")
-          println(("err", e))
-          e.printStackTrace()
-
+          logger.warn("Could not load the CapitalIQ identifiers from the remote dataset. Defaulting to local ones.", e)
           Vector.empty[CapitalIQ.Identifier]
       }
 
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala
index 03f11f4..f696fe5 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala
@@ -27,7 +27,7 @@ case class RevenueReportConfigDef(
   )
 
 object RevenueReportConfigDef:
-  given Decoder[RevenueReportConfigDef] = Decoder.instance[RevenueReportConfigDef] { c => 
+  given Decoder[RevenueReportConfigDef] = Decoder { c => 
     (
       c.get[String]("orgId"),
       c.get[String]("datasetId"),
@@ -45,7 +45,7 @@ object RevenueReportConfigDef:
   )
 
   object Identifiers:
-    given Decoder[Identifiers] = Decoder.instance[Identifiers] { c => 
+    given Decoder[Identifiers] = Decoder { c => 
       (
         c.get[Option[String]]("local"),
         c.get[Option[String]]("dataset"),
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala
index ab47388..c5f5b5f 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala
@@ -1,8 +1,26 @@
 package com.quantemplate.capitaliq.common
 
-import java.nio.file.Paths
+import java.nio.file.Path
+import java.nio.file.FileSystems
+import scala.util.{Try, Using}
+import scala.io.{Source, BufferedSource}
 
-def getPath(rawPath: String, base: os.Path = os.pwd) =
-  if Paths.get(rawPath).isAbsolute
-    then os.Path(rawPath)
-    else os.Path(rawPath, base = base)
+object IO:
+  def pwd = System.getProperty("user.dir")
+
+  def stdin[A](fn: (BufferedSource => A)) = Using(Source.stdin)(fn)
+
+  def toPath(rawPath: String) =
+    FileSystems.getDefault.getPath(rawPath).normalize
+
+  def absolutePath(rawPath: String) = 
+    toPath(rawPath).toAbsolutePath
+
+  def readLines(path: Path) =
+    read(path)(_.getLines.toVector)
+
+  def readAll(path: Path) =
+    read(path)(_.getLines.mkString("\n"))
+
+  def read[A](path: Path): (BufferedSource => A) => Try[A] =
+     Using(Source.fromFile(path.toFile))
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala
index 73faa5a..f7b272e 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala
@@ -147,15 +147,15 @@ object CapitalIQ:
 
     case class MnemonicResponse(error: String, rows: Option[Rows])
     object MnemonicResponse:
-      given Decoder[RawResponse.MnemonicResponse] = Decoder.instance[RawResponse.MnemonicResponse] { cursor =>
+      given Decoder[RawResponse.MnemonicResponse] = Decoder { c =>
         for 
-          error <- cursor.get[String]("ErrMsg")
-          jsonRows <- cursor.get[Option[Vector[Json]]]("Rows")
+          error <- c.get[String]("ErrMsg")
+          jsonRows <- c.get[Option[Vector[Json]]]("Rows")
           rows <- jsonRows.traverse(_.traverse(_.hcursor.get[Vector[String]]("Row")))
         yield MnemonicResponse(error, rows)
       }
 
-    given Decoder[RawResponse] = Decoder.instance[RawResponse](
+    given Decoder[RawResponse] = Decoder(
       _.get[Vector[RawResponse.MnemonicResponse]]("GDSSDKResponse").map(RawResponse(_))
     )
 
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala
index 2901348..bf593aa 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala
@@ -20,6 +20,3 @@ object Identifiers:
         .toVector
         .map(_.split(',').headOption.getOrElse("")): _*
     )
-    
-  def loadFromStdin() = 
-    Identifiers(Source.stdin.getLines.toVector: _*)
diff --git a/data/revReport.yml b/data/revReport.yml
index efa1eb2..de08f56 100644
--- a/data/revReport.yml
+++ b/data/revReport.yml
@@ -6,11 +6,12 @@ params:
   from: 1988-12-31
   to: 2018-12-31
   identifiers:
+    # local: /Users/gbielski/Projects/qt/data-ingress/data/capitaliq-identifiers.txt
     # local: ./capitaliq-identifiers.txt
     # dataset: d-pjuq6lofrnbszq5zntfy4rvv
-    inline:
-      - IQ121238
-      - IQ598807
-      - IQ723344
-      - IQ956057
+    # inline:
+      # - IQ121238
+      # - IQ598807
+      # - IQ723344
+      # - IQ956057
       # - InvalidID

From 8667fe0db253d9e3cda98c006c0367034344ebb8 Mon Sep 17 00:00:00 2001
From: Grzegorz Bielski <gbielski@quantemplate.com>
Date: Mon, 19 Apr 2021 12:39:06 +0200
Subject: [PATCH 4/9] include demo account flag in config

---
 README.md                                                 | 2 ++
 capitaliq/src/main/resources/application.conf             | 8 +++++---
 .../scala/com/quantemplate/capitaliq/common/Config.scala  | 4 +++-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 426704e..48f47e8 100644
--- a/README.md
+++ b/README.md
@@ -75,6 +75,7 @@ An example set of identifiers could be found in the `./data/capitaliq-identifier
     ```
     CAPITALIQ_API_USERNAME=<api username you got with Capital IQ API license>
     CAPITALIQ_API_PASSWORD=<password for the corresponding Capital IQ API user>
+    CAPITALIQ_DEMO_ACCOUNT=<indicates whether the Capital IQ demo account is used>
     QT_ENV=<name of the Quantemplate environment, should be `prod` for any consumers>
     QT_AUTH_REALM=<name of the Quantemplate auth ream, should be `qt` for any consumers>
     QT_CLIENT_ID=<id of the api-client user generated by the Quantemplate team>
@@ -85,6 +86,7 @@ An example set of identifiers could be found in the `./data/capitaliq-identifier
     ```
     CAPITALIQ_API_USERNAME=apiadmin@quantemplate.com
     CAPITALIQ_API_PASSWORD=<password from dev credentials>
+    CAPITALIQ_DEMO_ACCOUNT=true
     QT_ENV=dev
     QT_AUTH_REALM=test
     QT_CLIENT_ID=<id of the api-client user>
diff --git a/capitaliq/src/main/resources/application.conf b/capitaliq/src/main/resources/application.conf
index 6e96510..626b14e 100644
--- a/capitaliq/src/main/resources/application.conf
+++ b/capitaliq/src/main/resources/application.conf
@@ -2,9 +2,11 @@ capitaliq {
   endpoint = "https://api-ciq.marketintelligence.spglobal.com/gdsapi/rest/v3/clientservice.json"
 
   # number of mnemonics / identifiers used in a single HTTP request
-  # 500 for prod account
-  # 100 for trial account
-  mnemonicsPerRequest = 100
+  mnemonicsPerRequestInDemoAccount = 100
+  mnemonicsPerRequestInProdAccount = 500
+
+  demoAccount = true
+  demoAccount = $?CAPITALIQ_DEMO_ACCOUNT}
 
   credentials {
     username = ${CAPITALIQ_API_USERNAME}
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/Config.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/Config.scala
index 70a04e4..772df1e 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/Config.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/Config.scala
@@ -35,7 +35,9 @@ object Config:
           conf.getString("capitaliq.credentials.username"),
           conf.getString("capitaliq.credentials.password"),
         ),
-        mnemonicsPerRequest = conf.getInt("capitaliq.mnemonicsPerRequest")
+        mnemonicsPerRequest = if conf.getBoolean("capitaliq.demoAccount")
+          then conf.getInt("capitaliq.mnemonicsPerRequestInDemoAccount")
+          else conf.getInt("capitaliq.mnemonicsPerRequestInProdAccount")
       ),
       Quantemplate(
         auth = Quantemplate.Auth(

From 681035a140c1f6180ccc7ac79e52441d5c5b9d6a Mon Sep 17 00:00:00 2001
From: Grzegorz Bielski <gbielski@quantemplate.com>
Date: Mon, 19 Apr 2021 14:55:03 +0200
Subject: [PATCH 5/9] update scala version, make simple test setup

---
 build.sbt                                     | 20 ++++--
 capitaliq/src/main/resources/application.conf |  3 +-
 .../capitaliq/common/HttpService.scala        |  1 -
 .../capitaliq/CapitalIQServiceSpec.scala      | 62 +++++++++++++++++++
 data/revReport.yml                            |  4 +-
 project/build.properties                      |  2 +-
 project/plugins.sbt                           |  1 -
 7 files changed, 81 insertions(+), 12 deletions(-)
 create mode 100644 capitaliq/src/test/scala/com/quantemplate/domain/capitaliq/CapitalIQServiceSpec.scala

diff --git a/build.sbt b/build.sbt
index 83c5165..818f800 100644
--- a/build.sbt
+++ b/build.sbt
@@ -1,7 +1,7 @@
 name := "data-ingress"
 
 ThisBuild / organization := "com.quantemplate"
-ThisBuild / scalaVersion := "3.0.0-RC1"
+ThisBuild / scalaVersion := "3.0.0-RC2"
 ThisBuild / version := "1.0"
 
 val AkkaVersion = "2.6.12"
@@ -27,13 +27,23 @@ lazy val capitaliq = (project in file("capitaliq"))
       "io.circe" %% "circe-parser" % CirceVersion,
       "io.circe" %% "circe-yaml" % "0.13.1",
       // misc
-      "org.scalatest" %% "scalatest" % "3.1.0" % Test,
-      "ch.qos.logback" % "logback-classic" % "1.2.3",
-      "com.typesafe" % "config" % "1.4.1",
       "org.typelevel" %% "cats-core" % "2.4.2",
       "com.norbitltd" %% "spoiwo" % "1.7.0",
       "com.github.scopt" %% "scopt" % "4.0.1",
-    ).map(_.withDottyCompat(scalaVersion.value)),
+    ).map(_.cross(CrossVersion.for3Use2_13)),
+    
+    // java deps
+    libraryDependencies ++= Seq(
+      "ch.qos.logback" % "logback-classic" % "1.2.3",
+      "com.typesafe" % "config" % "1.4.1",
+      "org.mockito" % "mockito-core" % "3.9.0"
+    ),
+
+    // native scala 3 deps
+    libraryDependencies ++= Seq(
+      "org.scalameta" %% "munit" % "0.7.23" % Test
+    ),
+
     assembly / mainClass := Some("com.quantemplate.capitaliq.Main")
   )
 
diff --git a/capitaliq/src/main/resources/application.conf b/capitaliq/src/main/resources/application.conf
index 626b14e..685cabb 100644
--- a/capitaliq/src/main/resources/application.conf
+++ b/capitaliq/src/main/resources/application.conf
@@ -5,8 +5,7 @@ capitaliq {
   mnemonicsPerRequestInDemoAccount = 100
   mnemonicsPerRequestInProdAccount = 500
 
-  demoAccount = true
-  demoAccount = $?CAPITALIQ_DEMO_ACCOUNT}
+  demoAccount = ${CAPITALIQ_DEMO_ACCOUNT}
 
   credentials {
     username = ${CAPITALIQ_API_USERNAME}
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala
index 70fb808..efcfff7 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala
@@ -25,7 +25,6 @@ class HttpService(using system: ActorSystem[_]):
     for
       res <- GET(endpoint, auth)
       body <- getResponseBody(res)
-      // result <- Unmarshal(body).to[B]
     yield body.utf8String
 
   def post[B: Decoder](
diff --git a/capitaliq/src/test/scala/com/quantemplate/domain/capitaliq/CapitalIQServiceSpec.scala b/capitaliq/src/test/scala/com/quantemplate/domain/capitaliq/CapitalIQServiceSpec.scala
new file mode 100644
index 0000000..875e1d9
--- /dev/null
+++ b/capitaliq/src/test/scala/com/quantemplate/domain/capitaliq/CapitalIQServiceSpec.scala
@@ -0,0 +1,62 @@
+package com.quantemplate.capitaliq.domain
+
+import munit.FunSuite
+import org.mockito.Mockito.*
+import org.mockito.ArgumentMatchers.*
+import scala.concurrent.Future
+import scala.concurrent.ExecutionContext
+import akka.actor.typed.ActorSystem
+import akka.actor.typed.scaladsl.Behaviors
+import cats.syntax.option.*
+
+import com.quantemplate.capitaliq.common.{Config, HttpService}
+import com.quantemplate.capitaliq.domain.CapitalIQ.*
+import com.quantemplate.capitaliq.domain.CapitalIQService.*
+
+class CapitalIQServiceSpec extends FunSuite:
+  test("the response adaptation") {
+    new Context():
+      run {
+        val mockHttpService = mock(classOf[HttpService])
+
+        val mnemonic = Mnemonic.IQ_COMPANY_NAME_LONG(Identifier("IQ12334"))
+        val rows = Vector(Vector("PZU"))
+
+        val rawResponse = RawResponse(
+          Vector(RawResponse.MnemonicResponse("", rows.some))
+        )
+
+        when(
+          mockHttpService.post[Request, RawResponse](any(), any(), any())(any(), any())
+        ).thenReturn(Future.successful(rawResponse))
+        
+        val service = CapitalIQService(mockHttpService)
+
+        for 
+          result <- service.sendRequest(Request(Vector(mnemonic)))
+          _ = assertEquals(result, Vector(Response(mnemonic, rows)))
+        yield ()
+      }
+  }
+
+class Context:
+  given system: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "test-system")
+  given ExecutionContext = system.executionContext
+  given Config = Config(
+    capitaliq = Config.CapitalIQ(
+      endpoint = "http://",
+      mnemonicsPerRequest = 100,
+      credentials = Config.CapitalIQ.Credentials("username", "password"),
+    ),
+    quantemplate = Config.Quantemplate(
+      auth = Config.Quantemplate.Auth("http://", "clientid", "clientSecret"),
+      api = Config.Quantemplate.Api("http://")
+    )
+  )
+
+  def run[T](code: => T) =  
+    try
+      code
+    finally
+      system.terminate()
+
diff --git a/data/revReport.yml b/data/revReport.yml
index de08f56..af897d3 100644
--- a/data/revReport.yml
+++ b/data/revReport.yml
@@ -9,8 +9,8 @@ params:
     # local: /Users/gbielski/Projects/qt/data-ingress/data/capitaliq-identifiers.txt
     # local: ./capitaliq-identifiers.txt
     # dataset: d-pjuq6lofrnbszq5zntfy4rvv
-    # inline:
-      # - IQ121238
+    inline:
+      - IQ121238
       # - IQ598807
       # - IQ723344
       # - IQ956057
diff --git a/project/build.properties b/project/build.properties
index dbae93b..e67343a 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -1 +1 @@
-sbt.version=1.4.9
+sbt.version=1.5.0
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 54dee10..b6a6320 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,3 +1,2 @@
-addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "0.5.3")
 addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0")
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6")

From 5967e449ae25e13e665324280918555a4bec29f0 Mon Sep 17 00:00:00 2001
From: Grzegorz Bielski <gbielski@quantemplate.com>
Date: Mon, 19 Apr 2021 16:15:34 +0200
Subject: [PATCH 6/9] remove junk files

---
 data/capitaliq-identifiers.csv | 850 ---------------------------------
 1 file changed, 850 deletions(-)
 delete mode 100644 data/capitaliq-identifiers.csv

diff --git a/data/capitaliq-identifiers.csv b/data/capitaliq-identifiers.csv
deleted file mode 100644
index 6d5b48c..0000000
--- a/data/capitaliq-identifiers.csv
+++ /dev/null
@@ -1,850 +0,0 @@
-identifiers,
-IQ121238,
-IQ598807,
-IQ723344,
-IQ956057,
-IQ250388,
-IQ5471113,
-IQ876215,
-IQ162456,
-IQ8201008,
-IQ22517019,
-IQ22663157,
-IQ875826,
-IQ992287,
-IQ8248315,
-IQ20551569,
-IQ874488,
-IQ341546,
-IQ876731,
-IQ8210142,
-IQ1082017,
-IQ698550,
-IQ876955,
-IQ5467549,
-IQ8475394,
-IQ882901,
-IQ7605761,
-IQ875328,
-IQ36739404,
-IQ878160,
-IQ2969167,
-IQ2159401,
-IQ877802,
-IQ3680446,
-IQ5477675,
-IQ303566889,
-IQ8125089,
-IQ882957,
-IQ4512131,
-IQ883670,
-IQ295539,
-IQ22510313,
-IQ4026111,
-IQ22729379,
-IQ26471214,
-IQ8475467,
-IQ37612389,
-IQ4481454,
-IQ8474885,
-IQ215098026,
-IQ2482281,
-IQ5491822,
-IQ27515847,
-IQ5478545,
-IQ883047,
-IQ12732661,
-IQ876952,
-IQ3561426,
-IQ874863,
-IQ240824094,
-IQ1051073,
-IQ199668,
-IQ20742105,
-IQ5470931,
-IQ874600,
-IQ41356562,
-IQ22533749,
-IQ875917,
-IQ250307,
-IQ4497069,
-IQ9235665,
-IQ966841,
-IQ883901,
-IQ20970264,
-IQ965915,
-IQ8221280,
-IQ5495596,
-IQ8105207,
-IQ1542889,
-IQ882762,
-IQ1882861,
-IQ13335251,
-IQ13483916,
-IQ12933390,
-IQ22039549,
-IQ8952000,
-IQ8237245,
-IQ8642193,
-IQ26972175,
-IQ259879432,
-IQ8292712,
-IQ225687232,
-IQ877323,
-IQ5476832,
-IQ22547601,
-IQ7884005,
-IQ26326530,
-IQ22134956,
-IQ5525514,
-IQ5566600,
-IQ10245092,
-IQ318898,
-IQ22548905,
-IQ877527,
-IQ307183867,
-IQ878130,
-IQ5569241,
-IQ12205230,
-IQ20527977,
-IQ5473193,
-IQ8982543,
-IQ26934166,
-IQ5604059,
-IQ5686144,
-IQ30921662,
-IQ10639513,
-IQ875073,
-IQ53474567,
-IQ32671961,
-IQ873776,
-IQ5533523,
-IQ615970073,
-IQ9514473,
-IQ59372417,
-IQ32777643,
-IQ261155221,
-IQ11519044,
-IQ26841815,
-IQ5569256,
-IQ11461103,
-IQ20394473,
-IQ20457561,
-IQ138540308,
-IQ20357747,
-IQ7683448,
-IQ310010418,
-IQ245593332,
-IQ9450216,
-IQ38010476,
-IQ8751533,
-IQ224448138,
-IQ20929816,
-IQ28526399,
-IQ21790041,
-IQ879281,
-IQ22709629,
-IQ233447435,
-IQ9003966,
-IQ22830143,
-IQ53474451,
-IQ20644228,
-IQ6462568,
-IQ128394122,
-IQ5554951,
-IQ5525485,
-IQ10580089,
-IQ20406261,
-IQ20334327,
-IQ32879143,
-IQ59290204,
-IQ35622138,
-IQ22530726,
-IQ2395875,
-IQ9285238,
-IQ20968703,
-IQ20411372,
-IQ36066453,
-IQ20405257,
-IQ41542793,
-IQ6499589,
-IQ24351707,
-IQ254768850,
-IQ35045462,
-IQ876389,
-IQ252124,
-IQ699678,
-IQ20376386,
-IQ30777099,
-IQ36070234,
-IQ20442970,
-IQ5705453,
-IQ20357911,
-IQ8951913,
-IQ20875991,
-IQ34360884,
-IQ45364909,
-IQ20439959,
-IQ419608830,
-IQ20388152,
-IQ142687475,
-IQ25603857,
-IQ20401153,
-IQ44731152,
-IQ9638335,
-IQ880319,
-IQ37530273,
-IQ8237517,
-IQ45364901,
-IQ9104794,
-IQ118758025,
-IQ23136532,
-IQ20385336,
-IQ47019555,
-IQ280102994,
-IQ6182739,
-IQ24389424,
-IQ98911479,
-IQ118529610,
-IQ34360875,
-IQ6096740,
-IQ206594311,
-IQ20351433,
-IQ20387499,
-IQ20357133,
-IQ25853564,
-IQ884289,
-IQ26840491,
-IQ9446689,
-IQ52657639,
-IQ22817884,
-IQ105309130,
-IQ36824712,
-IQ20342548,
-IQ41542810,
-IQ8863946,
-IQ26425108,
-IQ9317591,
-IQ9648039,
-IQ5435672,
-IQ9898706,
-IQ4509080,
-IQ20392649,
-IQ25601530,
-IQ11251263,
-IQ13338102,
-IQ20333219,
-IQ23956851,
-IQ9383541,
-IQ20928564,
-IQ33059622,
-IQ10837576,
-IQ39420520,
-IQ49756433,
-IQ20386271,
-IQ20387422,
-IQ22519534,
-IQ13389347,
-IQ9616853,
-IQ582343023,
-IQ47286427,
-IQ212327509,
-IQ30585041,
-IQ11354999,
-IQ26539734,
-IQ129791710,
-IQ54412624,
-IQ98910795,
-IQ11459490,
-IQ30746047,
-IQ4493353,
-IQ36070207,
-IQ36218630,
-IQ20356961,
-IQ233903518,
-IQ142039532,
-IQ47938242,
-IQ20388408,
-IQ105289664,
-IQ30612714,
-IQ13483021,
-IQ5449547,
-IQ5514776,
-IQ53474509,
-IQ34430293,
-IQ109965608,
-IQ30468440,
-IQ9648367,
-IQ59676977,
-IQ28506082,
-IQ29785842,
-IQ9648084,
-IQ38928712,
-IQ49943055,
-IQ20385355,
-IQ51446644,
-IQ47285586,
-IQ250801799,
-IQ20406066,
-IQ30746764,
-IQ20440724,
-IQ20406489,
-IQ46949286,
-IQ49273432,
-IQ9648044,
-IQ208360255,
-IQ36308773,
-IQ20376809,
-IQ20384809,
-IQ10167110,
-IQ47733696,
-IQ20379938,
-IQ34360936,
-IQ9265738,
-IQ20387321,
-IQ9264708,
-IQ20405282,
-IQ30446153,
-IQ7958370,
-IQ30589185,
-IQ12004130,
-IQ107792708,
-IQ20387682,
-IQ30864618,
-IQ20384756,
-IQ142681075,
-IQ9663375,
-IQ29226678,
-IQ104919719,
-IQ20357359,
-IQ20406865,
-IQ879972,
-IQ117920629,
-IQ83672799,
-IQ3448881,
-IQ9648086,
-IQ11311924,
-IQ23019707,
-IQ217249500,
-IQ20399470,
-IQ225000580,
-IQ20333227,
-IQ236534039,
-IQ28093936,
-IQ321937608,
-IQ30445681,
-IQ13199091,
-IQ552051031,
-IQ22249494,
-IQ9648296,
-IQ21927946,
-IQ317398442,
-IQ22453364,
-IQ11900969,
-IQ83586150,
-IQ20405278,
-IQ34579314,
-IQ138470937,
-IQ4462958,
-IQ112208069,
-IQ46950019,
-IQ20823232,
-IQ20376805,
-IQ30607633,
-IQ20356260,
-IQ27142992,
-IQ22929324,
-IQ46910621,
-IQ416203171,
-IQ12913444,
-IQ20407950,
-IQ42114936,
-IQ29383141,
-IQ106886296,
-IQ114576425,
-IQ84209473,
-IQ31093,
-IQ169008,
-IQ250309,
-IQ310818,
-IQ632483,
-IQ678651,
-IQ722875,
-IQ846304,
-IQ872445,
-IQ874020,
-IQ875292,
-IQ879079,
-IQ880335,
-IQ881280,
-IQ881579,
-IQ918690,
-IQ973824,
-IQ1028109,
-IQ1056331,
-IQ1511109,
-IQ1515993,
-IQ1529888,
-IQ1549703,
-IQ2745726,
-IQ2759623,
-IQ2897241,
-IQ2994821,
-IQ3649650,
-IQ4090862,
-IQ4096555,
-IQ4163031,
-IQ4165794,
-IQ4168338,
-IQ4183677,
-IQ4205936,
-IQ4210427,
-IQ4217488,
-IQ4222324,
-IQ4224615,
-IQ4249211,
-IQ4251241,
-IQ4279638,
-IQ4303310,
-IQ4332750,
-//IQ4347511 invalid,
-IQ4460048,
-IQ4472553,
-IQ4481550,
-IQ4494248,
-IQ4497162,
-IQ4559043,
-IQ4561438,
-IQ4586814,
-IQ4662943,
-IQ4717183,
-IQ4730735,
-IQ4762761,
-IQ4772278,
-IQ5068910,
-IQ5375495,
-IQ5383311,
-IQ5410293,
-IQ5423523,
-IQ5466287,
-IQ5477916,
-IQ5484075,
-IQ5491169,
-IQ5496125,
-IQ5507600,
-IQ5510525,
-IQ5525297,
-IQ5538187,
-IQ5538747,
-IQ5542158,
-IQ5547143,
-IQ5547613,
-IQ5554373,
-IQ5569286,
-IQ5579426,
-IQ5647001,
-IQ5704985,
-IQ5718690,
-IQ5723278,
-IQ5726675,
-IQ5774517,
-IQ5885638,
-IQ5901538,
-IQ5924809,
-IQ6139502,
-IQ6439926,
-IQ6463183,
-IQ6469667,
-IQ6475664,
-IQ6491233,
-IQ6564312,
-IQ6565183,
-IQ6613221,
-IQ6688088,
-IQ6699370,
-IQ6700017,
-IQ6812156,
-IQ6819792,
-IQ6921281,
-IQ6948694,
-IQ6949267,
-IQ7603981,
-IQ7918016,
-IQ7960416,
-IQ8024164,
-IQ8100801,
-IQ8101330,
-IQ8142882,
-IQ8190935,
-IQ8264274,
-IQ8359879,
-IQ9377948,
-IQ9450267,
-IQ9460690,
-IQ9461042,
-IQ9550032,
-IQ9560354,
-IQ9587298,
-IQ9638878,
-IQ9794004,
-IQ9899048,
-IQ9916947,
-IQ10052670,
-IQ11212063,
-IQ11323005,
-IQ11712739,
-IQ12058656,
-IQ12455886,
-IQ12456448,
-IQ12466795,
-IQ12784598,
-IQ12785981,
-IQ12868386,
-IQ12913479,
-IQ12932470,
-IQ13011212,
-IQ13154195,
-IQ13408154,
-IQ13610052,
-IQ20417852,
-IQ20475401,
-IQ20551258,
-IQ20670757,
-IQ20703772,
-IQ20724600,
-IQ20757724,
-IQ20784162,
-IQ20882165,
-IQ20986395,
-IQ21813186,
-IQ21854837,
-IQ21935579,
-IQ22095148,
-IQ22239048,
-IQ22383525,
-IQ22383606,
-IQ22431794,
-IQ22498272,
-IQ22511361,
-IQ22515611,
-IQ22515844,
-IQ22515964,
-IQ22516125,
-IQ22517048,
-IQ22534319,
-IQ22534512,
-IQ22547744,
-IQ22580222,
-IQ22581143,
-IQ22581472,
-IQ22631649,
-IQ22638720,
-IQ22728611,
-IQ22783832,
-IQ22798804,
-IQ22799264,
-IQ22905959,
-IQ22926403,
-IQ22931810,
-IQ23018346,
-IQ23021162,
-IQ23021452,
-IQ23035756,
-IQ23048633,
-IQ23063990,
-IQ23076171,
-IQ23185132,
-IQ23221640,
-IQ23294481,
-IQ23313835,
-IQ23335868,
-IQ23336901,
-IQ23631262,
-IQ23751882,
-IQ23756872,
-IQ23780053,
-IQ23958224,
-IQ24393771,
-IQ24861410,
-IQ25473977,
-IQ25525075,
-IQ25746444,
-IQ25813840,
-IQ25833297,
-IQ25835239,
-IQ25870800,
-IQ25878705,
-IQ25912594,
-IQ25913389,
-IQ25913460,
-IQ25915583,
-IQ25935856,
-IQ26023649,
-IQ26079324,
-IQ26307837,
-IQ26321017,
-IQ26442394,
-IQ26502798,
-IQ26554716,
-IQ26671175,
-IQ26763923,
-IQ26935576,
-IQ26935957,
-IQ26937924,
-IQ26954558,
-IQ26954965,
-IQ26955671,
-IQ26956943,
-IQ26962223,
-IQ26971203,
-IQ26971884,
-IQ26971963,
-IQ26971979,
-IQ26972138,
-IQ26972384,
-IQ26972531,
-IQ26972634,
-IQ26972671,
-IQ27129412,
-IQ27204765,
-IQ27400044,
-IQ27484055,
-IQ27490077,
-IQ27580696,
-IQ27580703,
-IQ27580715,
-IQ27606887,
-IQ27717292,
-IQ27868361,
-IQ27903956,
-IQ28100928,
-IQ28368577,
-IQ28387463,
-IQ28389647,
-IQ28942515,
-IQ29037395,
-IQ29183054,
-IQ29291881,
-IQ29601835,
-IQ30343267,
-IQ30441904,
-IQ30456609,
-IQ30465921,
-IQ30490816,
-IQ30512591,
-IQ30515896,
-IQ30601911,
-IQ30613257,
-IQ30637238,
-IQ30683814,
-IQ30705424,
-IQ30739286,
-IQ30789560,
-IQ30795690,
-IQ30884638,
-IQ30915913,
-IQ30964685,
-IQ31047931,
-IQ31065816,
-IQ31108425,
-IQ31131565,
-IQ31134863,
-IQ31136645,
-IQ31136770,
-IQ32589022,
-IQ32907389,
-IQ33350617,
-IQ33404477,
-IQ33597720,
-IQ34032519,
-IQ34360580,
-IQ34360903,
-IQ34399111,
-IQ34594700,
-IQ34648845,
-IQ34791159,
-IQ34791294,
-IQ35101458,
-IQ35134316,
-IQ35144642,
-IQ35392019,
-IQ35412230,
-IQ35424160,
-IQ35674202,
-IQ35749702,
-IQ35960436,
-IQ36621656,
-IQ36837622,
-IQ37262203,
-IQ37507853,
-IQ37782812,
-IQ38919390,
-IQ38991075,
-IQ39313436,
-IQ39601472,
-IQ39636404,
-IQ39637638,
-IQ39642860,
-IQ40153601,
-IQ40254139,
-IQ40311435,
-IQ40311458,
-IQ40332961,
-IQ40453340,
-IQ41087044,
-IQ42434220,
-IQ42439675,
-IQ42680296,
-IQ42746052,
-IQ43188556,
-IQ44459504,
-IQ44519103,
-IQ45978451,
-IQ46239777,
-IQ46367675,
-IQ46511782,
-IQ46638310,
-IQ46692499,
-IQ46771171,
-IQ46855213,
-IQ46865535,
-IQ46883648,
-IQ46884369,
-IQ46894216,
-IQ46895330,
-IQ46904183,
-IQ46954509,
-IQ46959086,
-IQ47014971,
-IQ47017185,
-IQ47064062,
-IQ47149593,
-IQ47439243,
-IQ47468586,
-IQ47475952,
-IQ47691832,
-IQ47838075,
-IQ47938173,
-IQ48970871,
-IQ49060929,
-IQ49380472,
-IQ49410153,
-IQ49411434,
-IQ49444150,
-IQ49516174,
-IQ49518798,
-IQ49545850,
-IQ49548668,
-IQ49650376,
-IQ49685718,
-IQ50015159,
-IQ50842667,
-IQ50876655,
-IQ51614649,
-IQ52034375,
-IQ52037418,
-IQ52139112,
-IQ52292994,
-IQ52324618,
-IQ52350790,
-IQ52414484,
-IQ52514348,
-IQ52715887,
-IQ53039417,
-IQ54075352,
-IQ54192726,
-IQ54408712,
-IQ54697432,
-IQ54838173,
-IQ54914944,
-IQ58106132,
-IQ58138359,
-IQ58258316,
-IQ58258443,
-IQ58859461,
-IQ59060203,
-IQ59322303,
-IQ59404618,
-IQ59404714,
-IQ59677595,
-IQ60229152,
-IQ60422576,
-IQ61350252,
-IQ62202991,
-IQ62315508,
-IQ79692443,
-IQ83478970,
-IQ83970849,
-IQ83984401,
-IQ95096703,
-IQ98741050,
-IQ99084162,
-IQ99937902,
-IQ100600351,
-IQ105289685,
-IQ106531222,
-IQ106631150,
-IQ109106005,
-IQ109134014,
-IQ109530569,
-IQ111222612,
-IQ112943576,
-IQ113534309,
-IQ113589908,
-IQ115789121,
-IQ116848128,
-IQ116959809,
-IQ117217007,
-IQ118031862,
-IQ118799766,
-IQ127913018,
-IQ129566475,
-IQ129642709,
-IQ132919045,
-IQ133764203,
-IQ134031773,
-IQ138304877,
-IQ139949085,
-IQ140722841,
-IQ140731306,
-IQ140786833,
-IQ141069742,
-IQ143995662,
-IQ144213439,
-IQ144398172,
-IQ144934211,
-IQ145637033,
-IQ166398410,
-IQ171518824,
-IQ207031891,
-IQ214891946,
-IQ216123401,
-IQ216542411,
-IQ216716050,
-IQ222661302,
-IQ225107913,
-IQ225463127,
-IQ232486210,
-IQ232732478,
-IQ241498873,
-IQ244663058,
-IQ245183346,
-IQ248535154,
-IQ248701560,
-IQ254045238,
-IQ256361384,
-IQ264279640,
-IQ264545174,
-IQ266784727,
-IQ267454529,
-IQ275369484,
-IQ279664808,
-IQ281069021,
-IQ306614602,
-IQ318869095,
-IQ320329041,
-IQ383989486,
-IQ420215945,
-IQ530308476,
-IQ47732041,
-IQ545930207,

From f0b88954edb91857505a5473e049ddb83f8f4d0f Mon Sep 17 00:00:00 2001
From: Grzegorz Bielski <gbielski@quantemplate.com>
Date: Mon, 19 Apr 2021 17:06:53 +0200
Subject: [PATCH 7/9] update readme

---
 README.md                                     | 17 ++++++++++----
 build.sbt                                     |  4 ++--
 .../revenuereport/RevenueReportCmd.scala      |  8 ++++++-
 .../capitaliq/domain/Identifiers.scala        |  2 +-
 data/revReport.yml                            | 23 +++++++++----------
 5 files changed, 34 insertions(+), 20 deletions(-)

diff --git a/README.md b/README.md
index 48f47e8..6338a40 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 
 Integrations with third-party data providers such as [Capital IQ](https://www.capitaliq.com) leveraging the [Quantemplate Data Ingress API](https://quantemplate.readme.io/docs/getting-started#-data-ingress).
 
-If you need help please contact as at support@quantemplate.com
+If you need help please contact us at support@quantemplate.com
 
 
 # Capital IQ
@@ -26,9 +26,18 @@ The diagram below describes a potential integration pattern where CapitalIQ data
 
 - Generating a total revenue report from the CapitalIQ data and uploading it to the Quantemplate dataset
 
-    ```sh
-    cat ./data/capitaliq-identifiers.txt | java -jar ./capitaliq/target/scala-3.0.0-RC1/capitaliq-assembly-1.0.jar generateRevenueReport --orgId c-my-small-insuranc-ltdzfd --datasetId d-e4tf3yyxerabcvicidv5oyey --currency USD --from 1988-12-31 --to 2018-12-31
-    ```
+    - with yaml config:
+        ```sh
+        java -jar ./capitaliq/target/scala-3.0.0-RC2/capitaliq-assembly-1.0.jar apply ./data/revReport.yml
+        ```
+
+        Check out the [config file](./data/revReport.yml)
+
+
+    - with CLI args:
+        ```sh
+        cat ./data/capitaliq-identifiers.txt | java -jar ./capitaliq/target/scala-3.0.0-RC2/capitaliq-assembly-1.0.jar generateRevenueReport --orgId c-my-small-insuranc-ltdzfd --datasetId d-e4tf3yyxerabcvicidv5oyey --currency USD --from 1988-12-31 --to 2018-12-31
+        ```
 
 
 
diff --git a/build.sbt b/build.sbt
index 818f800..95b5fef 100644
--- a/build.sbt
+++ b/build.sbt
@@ -36,7 +36,7 @@ lazy val capitaliq = (project in file("capitaliq"))
     libraryDependencies ++= Seq(
       "ch.qos.logback" % "logback-classic" % "1.2.3",
       "com.typesafe" % "config" % "1.4.1",
-      "org.mockito" % "mockito-core" % "3.9.0"
+      "org.mockito" % "mockito-core" % "3.9.0" % Test
     ),
 
     // native scala 3 deps
@@ -44,7 +44,7 @@ lazy val capitaliq = (project in file("capitaliq"))
       "org.scalameta" %% "munit" % "0.7.23" % Test
     ),
 
-    assembly / mainClass := Some("com.quantemplate.capitaliq.Main")
+    assembly / mainClass := Some("com.quantemplate.capitaliq.Main"),
   )
 
 Compile / run / fork := true
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala
index 13981e8..186f34a 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala
@@ -88,9 +88,15 @@ class RevenueReportCmd:
       }
 
   private def run(config: CmdConfig) =
+    val ids = config.identifiers.distinct
+
+    if ids.isEmpty then 
+      logger.error("No valid CapitalIQ identifiers were provided. Aborting")
+      Runtime.getRuntime.halt(1)
+
     revenueReport
       .generateSpreadSheet(
-        ids = config.identifiers.distinct,
+        ids = ids,
         range = (
           config.from,
           config.to
diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala
index bf593aa..8a608e6 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala
@@ -5,7 +5,7 @@ import scala.io.Source
 import CapitalIQ.Identifier
 
 object Identifiers:
-  def apply(ids: String*) = 
+  def apply(ids: String*) =
     ids
       .filter(Identifier.isValid)
       .map(Identifier(_))
diff --git a/data/revReport.yml b/data/revReport.yml
index af897d3..c95a0a0 100644
--- a/data/revReport.yml
+++ b/data/revReport.yml
@@ -1,16 +1,15 @@
 command: generateRevenueReport
-params:
-  orgId: c-my-small-insuranc-ltdzfd
-  datasetId: d-e4tf3yyxerabcvicidv5oyey
-  currency: USD
-  from: 1988-12-31
-  to: 2018-12-31
-  identifiers:
-    # local: /Users/gbielski/Projects/qt/data-ingress/data/capitaliq-identifiers.txt
-    # local: ./capitaliq-identifiers.txt
-    # dataset: d-pjuq6lofrnbszq5zntfy4rvv
-    inline:
-      - IQ121238
+params: 
+  orgId: c-my-small-insuranc-ltdzfd       # Quantemplate organisationId
+  datasetId: d-e4tf3yyxerabcvicidv5oyey   # Quantemplate datasetId
+  currency: USD                           # Currency supported by the CapitalIQ
+  from: 1988-12-31                        # Starting date in yyy-mm-dd format for the CapitalIQ query
+  to: 2018-12-31                          # Ending date in yyy-mm-dd format for the CapitalIQ query
+  identifiers:                            # CapitalIQ identifiers, that could be downloaded from:
+    # dataset: d-pjuq6lofrnbszq5zntfy4rvv #  - remote Quantemplate `dataset` (currently it only takes data from the first column)                                                  
+    # local: ./capitaliq-identifiers.txt  #  - `local` .txt file,  (relative file path will be resolved against the config file)
+    inline:                               #  - defined `inline` in the config file
+      # - IQ121238                        # Multiple sources of identifiers will be merged together
       # - IQ598807
       # - IQ723344
       # - IQ956057

From 21fc08b5d3106bada39e1b2734912efac3743cd6 Mon Sep 17 00:00:00 2001
From: Grzegorz Bielski <gbielski@quantemplate.com>
Date: Tue, 20 Apr 2021 10:41:05 +0200
Subject: [PATCH 8/9] update repo name

---
 README.md | 8 ++++----
 build.sbt | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 6338a40..a3b03ca 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
-# data-ingress
+# api-integrations
 
-Integrations with third-party data providers such as [Capital IQ](https://www.capitaliq.com) leveraging the [Quantemplate Data Ingress API](https://quantemplate.readme.io/docs/getting-started#-data-ingress).
+Integrations with third-party data providers such as [Capital IQ](https://www.capitaliq.com) leveraging the [Quantemplate API](https://quantemplate.readme.io/docs/getting-started).
 
 If you need help please contact us at support@quantemplate.com
 
@@ -28,7 +28,7 @@ The diagram below describes a potential integration pattern where CapitalIQ data
 
     - with yaml config:
         ```sh
-        java -jar ./capitaliq/target/scala-3.0.0-RC2/capitaliq-assembly-1.0.jar apply ./data/revReport.yml
+        java -jar ./capitaliq/target/scala-3.0.0-RC2/capitaliq-assembly-0.1.0.jar apply ./data/revReport.yml
         ```
 
         Check out the [config file](./data/revReport.yml)
@@ -36,7 +36,7 @@ The diagram below describes a potential integration pattern where CapitalIQ data
 
     - with CLI args:
         ```sh
-        cat ./data/capitaliq-identifiers.txt | java -jar ./capitaliq/target/scala-3.0.0-RC2/capitaliq-assembly-1.0.jar generateRevenueReport --orgId c-my-small-insuranc-ltdzfd --datasetId d-e4tf3yyxerabcvicidv5oyey --currency USD --from 1988-12-31 --to 2018-12-31
+        cat ./data/capitaliq-identifiers.txt | java -jar ./capitaliq/target/scala-3.0.0-RC2/capitaliq-assembly-0.1.0.jar generateRevenueReport --orgId c-my-small-insuranc-ltdzfd --datasetId d-e4tf3yyxerabcvicidv5oyey --currency USD --from 1988-12-31 --to 2018-12-31
         ```
 
 
diff --git a/build.sbt b/build.sbt
index 95b5fef..1810d0a 100644
--- a/build.sbt
+++ b/build.sbt
@@ -1,8 +1,8 @@
-name := "data-ingress"
+name := "api-integrations"
 
 ThisBuild / organization := "com.quantemplate"
 ThisBuild / scalaVersion := "3.0.0-RC2"
-ThisBuild / version := "1.0"
+ThisBuild / version := "0.1.0"
 
 val AkkaVersion = "2.6.12"
 val AkkaHttpVersion = "10.2.4"

From 87b1e06159374d5de25ff081cbf00ba1fe148f3f Mon Sep 17 00:00:00 2001
From: Grzegorz Bielski <gbielski@quantemplate.com>
Date: Tue, 20 Apr 2021 11:50:09 +0200
Subject: [PATCH 9/9] make revenue report cmd members private

---
 .../commands/revenuereport/RevenueReportCmd.scala  | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala
index 186f34a..9a799f7 100644
--- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala
+++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala
@@ -15,15 +15,15 @@ import com.quantemplate.capitaliq.qt.*
 
 
 class RevenueReportCmd:
-  given Config = Config.load()
-  given system: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "capitaliq")
-  given ExecutionContext = system.executionContext
+  private given Config = Config.load()
+  private given system: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "capitaliq")
+  private given ExecutionContext = system.executionContext
 
-  lazy val logger = LoggerFactory.getLogger(getClass)
+  private lazy val logger = LoggerFactory.getLogger(getClass)
 
-  val httpService = HttpService()
-  val qtService = QTService(httpService)
-  val revenueReport = RevenueReport(CapitalIQService(httpService), qtService)
+  private val httpService = HttpService()
+  private val qtService = QTService(httpService)
+  private val revenueReport = RevenueReport(CapitalIQService(httpService), qtService)
 
   def fromCli(args: Array[String]) =
     RevenueReportArgsParser.parse(args)