From 2bdaa2485ca9158e3a1f480d8f16c4d95638a188 Mon Sep 17 00:00:00 2001 From: Grzegorz Bielski <gbielski@quantemplate.com> Date: Thu, 15 Apr 2021 17:44:40 +0200 Subject: [PATCH 1/9] add yml parser for revenue report --- build.sbt | 4 +- .../com/quantemplate/capitaliq/Main.scala | 4 +- .../capitaliq/commands/ConfigDef.scala | 3 + .../commands/ConfigDefInterpreterCmd.scala | 55 ++++++++++++++++ .../capitaliq/commands/RevenueReportCmd.scala | 52 ---------------- .../commands/revenuereport/Config.scala | 14 +++++ .../{ => revenuereport}/RevenueReport.scala | 4 +- .../RevenueReportArgsParser.scala | 30 ++++++--- .../revenuereport/RevenueReportCmd.scala | 62 +++++++++++++++++++ .../RevenueReportConfigDef.scala | 58 +++++++++++++++++ .../commands/revenuereport/cmdName.scala | 3 + .../capitaliq/common/HttpService.scala | 19 ++++++ .../quantemplate/capitaliq/common/IO.scala | 8 +++ .../capitaliq/domain/CapitalIQ.scala | 8 +-- .../quantemplate/capitaliq/qt/QTService.scala | 19 +++++- data/revReport.yml | 17 +++++ 16 files changed, 292 insertions(+), 68 deletions(-) create mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDef.scala create mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala delete mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReportCmd.scala create mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala rename capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/{ => revenuereport}/RevenueReport.scala (97%) rename capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/{ => revenuereport}/RevenueReportArgsParser.scala (70%) create mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala create mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala create mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/cmdName.scala create mode 100644 capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala create mode 100644 data/revReport.yml diff --git a/build.sbt b/build.sbt index 30fe7d7..89bed78 100644 --- a/build.sbt +++ b/build.sbt @@ -25,13 +25,15 @@ lazy val capitaliq = (project in file("capitaliq")) // circe "io.circe" %% "circe-core" % CirceVersion, "io.circe" %% "circe-parser" % CirceVersion, + "io.circe" %% "circe-yaml" % "0.13.1", // misc "org.scalatest" %% "scalatest" % "3.1.0" % Test, "ch.qos.logback" % "logback-classic" % "1.2.3", "com.typesafe" % "config" % "1.4.1", "org.typelevel" %% "cats-core" % "2.4.2", "com.norbitltd" %% "spoiwo" % "1.7.0", - "com.github.scopt" %% "scopt" % "4.0.1" + "com.github.scopt" %% "scopt" % "4.0.1", + "com.lihaoyi" %% "os-lib" % "0.7.4" ).map(_.withDottyCompat(scalaVersion.value)), assembly / mainClass := Some("com.quantemplate.capitaliq.Main") ) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/Main.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/Main.scala index 4a9bbfa..2a1c646 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/Main.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/Main.scala @@ -3,12 +3,14 @@ package com.quantemplate.capitaliq import org.slf4j.LoggerFactory import com.quantemplate.capitaliq.commands.* +import com.quantemplate.capitaliq.commands.revenuereport.* object Main: lazy val logger = LoggerFactory.getLogger(getClass) def main(args: Array[String]): Unit = args match - case Array("generateRevenueReport", _*) => RevenueReportCmd.run(args) + case Array(`configDefInterpreterCmdName`, _*) => ConfigDefInterpreterCmd.fromCli(args) + case Array(`revenueReportCmdName`, _*) => RevenueReportCmd().fromCli(args) case _ => logger.error("Unsupported command") System.exit(1) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDef.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDef.scala new file mode 100644 index 0000000..6e3b2b8 --- /dev/null +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDef.scala @@ -0,0 +1,3 @@ +package com.quantemplate.capitaliq.commands + +trait ConfigDef diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala new file mode 100644 index 0000000..5ce3e92 --- /dev/null +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala @@ -0,0 +1,55 @@ +package com.quantemplate.capitaliq.commands + +import java.time.LocalDate +import io.circe.{ Encoder, Decoder, Json, DecodingFailure } +import io.circe.yaml.{parser as ymlParser} +import io.circe.syntax.given +import cats.syntax.apply.given +import cats.instances.either.given +import scopt.OParser + +import com.quantemplate.capitaliq.common.* +import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier +import com.quantemplate.capitaliq.domain.Identifiers +import com.quantemplate.capitaliq.commands.revenuereport.* + +object ConfigDefInterpreterCmd: + def fromCli(args: Array[String]) = ConfigArgsParser.parse(args).map { cliConfig => + + val ymlPath = getPath(cliConfig.path) + val yml = os.read(ymlPath) + + parseYml(yml).map { + case config: RevenueReportConfigDef => RevenueReportCmd().fromConfigFile(config, ymlPath) + } + } + +private def parseYml(yml: String) = + ymlParser.parse(yml).flatMap(_.as[ConfigDef]).toOption + +given Decoder[ConfigDef] = Decoder.instance[ConfigDef] { c => + c.get[String]("command").flatMap { + case configDefInterpreterCmdName => c.get[RevenueReportConfigDef]("params") + } +} + +val configDefInterpreterCmdName = "apply" + +object ConfigArgsParser: + case class CliConfig(path: String = "") + + def parse(args: Array[String]) = OParser.parse(parser, args, CliConfig()) + + private lazy val builder = OParser.builder[CliConfig] + private lazy val parser = + import builder.* + + OParser.sequence( + programName("capitaliq-qt integration"), + head("capitaliq-qt", "0.0.1"), + opt[String](configDefInterpreterCmdName) + .action((path, c) => c.copy(path = path)) + .required + .text("Runs the command described in a file at a given path") + ) + diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReportCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReportCmd.scala deleted file mode 100644 index a3616c3..0000000 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReportCmd.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.quantemplate.capitaliq.commands - -import java.util.Calendar -import java.time.ZoneId -import akka.actor.typed.ActorSystem -import akka.actor.typed.scaladsl.Behaviors -import scala.concurrent.ExecutionContext -import scala.util.{Failure, Success} - -import com.quantemplate.capitaliq.common.* -import com.quantemplate.capitaliq.domain.* -import com.quantemplate.capitaliq.qt.* - -object RevenueReportCmd: - def run(args: Array[String]) = - RevenueReportArgsParser.parse(args).map { config => - given Config = Config.load() - given sys: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "capitaliq") - given ExecutionContext = sys.executionContext - - val logger = sys.log - val httpService = HttpService() - val revenueReport = RevenueReport(CapitalIQService(httpService), QTService(httpService)) - - revenueReport - .generateSpreadSheet( - ids = config.identifiers - .map(Identifiers(_*)) - .getOrElse { - logger.info("Loading the Capital IQ identifiers from the STDIN") - - Identifiers.loadFromStdin() - }, - range = ( - fromCalendar(config.from), - fromCalendar(config.to) - ), - currency = config.currency, - orgId = config.orgId, - datasetId = config.datasetId - ).onComplete { - case Failure(e) => - logger.error("Failed to generate the revenue report: {}", e.toString) - Runtime.getRuntime.halt(1) - - case Success(_) => - Runtime.getRuntime.halt(0) - } - } - - private def fromCalendar(cal: Calendar) = - cal.getTime.toInstant.atZone(ZoneId.systemDefault).toLocalDate diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala new file mode 100644 index 0000000..0caef0b --- /dev/null +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala @@ -0,0 +1,14 @@ +package com.quantemplate.capitaliq.commands.revenuereport + +import java.time.LocalDate + +import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier + +case class CmdConfig( + orgId: String, + datasetId: String, + from: LocalDate, + to: LocalDate, + currency: String, + identifiers: Option[Vector[Identifier]] +) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReport.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReport.scala similarity index 97% rename from capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReport.scala rename to capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReport.scala index 6bd98ec..6a7cbd8 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReport.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReport.scala @@ -1,4 +1,4 @@ -package com.quantemplate.capitaliq.commands +package com.quantemplate.capitaliq.commands.revenuereport import java.time.* import java.time.format.DateTimeFormatter @@ -51,7 +51,7 @@ class RevenueReport(capitalIqService: CapitalIQService, qtService: QTService)(us range: (LocalDate, LocalDate), currency: String ): Future[ReportRows] = - import range.{ _1 => start, _2 => end } + import range.{ _1 as start, _2 as end } val periodType = "IQ_FY" back (end.getYear - start.getYear) val asOfDate = end.format(DateTimeFormatter.ofPattern("MM/dd/yyyy")).some diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReportArgsParser.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportArgsParser.scala similarity index 70% rename from capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReportArgsParser.scala rename to capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportArgsParser.scala index 46b3b72..858b7d2 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/RevenueReportArgsParser.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportArgsParser.scala @@ -1,29 +1,45 @@ -package com.quantemplate.capitaliq.commands +package com.quantemplate.capitaliq.commands.revenuereport import java.util.Calendar +import java.time.ZoneId import scopt.OParser +import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier +import com.quantemplate.capitaliq.domain.Identifiers + object RevenueReportArgsParser: // OParser requires default args - case class Config( + case class CliConfig( orgId: String = "", datasetId: String = "", from: Calendar = Calendar.getInstance, to: Calendar = Calendar.getInstance, currency: String = "", identifiers: Option[Vector[String]] = None - ) - - def parse(args: Array[String]) = OParser.parse(parser, args, Config()) + ): + def toCmdConfig = CmdConfig( + orgId = orgId, + datasetId = datasetId, + from = fromCalendar(from), + to = fromCalendar(to), + currency = currency, + identifiers = identifiers.map(Identifiers(_*)) + ) + + private def fromCalendar(cal: Calendar) = + cal.getTime.toInstant.atZone(ZoneId.systemDefault).toLocalDate + + + def parse(args: Array[String]) = OParser.parse(parser, args, CliConfig()) - private lazy val builder = OParser.builder[Config] + private lazy val builder = OParser.builder[CliConfig] private lazy val parser = import builder.* OParser.sequence( programName("capitaliq-qt integration"), head("capitaliq-qt", "0.0.1"), - cmd("generateRevenueReport") + cmd(revenueReportCmdName) .children( opt[String]("orgId") .action((id, c) => c.copy(orgId = id)) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala new file mode 100644 index 0000000..9723b98 --- /dev/null +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala @@ -0,0 +1,62 @@ +package com.quantemplate.capitaliq.commands.revenuereport + +import akka.actor.typed.ActorSystem +import akka.actor.typed.scaladsl.Behaviors +import scala.concurrent.ExecutionContext +import scala.util.{Failure, Success} + +import com.quantemplate.capitaliq.common.* +import com.quantemplate.capitaliq.domain.* +import com.quantemplate.capitaliq.qt.* + + +class RevenueReportCmd: + given Config = Config.load() + given sys: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "capitaliq") + given ExecutionContext = sys.executionContext + + val logger = sys.log + val httpService = HttpService() + val qtService = QTService(httpService) + val revenueReport = RevenueReport(CapitalIQService(httpService), qtService) + + def fromCli(args: Array[String]) = + RevenueReportArgsParser.parse(args).map(_.toCmdConfig).map(run) + + def fromConfigFile(config: RevenueReportConfigDef, configPath: os.Path) = + val inlineIds = config.identifiers.flatMap(_.inline) + val localIds = config.identifiers.flatMap(_.local).map { rawPath => + Identifiers(os.read.lines(getPath(rawPath, configPath)): _*) + } + + // todo: load the ids from QT dataset + val remoteIds: Option[Vector[CapitalIQ.Identifier]] = ??? + + val identifiers = (inlineIds ++ localIds ++ remoteIds).reduceOption(_ ++ _) + + run(config.toCmdConfig(identifiers)) + + private def run(config: CmdConfig) = + revenueReport + .generateSpreadSheet( + ids = config.identifiers.getOrElse { + logger.info("Loading the Capital IQ identifiers from the STDIN") + + Identifiers.loadFromStdin() + }, + range = ( + config.from, + config.to + ), + currency = config.currency, + orgId = config.orgId, + datasetId = config.datasetId + ).onComplete { + case Failure(e) => + logger.error("Failed to generate the revenue report: {}", e.toString) + Runtime.getRuntime.halt(1) + + case Success(_) => + Runtime.getRuntime.halt(0) + } + diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala new file mode 100644 index 0000000..5595972 --- /dev/null +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala @@ -0,0 +1,58 @@ +package com.quantemplate.capitaliq.commands.revenuereport + +import java.nio.file.Paths +import java.time.LocalDate +import io.circe.{ Encoder, Decoder, Json, DecodingFailure } +import io.circe.yaml.{parser as ymlParser} +import io.circe.syntax.given +import cats.syntax.apply.given +import cats.instances.either.given +import scopt.OParser + +import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier +import com.quantemplate.capitaliq.commands.ConfigDef + +case class RevenueReportConfigDef( + orgId: String, + datasetId: String, + currency: String, + from: LocalDate, + to: LocalDate, + identifiers: Option[RevenueReportConfigDef.Identifiers] +) extends ConfigDef: + def toCmdConfig(loadedIds: Option[Vector[Identifier]]) = CmdConfig( + orgId = orgId, + datasetId = datasetId, + from = from, + to = to, + currency = currency, + identifiers = loadedIds + ) + +object RevenueReportConfigDef: + given Decoder[RevenueReportConfigDef] = Decoder.instance[RevenueReportConfigDef] { c => + ( + c.get[String]("orgId"), + c.get[String]("datasetId"), + c.get[String]("currency"), + c.get[LocalDate]("from"), + c.get[LocalDate]("to"), + c.get[Option[RevenueReportConfigDef.Identifiers]]("identifiers") + ).mapN(RevenueReportConfigDef.apply) + } + + case class Identifiers( + local: Option[String], + dataset: Option[String], + inline: Option[Vector[Identifier]] + ) + + object Identifiers: + given Decoder[Identifiers] = Decoder.instance[Identifiers] { c => + ( + c.get[Option[String]]("local"), + c.get[Option[String]]("dataset"), + c.get[Option[Vector[Identifier]]]("inline") + ).mapN(Identifiers.apply) + } + \ No newline at end of file diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/cmdName.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/cmdName.scala new file mode 100644 index 0000000..53f4b2b --- /dev/null +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/cmdName.scala @@ -0,0 +1,3 @@ +package com.quantemplate.capitaliq.commands.revenuereport + +val revenueReportCmdName = "generateRevenueReport" diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala index f91e7a2..2fe542d 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala @@ -17,6 +17,16 @@ class HttpService(using system: ActorSystem[_]): given ExecutionContext = system.executionContext + def get[B: Decoder]( + endpoint: String, + auth: Option[Authorization] + ): Future[B] = + for + res <- GET(endpoint, auth) + body <- getResponseBody(res) + result <- Unmarshal(body).to[B] + yield result + def post[B: Decoder]( endpoint: String, req: RequestEntity, @@ -63,5 +73,14 @@ class HttpService(using system: ActorSystem[_]): ) ) + private def GET(endpoint: String, auth: Option[Authorization]) = + Http().singleRequest( + HttpRequest( + method = HttpMethods.GET, + uri = endpoint, + headers = auth.map(Seq(_)).getOrElse(Seq.empty) + ) + ) + object HttpService: case class UploadResponse(statusCode: Int, body: String) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala new file mode 100644 index 0000000..8072c38 --- /dev/null +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala @@ -0,0 +1,8 @@ +package com.quantemplate.capitaliq.common + +import java.nio.file.Paths + +def getPath(rawPath: String, base: os.Path = os.pwd) = + if Paths.get(rawPath).isAbsolute + then os.Path(rawPath) + else os.Path(rawPath, base = os.pwd) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala index 09bd288..56102c3 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala @@ -1,14 +1,14 @@ package com.quantemplate.capitaliq.domain -import io.circe.{ Encoder, Decoder, Json, DecodingFailure } -import io.circe.syntax.* -import cats.syntax.traverse.{*, given} -import cats.instances.vector.{*, given} +import io.circe.{ Encoder, Decoder, Json } +import io.circe.syntax.given +import cats.syntax.traverse.given object CapitalIQ: opaque type Identifier = String object Identifier: def apply(s: String): Identifier = s + given Decoder[Identifier] = Decoder.decodeString.map(apply(_)) extension (i: Identifier) def unwrap: String = i diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala index dfafab1..73d2213 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala @@ -15,8 +15,12 @@ class QTService(httpService: HttpService)(using ec: ExecutionContext, conf: Conf lazy val logger = LoggerFactory.getLogger(getClass) + def datasetEndpoint(orgId: String, datasetId: String) = + s"${conf.quantemplate.api.baseUrl}/v1/organisations/$orgId/datasets/$datasetId" + + def uploadDataset(view: View, orgId: String, datasetId: String) = - val endpoint = s"${conf.quantemplate.api.baseUrl}/v1/organisations/$orgId/datasets/$datasetId" + val endpoint = datasetEndpoint(orgId, datasetId) for tokenRes <- getToken() @@ -33,6 +37,18 @@ class QTService(httpService: HttpService)(using ec: ExecutionContext, conf: Conf case other => throw DatasetUploadError(s"${other.statusCode}: ${other.body}") + def downloadADataset(orgId: String, datasetId: String) = + val endpoint = datasetEndpoint(orgId, datasetId) + + for + tokenRes <- getToken() + res <- httpService.get[String]( + endpoint, + Authorization(OAuth2BearerToken(tokenRes.accessToken)).some + ) + yield () + + def getToken(): Future[TokenResponse] = httpService.post[TokenResponse]( conf.quantemplate.auth.endpoint, @@ -43,6 +59,7 @@ class QTService(httpService: HttpService)(using ec: ExecutionContext, conf: Conf ).toEntity, None ) + object QTService: case class TokenResponse(accessToken: String) diff --git a/data/revReport.yml b/data/revReport.yml new file mode 100644 index 0000000..2493532 --- /dev/null +++ b/data/revReport.yml @@ -0,0 +1,17 @@ +command: generateRevenueReport +params: + orgId: c-my-small-insuranc-ltdzfd + datasetId: d-e4tf3yyxerabcvicidv5oyey + currency: USD + from: 1988-12-31 + to: 2018-12-31 + + # should handle loading identifiers from local file system, QT dataset, inline ones, STDIN + identifiers: + local: ../capitaliq-identifiers.txt + dataset: d-e4tf3yyxerabcvicidv5oyey + inline: + - IQ121238 + - IQ598807 + - IQ723344 + - IQ956057 From 4f783de94ca151ee75cabedff0b31f55b5bb9070 Mon Sep 17 00:00:00 2001 From: Grzegorz Bielski <gbielski@quantemplate.com> Date: Fri, 16 Apr 2021 16:13:07 +0200 Subject: [PATCH 2/9] load local, remote and inline identifiers --- .../commands/ConfigDefInterpreterCmd.scala | 36 +- .../commands/revenuereport/Config.scala | 2 +- .../RevenueReportArgsParser.scala | 7 +- .../revenuereport/RevenueReportCmd.scala | 63 +- .../RevenueReportConfigDef.scala | 7 +- .../capitaliq/common/HttpService.scala | 19 +- .../quantemplate/capitaliq/common/IO.scala | 2 +- .../capitaliq/domain/CapitalIQ.scala | 12 +- .../capitaliq/domain/Errors.scala | 2 +- .../capitaliq/domain/Identifiers.scala | 24 +- .../quantemplate/capitaliq/qt/QTService.scala | 14 +- data/capitaliq-identifiers.csv | 850 ++++++++++++++++++ data/revReport.yml | 7 +- 13 files changed, 974 insertions(+), 71 deletions(-) create mode 100644 data/capitaliq-identifiers.csv diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala index 5ce3e92..dc6ba3f 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala @@ -3,10 +3,9 @@ package com.quantemplate.capitaliq.commands import java.time.LocalDate import io.circe.{ Encoder, Decoder, Json, DecodingFailure } import io.circe.yaml.{parser as ymlParser} -import io.circe.syntax.given -import cats.syntax.apply.given -import cats.instances.either.given +import org.slf4j.LoggerFactory import scopt.OParser +import cats.syntax.bifunctor.given import com.quantemplate.capitaliq.common.* import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier @@ -14,18 +13,23 @@ import com.quantemplate.capitaliq.domain.Identifiers import com.quantemplate.capitaliq.commands.revenuereport.* object ConfigDefInterpreterCmd: - def fromCli(args: Array[String]) = ConfigArgsParser.parse(args).map { cliConfig => + lazy val logger = LoggerFactory.getLogger(getClass) - val ymlPath = getPath(cliConfig.path) - val yml = os.read(ymlPath) + def fromCli(args: Array[String]) = ConfigArgsParser.parse(args).map { cliConfig => + val configPath = getPath(cliConfig.path) - parseYml(yml).map { - case config: RevenueReportConfigDef => RevenueReportCmd().fromConfigFile(config, ymlPath) - } + loadConfig(configPath).bimap( + err => logger.error("Could not parse the config file", err), + { + case config: RevenueReportConfigDef => RevenueReportCmd().fromConfigFile(config, configPath) + } + ) } -private def parseYml(yml: String) = - ymlParser.parse(yml).flatMap(_.as[ConfigDef]).toOption +private def loadConfig(path: os.Path) = + val yml = os.read(path) + + ymlParser.parse(yml).flatMap(_.as[ConfigDef]) given Decoder[ConfigDef] = Decoder.instance[ConfigDef] { c => c.get[String]("command").flatMap { @@ -47,9 +51,13 @@ object ConfigArgsParser: OParser.sequence( programName("capitaliq-qt integration"), head("capitaliq-qt", "0.0.1"), - opt[String](configDefInterpreterCmdName) - .action((path, c) => c.copy(path = path)) - .required + cmd(configDefInterpreterCmdName) .text("Runs the command described in a file at a given path") + .children( + arg[String]("<config file>...") + .action((path, c) => c.copy(path = path)) + .unbounded + .required + ) ) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala index 0caef0b..36fd06c 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/Config.scala @@ -10,5 +10,5 @@ case class CmdConfig( from: LocalDate, to: LocalDate, currency: String, - identifiers: Option[Vector[Identifier]] + identifiers: Vector[Identifier] ) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportArgsParser.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportArgsParser.scala index 858b7d2..609a1a4 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportArgsParser.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportArgsParser.scala @@ -8,7 +8,6 @@ import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier import com.quantemplate.capitaliq.domain.Identifiers object RevenueReportArgsParser: - // OParser requires default args case class CliConfig( orgId: String = "", datasetId: String = "", @@ -17,19 +16,18 @@ object RevenueReportArgsParser: currency: String = "", identifiers: Option[Vector[String]] = None ): - def toCmdConfig = CmdConfig( + def toCmdConfig(fallbackIds: => Vector[Identifier]) = CmdConfig( orgId = orgId, datasetId = datasetId, from = fromCalendar(from), to = fromCalendar(to), currency = currency, - identifiers = identifiers.map(Identifiers(_*)) + identifiers = identifiers.map(Identifiers(_*)).getOrElse(fallbackIds) ) private def fromCalendar(cal: Calendar) = cal.getTime.toInstant.atZone(ZoneId.systemDefault).toLocalDate - def parse(args: Array[String]) = OParser.parse(parser, args, CliConfig()) private lazy val builder = OParser.builder[CliConfig] @@ -40,6 +38,7 @@ object RevenueReportArgsParser: programName("capitaliq-qt integration"), head("capitaliq-qt", "0.0.1"), cmd(revenueReportCmdName) + .text("Generates a revenue report from CapitalIQ data and uploads it to the Quantemplate dataset") .children( opt[String]("orgId") .action((id, c) => c.copy(orgId = id)) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala index 9723b98..7a05f43 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala @@ -4,6 +4,9 @@ import akka.actor.typed.ActorSystem import akka.actor.typed.scaladsl.Behaviors import scala.concurrent.ExecutionContext import scala.util.{Failure, Success} +import scala.concurrent.Future +import org.slf4j.LoggerFactory +import cats.syntax.traverse.given import com.quantemplate.capitaliq.common.* import com.quantemplate.capitaliq.domain.* @@ -15,35 +18,61 @@ class RevenueReportCmd: given sys: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "capitaliq") given ExecutionContext = sys.executionContext - val logger = sys.log + lazy val logger = LoggerFactory.getLogger(getClass) + val httpService = HttpService() val qtService = QTService(httpService) val revenueReport = RevenueReport(CapitalIQService(httpService), qtService) def fromCli(args: Array[String]) = - RevenueReportArgsParser.parse(args).map(_.toCmdConfig).map(run) + RevenueReportArgsParser.parse(args) + .map(_.toCmdConfig(loadIdentifiersFromStdin())) + .map(run) + + def fromConfigFile(config: RevenueReportConfigDef, configPath: os.Path) = + loadIdentifiersFromConfig(config, configPath) + .map(_.getOrElse(loadIdentifiersFromStdin())) + .map(config.toCmdConfig(_)) + .map(run) - def fromConfigFile(config: RevenueReportConfigDef, configPath: os.Path) = - val inlineIds = config.identifiers.flatMap(_.inline) - val localIds = config.identifiers.flatMap(_.local).map { rawPath => - Identifiers(os.read.lines(getPath(rawPath, configPath)): _*) - } + private def loadIdentifiersFromStdin() = { + logger.info("Loading the Capital IQ identifiers from the STDIN") - // todo: load the ids from QT dataset - val remoteIds: Option[Vector[CapitalIQ.Identifier]] = ??? + Identifiers.loadFromStdin() + } - val identifiers = (inlineIds ++ localIds ++ remoteIds).reduceOption(_ ++ _) + private def loadIdentifiersFromConfig(config: RevenueReportConfigDef, configPath: os.Path) = + config.identifiers + .flatMap(_.dataset) + .map(loadIdentifiersFromDataset(config.orgId, _)) + .sequence + .map { remoteIds => + val inlineIds = config.identifiers.flatMap(_.inline) + val localIds = config.identifiers.flatMap(_.local).map { rawPath => + Identifiers( + os.read.lines(getPath(rawPath, configPath / os.up)): _* + ) + } + + (inlineIds ++ localIds ++ remoteIds).reduceOption(_ ++ _) + } - run(config.toCmdConfig(identifiers)) + private def loadIdentifiersFromDataset(orgId: String, datasetId: String) = + qtService.downloadDataset(orgId, datasetId) + .map(Identifiers.loadFromCsvString) + .recover { + case e: Throwable => + logger.warn("Could not load the identifiers from the remote dataset") + println(("err", e)) + e.printStackTrace() - private def run(config: CmdConfig) = + Vector.empty[CapitalIQ.Identifier] + } + + private def run(config: CmdConfig) = revenueReport .generateSpreadSheet( - ids = config.identifiers.getOrElse { - logger.info("Loading the Capital IQ identifiers from the STDIN") - - Identifiers.loadFromStdin() - }, + ids = config.identifiers.distinct, range = ( config.from, config.to diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala index 5595972..03f11f4 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala @@ -1,13 +1,10 @@ package com.quantemplate.capitaliq.commands.revenuereport -import java.nio.file.Paths import java.time.LocalDate -import io.circe.{ Encoder, Decoder, Json, DecodingFailure } +import io.circe.Decoder import io.circe.yaml.{parser as ymlParser} import io.circe.syntax.given import cats.syntax.apply.given -import cats.instances.either.given -import scopt.OParser import com.quantemplate.capitaliq.domain.CapitalIQ.Identifier import com.quantemplate.capitaliq.commands.ConfigDef @@ -20,7 +17,7 @@ case class RevenueReportConfigDef( to: LocalDate, identifiers: Option[RevenueReportConfigDef.Identifiers] ) extends ConfigDef: - def toCmdConfig(loadedIds: Option[Vector[Identifier]]) = CmdConfig( + def toCmdConfig(loadedIds: => Vector[Identifier]) = CmdConfig( orgId = orgId, datasetId = datasetId, from = from, diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala index 2fe542d..70fb808 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala @@ -12,20 +12,21 @@ import akka.util.ByteString import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.* import io.circe.{Encoder, Decoder} +// TODO: improve error handling by wrapping responses in Either / Try class HttpService(using system: ActorSystem[_]): import HttpService.* given ExecutionContext = system.executionContext - def get[B: Decoder]( + def get( endpoint: String, auth: Option[Authorization] - ): Future[B] = + ): Future[String] = for res <- GET(endpoint, auth) body <- getResponseBody(res) - result <- Unmarshal(body).to[B] - yield result + // result <- Unmarshal(body).to[B] + yield body.utf8String def post[B: Decoder]( endpoint: String, @@ -50,15 +51,15 @@ class HttpService(using system: ActorSystem[_]): result <- Unmarshal(body).to[B] yield result - def upload( + def post( endpoint: String, bytes: Array[Byte], - auth: Option[Authorization] = None - ): Future[UploadResponse] = + auth: Option[Authorization] + ): Future[Response] = for res <- POST(endpoint, HttpEntity(bytes), auth) body <- getResponseBody(res) - yield HttpService.UploadResponse(res.status.intValue, body.utf8String) + yield HttpService.Response(res.status.intValue, body.utf8String) private def getResponseBody(res: HttpResponse) = res.entity.dataBytes.runFold(ByteString.empty)(_ ++ _) @@ -83,4 +84,4 @@ class HttpService(using system: ActorSystem[_]): ) object HttpService: - case class UploadResponse(statusCode: Int, body: String) + case class Response(statusCode: Int, body: String) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala index 8072c38..ab47388 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala @@ -5,4 +5,4 @@ import java.nio.file.Paths def getPath(rawPath: String, base: os.Path = os.pwd) = if Paths.get(rawPath).isAbsolute then os.Path(rawPath) - else os.Path(rawPath, base = os.pwd) + else os.Path(rawPath, base = base) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala index 56102c3..73faa5a 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala @@ -3,12 +3,22 @@ package com.quantemplate.capitaliq.domain import io.circe.{ Encoder, Decoder, Json } import io.circe.syntax.given import cats.syntax.traverse.given +import cats.syntax.applicative.given object CapitalIQ: opaque type Identifier = String object Identifier: + val prefix = "IQ" + def apply(s: String): Identifier = s - given Decoder[Identifier] = Decoder.decodeString.map(apply(_)) + + // this might use some improved validation + def isValid(s: String) = s.startsWith(prefix) + + given Decoder[Identifier] = Decoder.decodeString + .ensure(isValid, "Not a valid CapitalIQ identifier") + .map(Identifier(_)) + extension (i: Identifier) def unwrap: String = i diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Errors.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Errors.scala index e1eabca..c94b4b6 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Errors.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Errors.scala @@ -12,7 +12,7 @@ case class UnrecognizedServiceError(msg: String) extends DomainError: override def toString = s"Unrecognized Capital IQ API error: $msg" case class MnemonicsError(errors: Seq[DomainError]) extends DomainError: - override def toString = errors.mkString + override def toString = errors.mkString(",") case object DailyMnemonicLimitReachedError extends DomainError: override def toString = s"Reached daily limit of 24000 requested mnemonics" diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala index 7f89195..2901348 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala @@ -2,14 +2,24 @@ package com.quantemplate.capitaliq.domain import scala.io.Source -object Identifiers: - val commentToken = "//" +import CapitalIQ.Identifier +object Identifiers: def apply(ids: String*) = - ids.map(CapitalIQ.Identifier(_)).toVector + ids + .filter(Identifier.isValid) + .map(Identifier(_)) + .toVector + + // (!) assuming the dataset uses ',' as a separator + // and the identifiers are in the first column + def loadFromCsvString(str: String) = + Identifiers( + str + .split('\n') + .toVector + .map(_.split(',').headOption.getOrElse("")): _* + ) def loadFromStdin() = - Source.stdin.getLines - .filter(!_.startsWith(commentToken)) - .map(CapitalIQ.Identifier(_)) - .toVector + Identifiers(Source.stdin.getLines.toVector: _*) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala index 73d2213..d0f7ddc 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/qt/QTService.scala @@ -15,7 +15,7 @@ class QTService(httpService: HttpService)(using ec: ExecutionContext, conf: Conf lazy val logger = LoggerFactory.getLogger(getClass) - def datasetEndpoint(orgId: String, datasetId: String) = + private def datasetEndpoint(orgId: String, datasetId: String) = s"${conf.quantemplate.api.baseUrl}/v1/organisations/$orgId/datasets/$datasetId" @@ -24,29 +24,29 @@ class QTService(httpService: HttpService)(using ec: ExecutionContext, conf: Conf for tokenRes <- getToken() - res <- httpService.upload( + res <- httpService.post( endpoint, view.toBytes, Authorization(OAuth2BearerToken(tokenRes.accessToken)).some ) yield res match - case HttpService.UploadResponse(200, _) => () - case HttpService.UploadResponse(403, _) => + case HttpService.Response(200, _) => () + case HttpService.Response(403, _) => throw DatasetUploadError("403: Did you share the dataset with whole org?") case other => throw DatasetUploadError(s"${other.statusCode}: ${other.body}") - def downloadADataset(orgId: String, datasetId: String) = + def downloadDataset(orgId: String, datasetId: String) = val endpoint = datasetEndpoint(orgId, datasetId) for tokenRes <- getToken() - res <- httpService.get[String]( + res <- httpService.get( endpoint, Authorization(OAuth2BearerToken(tokenRes.accessToken)).some ) - yield () + yield res def getToken(): Future[TokenResponse] = diff --git a/data/capitaliq-identifiers.csv b/data/capitaliq-identifiers.csv new file mode 100644 index 0000000..6d5b48c --- /dev/null +++ b/data/capitaliq-identifiers.csv @@ -0,0 +1,850 @@ +identifiers, +IQ121238, +IQ598807, +IQ723344, +IQ956057, +IQ250388, +IQ5471113, +IQ876215, +IQ162456, +IQ8201008, +IQ22517019, +IQ22663157, +IQ875826, +IQ992287, +IQ8248315, +IQ20551569, +IQ874488, +IQ341546, +IQ876731, +IQ8210142, +IQ1082017, +IQ698550, +IQ876955, +IQ5467549, +IQ8475394, +IQ882901, +IQ7605761, +IQ875328, +IQ36739404, +IQ878160, +IQ2969167, +IQ2159401, +IQ877802, +IQ3680446, +IQ5477675, +IQ303566889, +IQ8125089, +IQ882957, +IQ4512131, +IQ883670, +IQ295539, +IQ22510313, +IQ4026111, +IQ22729379, +IQ26471214, +IQ8475467, +IQ37612389, +IQ4481454, +IQ8474885, +IQ215098026, +IQ2482281, +IQ5491822, +IQ27515847, +IQ5478545, +IQ883047, +IQ12732661, +IQ876952, +IQ3561426, +IQ874863, +IQ240824094, +IQ1051073, +IQ199668, +IQ20742105, +IQ5470931, +IQ874600, +IQ41356562, +IQ22533749, +IQ875917, +IQ250307, +IQ4497069, +IQ9235665, +IQ966841, +IQ883901, +IQ20970264, +IQ965915, +IQ8221280, +IQ5495596, +IQ8105207, +IQ1542889, +IQ882762, +IQ1882861, +IQ13335251, +IQ13483916, +IQ12933390, +IQ22039549, +IQ8952000, +IQ8237245, +IQ8642193, +IQ26972175, +IQ259879432, +IQ8292712, +IQ225687232, +IQ877323, +IQ5476832, +IQ22547601, +IQ7884005, +IQ26326530, +IQ22134956, +IQ5525514, +IQ5566600, +IQ10245092, +IQ318898, +IQ22548905, +IQ877527, +IQ307183867, +IQ878130, +IQ5569241, +IQ12205230, +IQ20527977, +IQ5473193, +IQ8982543, +IQ26934166, +IQ5604059, +IQ5686144, +IQ30921662, +IQ10639513, +IQ875073, +IQ53474567, +IQ32671961, +IQ873776, +IQ5533523, +IQ615970073, +IQ9514473, +IQ59372417, +IQ32777643, +IQ261155221, +IQ11519044, +IQ26841815, +IQ5569256, +IQ11461103, +IQ20394473, +IQ20457561, +IQ138540308, +IQ20357747, +IQ7683448, +IQ310010418, +IQ245593332, +IQ9450216, +IQ38010476, +IQ8751533, +IQ224448138, +IQ20929816, +IQ28526399, +IQ21790041, +IQ879281, +IQ22709629, +IQ233447435, +IQ9003966, +IQ22830143, +IQ53474451, +IQ20644228, +IQ6462568, +IQ128394122, +IQ5554951, +IQ5525485, +IQ10580089, +IQ20406261, +IQ20334327, +IQ32879143, +IQ59290204, +IQ35622138, +IQ22530726, +IQ2395875, +IQ9285238, +IQ20968703, +IQ20411372, +IQ36066453, +IQ20405257, +IQ41542793, +IQ6499589, +IQ24351707, +IQ254768850, +IQ35045462, +IQ876389, +IQ252124, +IQ699678, +IQ20376386, +IQ30777099, +IQ36070234, +IQ20442970, +IQ5705453, +IQ20357911, +IQ8951913, +IQ20875991, +IQ34360884, +IQ45364909, +IQ20439959, +IQ419608830, +IQ20388152, +IQ142687475, +IQ25603857, +IQ20401153, +IQ44731152, +IQ9638335, +IQ880319, +IQ37530273, +IQ8237517, +IQ45364901, +IQ9104794, +IQ118758025, +IQ23136532, +IQ20385336, +IQ47019555, +IQ280102994, +IQ6182739, +IQ24389424, +IQ98911479, +IQ118529610, +IQ34360875, +IQ6096740, +IQ206594311, +IQ20351433, +IQ20387499, +IQ20357133, +IQ25853564, +IQ884289, +IQ26840491, +IQ9446689, +IQ52657639, +IQ22817884, +IQ105309130, +IQ36824712, +IQ20342548, +IQ41542810, +IQ8863946, +IQ26425108, +IQ9317591, +IQ9648039, +IQ5435672, +IQ9898706, +IQ4509080, +IQ20392649, +IQ25601530, +IQ11251263, +IQ13338102, +IQ20333219, +IQ23956851, +IQ9383541, +IQ20928564, +IQ33059622, +IQ10837576, +IQ39420520, +IQ49756433, +IQ20386271, +IQ20387422, +IQ22519534, +IQ13389347, +IQ9616853, +IQ582343023, +IQ47286427, +IQ212327509, +IQ30585041, +IQ11354999, +IQ26539734, +IQ129791710, +IQ54412624, +IQ98910795, +IQ11459490, +IQ30746047, +IQ4493353, +IQ36070207, +IQ36218630, +IQ20356961, +IQ233903518, +IQ142039532, +IQ47938242, +IQ20388408, +IQ105289664, +IQ30612714, +IQ13483021, +IQ5449547, +IQ5514776, +IQ53474509, +IQ34430293, +IQ109965608, +IQ30468440, +IQ9648367, +IQ59676977, +IQ28506082, +IQ29785842, +IQ9648084, +IQ38928712, +IQ49943055, +IQ20385355, +IQ51446644, +IQ47285586, +IQ250801799, +IQ20406066, +IQ30746764, +IQ20440724, +IQ20406489, +IQ46949286, +IQ49273432, +IQ9648044, +IQ208360255, +IQ36308773, +IQ20376809, +IQ20384809, +IQ10167110, +IQ47733696, +IQ20379938, +IQ34360936, +IQ9265738, +IQ20387321, +IQ9264708, +IQ20405282, +IQ30446153, +IQ7958370, +IQ30589185, +IQ12004130, +IQ107792708, +IQ20387682, +IQ30864618, +IQ20384756, +IQ142681075, +IQ9663375, +IQ29226678, +IQ104919719, +IQ20357359, +IQ20406865, +IQ879972, +IQ117920629, +IQ83672799, +IQ3448881, +IQ9648086, +IQ11311924, +IQ23019707, +IQ217249500, +IQ20399470, +IQ225000580, +IQ20333227, +IQ236534039, +IQ28093936, +IQ321937608, +IQ30445681, +IQ13199091, +IQ552051031, +IQ22249494, +IQ9648296, +IQ21927946, +IQ317398442, +IQ22453364, +IQ11900969, +IQ83586150, +IQ20405278, +IQ34579314, +IQ138470937, +IQ4462958, +IQ112208069, +IQ46950019, +IQ20823232, +IQ20376805, +IQ30607633, +IQ20356260, +IQ27142992, +IQ22929324, +IQ46910621, +IQ416203171, +IQ12913444, +IQ20407950, +IQ42114936, +IQ29383141, +IQ106886296, +IQ114576425, +IQ84209473, +IQ31093, +IQ169008, +IQ250309, +IQ310818, +IQ632483, +IQ678651, +IQ722875, +IQ846304, +IQ872445, +IQ874020, +IQ875292, +IQ879079, +IQ880335, +IQ881280, +IQ881579, +IQ918690, +IQ973824, +IQ1028109, +IQ1056331, +IQ1511109, +IQ1515993, +IQ1529888, +IQ1549703, +IQ2745726, +IQ2759623, +IQ2897241, +IQ2994821, +IQ3649650, +IQ4090862, +IQ4096555, +IQ4163031, +IQ4165794, +IQ4168338, +IQ4183677, +IQ4205936, +IQ4210427, +IQ4217488, +IQ4222324, +IQ4224615, +IQ4249211, +IQ4251241, +IQ4279638, +IQ4303310, +IQ4332750, +//IQ4347511 invalid, +IQ4460048, +IQ4472553, +IQ4481550, +IQ4494248, +IQ4497162, +IQ4559043, +IQ4561438, +IQ4586814, +IQ4662943, +IQ4717183, +IQ4730735, +IQ4762761, +IQ4772278, +IQ5068910, +IQ5375495, +IQ5383311, +IQ5410293, +IQ5423523, +IQ5466287, +IQ5477916, +IQ5484075, +IQ5491169, +IQ5496125, +IQ5507600, +IQ5510525, +IQ5525297, +IQ5538187, +IQ5538747, +IQ5542158, +IQ5547143, +IQ5547613, +IQ5554373, +IQ5569286, +IQ5579426, +IQ5647001, +IQ5704985, +IQ5718690, +IQ5723278, +IQ5726675, +IQ5774517, +IQ5885638, +IQ5901538, +IQ5924809, +IQ6139502, +IQ6439926, +IQ6463183, +IQ6469667, +IQ6475664, +IQ6491233, +IQ6564312, +IQ6565183, +IQ6613221, +IQ6688088, +IQ6699370, +IQ6700017, +IQ6812156, +IQ6819792, +IQ6921281, +IQ6948694, +IQ6949267, +IQ7603981, +IQ7918016, +IQ7960416, +IQ8024164, +IQ8100801, +IQ8101330, +IQ8142882, +IQ8190935, +IQ8264274, +IQ8359879, +IQ9377948, +IQ9450267, +IQ9460690, +IQ9461042, +IQ9550032, +IQ9560354, +IQ9587298, +IQ9638878, +IQ9794004, +IQ9899048, +IQ9916947, +IQ10052670, +IQ11212063, +IQ11323005, +IQ11712739, +IQ12058656, +IQ12455886, +IQ12456448, +IQ12466795, +IQ12784598, +IQ12785981, +IQ12868386, +IQ12913479, +IQ12932470, +IQ13011212, +IQ13154195, +IQ13408154, +IQ13610052, +IQ20417852, +IQ20475401, +IQ20551258, +IQ20670757, +IQ20703772, +IQ20724600, +IQ20757724, +IQ20784162, +IQ20882165, +IQ20986395, +IQ21813186, +IQ21854837, +IQ21935579, +IQ22095148, +IQ22239048, +IQ22383525, +IQ22383606, +IQ22431794, +IQ22498272, +IQ22511361, +IQ22515611, +IQ22515844, +IQ22515964, +IQ22516125, +IQ22517048, +IQ22534319, +IQ22534512, +IQ22547744, +IQ22580222, +IQ22581143, +IQ22581472, +IQ22631649, +IQ22638720, +IQ22728611, +IQ22783832, +IQ22798804, +IQ22799264, +IQ22905959, +IQ22926403, +IQ22931810, +IQ23018346, +IQ23021162, +IQ23021452, +IQ23035756, +IQ23048633, +IQ23063990, +IQ23076171, +IQ23185132, +IQ23221640, +IQ23294481, +IQ23313835, +IQ23335868, +IQ23336901, +IQ23631262, +IQ23751882, +IQ23756872, +IQ23780053, +IQ23958224, +IQ24393771, +IQ24861410, +IQ25473977, +IQ25525075, +IQ25746444, +IQ25813840, +IQ25833297, +IQ25835239, +IQ25870800, +IQ25878705, +IQ25912594, +IQ25913389, +IQ25913460, +IQ25915583, +IQ25935856, +IQ26023649, +IQ26079324, +IQ26307837, +IQ26321017, +IQ26442394, +IQ26502798, +IQ26554716, +IQ26671175, +IQ26763923, +IQ26935576, +IQ26935957, +IQ26937924, +IQ26954558, +IQ26954965, +IQ26955671, +IQ26956943, +IQ26962223, +IQ26971203, +IQ26971884, +IQ26971963, +IQ26971979, +IQ26972138, +IQ26972384, +IQ26972531, +IQ26972634, +IQ26972671, +IQ27129412, +IQ27204765, +IQ27400044, +IQ27484055, +IQ27490077, +IQ27580696, +IQ27580703, +IQ27580715, +IQ27606887, +IQ27717292, +IQ27868361, +IQ27903956, +IQ28100928, +IQ28368577, +IQ28387463, +IQ28389647, +IQ28942515, +IQ29037395, +IQ29183054, +IQ29291881, +IQ29601835, +IQ30343267, +IQ30441904, +IQ30456609, +IQ30465921, +IQ30490816, +IQ30512591, +IQ30515896, +IQ30601911, +IQ30613257, +IQ30637238, +IQ30683814, +IQ30705424, +IQ30739286, +IQ30789560, +IQ30795690, +IQ30884638, +IQ30915913, +IQ30964685, +IQ31047931, +IQ31065816, +IQ31108425, +IQ31131565, +IQ31134863, +IQ31136645, +IQ31136770, +IQ32589022, +IQ32907389, +IQ33350617, +IQ33404477, +IQ33597720, +IQ34032519, +IQ34360580, +IQ34360903, +IQ34399111, +IQ34594700, +IQ34648845, +IQ34791159, +IQ34791294, +IQ35101458, +IQ35134316, +IQ35144642, +IQ35392019, +IQ35412230, +IQ35424160, +IQ35674202, +IQ35749702, +IQ35960436, +IQ36621656, +IQ36837622, +IQ37262203, +IQ37507853, +IQ37782812, +IQ38919390, +IQ38991075, +IQ39313436, +IQ39601472, +IQ39636404, +IQ39637638, +IQ39642860, +IQ40153601, +IQ40254139, +IQ40311435, +IQ40311458, +IQ40332961, +IQ40453340, +IQ41087044, +IQ42434220, +IQ42439675, +IQ42680296, +IQ42746052, +IQ43188556, +IQ44459504, +IQ44519103, +IQ45978451, +IQ46239777, +IQ46367675, +IQ46511782, +IQ46638310, +IQ46692499, +IQ46771171, +IQ46855213, +IQ46865535, +IQ46883648, +IQ46884369, +IQ46894216, +IQ46895330, +IQ46904183, +IQ46954509, +IQ46959086, +IQ47014971, +IQ47017185, +IQ47064062, +IQ47149593, +IQ47439243, +IQ47468586, +IQ47475952, +IQ47691832, +IQ47838075, +IQ47938173, +IQ48970871, +IQ49060929, +IQ49380472, +IQ49410153, +IQ49411434, +IQ49444150, +IQ49516174, +IQ49518798, +IQ49545850, +IQ49548668, +IQ49650376, +IQ49685718, +IQ50015159, +IQ50842667, +IQ50876655, +IQ51614649, +IQ52034375, +IQ52037418, +IQ52139112, +IQ52292994, +IQ52324618, +IQ52350790, +IQ52414484, +IQ52514348, +IQ52715887, +IQ53039417, +IQ54075352, +IQ54192726, +IQ54408712, +IQ54697432, +IQ54838173, +IQ54914944, +IQ58106132, +IQ58138359, +IQ58258316, +IQ58258443, +IQ58859461, +IQ59060203, +IQ59322303, +IQ59404618, +IQ59404714, +IQ59677595, +IQ60229152, +IQ60422576, +IQ61350252, +IQ62202991, +IQ62315508, +IQ79692443, +IQ83478970, +IQ83970849, +IQ83984401, +IQ95096703, +IQ98741050, +IQ99084162, +IQ99937902, +IQ100600351, +IQ105289685, +IQ106531222, +IQ106631150, +IQ109106005, +IQ109134014, +IQ109530569, +IQ111222612, +IQ112943576, +IQ113534309, +IQ113589908, +IQ115789121, +IQ116848128, +IQ116959809, +IQ117217007, +IQ118031862, +IQ118799766, +IQ127913018, +IQ129566475, +IQ129642709, +IQ132919045, +IQ133764203, +IQ134031773, +IQ138304877, +IQ139949085, +IQ140722841, +IQ140731306, +IQ140786833, +IQ141069742, +IQ143995662, +IQ144213439, +IQ144398172, +IQ144934211, +IQ145637033, +IQ166398410, +IQ171518824, +IQ207031891, +IQ214891946, +IQ216123401, +IQ216542411, +IQ216716050, +IQ222661302, +IQ225107913, +IQ225463127, +IQ232486210, +IQ232732478, +IQ241498873, +IQ244663058, +IQ245183346, +IQ248535154, +IQ248701560, +IQ254045238, +IQ256361384, +IQ264279640, +IQ264545174, +IQ266784727, +IQ267454529, +IQ275369484, +IQ279664808, +IQ281069021, +IQ306614602, +IQ318869095, +IQ320329041, +IQ383989486, +IQ420215945, +IQ530308476, +IQ47732041, +IQ545930207, diff --git a/data/revReport.yml b/data/revReport.yml index 2493532..efa1eb2 100644 --- a/data/revReport.yml +++ b/data/revReport.yml @@ -5,13 +5,12 @@ params: currency: USD from: 1988-12-31 to: 2018-12-31 - - # should handle loading identifiers from local file system, QT dataset, inline ones, STDIN identifiers: - local: ../capitaliq-identifiers.txt - dataset: d-e4tf3yyxerabcvicidv5oyey + # local: ./capitaliq-identifiers.txt + # dataset: d-pjuq6lofrnbszq5zntfy4rvv inline: - IQ121238 - IQ598807 - IQ723344 - IQ956057 + # - InvalidID From be3ab3c7af8d171ca05a0981a01771aed0743b28 Mon Sep 17 00:00:00 2001 From: Grzegorz Bielski <gbielski@quantemplate.com> Date: Mon, 19 Apr 2021 11:46:54 +0200 Subject: [PATCH 3/9] use java api instead of os lib --- build.sbt | 1 - .../commands/ConfigDefInterpreterCmd.scala | 12 +++-- .../revenuereport/RevenueReportCmd.scala | 50 +++++++++++++------ .../RevenueReportConfigDef.scala | 4 +- .../quantemplate/capitaliq/common/IO.scala | 28 +++++++++-- .../capitaliq/domain/CapitalIQ.scala | 8 +-- .../capitaliq/domain/Identifiers.scala | 3 -- data/revReport.yml | 11 ++-- 8 files changed, 76 insertions(+), 41 deletions(-) diff --git a/build.sbt b/build.sbt index 89bed78..83c5165 100644 --- a/build.sbt +++ b/build.sbt @@ -33,7 +33,6 @@ lazy val capitaliq = (project in file("capitaliq")) "org.typelevel" %% "cats-core" % "2.4.2", "com.norbitltd" %% "spoiwo" % "1.7.0", "com.github.scopt" %% "scopt" % "4.0.1", - "com.lihaoyi" %% "os-lib" % "0.7.4" ).map(_.withDottyCompat(scalaVersion.value)), assembly / mainClass := Some("com.quantemplate.capitaliq.Main") ) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala index dc6ba3f..8319915 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/ConfigDefInterpreterCmd.scala @@ -1,6 +1,7 @@ package com.quantemplate.capitaliq.commands import java.time.LocalDate +import java.nio.file.Path import io.circe.{ Encoder, Decoder, Json, DecodingFailure } import io.circe.yaml.{parser as ymlParser} import org.slf4j.LoggerFactory @@ -16,7 +17,7 @@ object ConfigDefInterpreterCmd: lazy val logger = LoggerFactory.getLogger(getClass) def fromCli(args: Array[String]) = ConfigArgsParser.parse(args).map { cliConfig => - val configPath = getPath(cliConfig.path) + val configPath = IO.absolutePath(cliConfig.path) loadConfig(configPath).bimap( err => logger.error("Could not parse the config file", err), @@ -26,10 +27,11 @@ object ConfigDefInterpreterCmd: ) } -private def loadConfig(path: os.Path) = - val yml = os.read(path) - - ymlParser.parse(yml).flatMap(_.as[ConfigDef]) +private def loadConfig(path: Path) = + IO.readAll(path) + .toEither + .flatMap(ymlParser.parse(_)) + .flatMap(_.as[ConfigDef]) given Decoder[ConfigDef] = Decoder.instance[ConfigDef] { c => c.get[String]("command").flatMap { diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala index 7a05f43..13981e8 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala @@ -1,5 +1,6 @@ package com.quantemplate.capitaliq.commands.revenuereport +import java.nio.file.Path import akka.actor.typed.ActorSystem import akka.actor.typed.scaladsl.Behaviors import scala.concurrent.ExecutionContext @@ -15,8 +16,8 @@ import com.quantemplate.capitaliq.qt.* class RevenueReportCmd: given Config = Config.load() - given sys: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "capitaliq") - given ExecutionContext = sys.executionContext + given system: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "capitaliq") + given ExecutionContext = system.executionContext lazy val logger = LoggerFactory.getLogger(getClass) @@ -29,7 +30,7 @@ class RevenueReportCmd: .map(_.toCmdConfig(loadIdentifiersFromStdin())) .map(run) - def fromConfigFile(config: RevenueReportConfigDef, configPath: os.Path) = + def fromConfigFile(config: RevenueReportConfigDef, configPath: Path) = loadIdentifiersFromConfig(config, configPath) .map(_.getOrElse(loadIdentifiersFromStdin())) .map(config.toCmdConfig(_)) @@ -38,34 +39,51 @@ class RevenueReportCmd: private def loadIdentifiersFromStdin() = { logger.info("Loading the Capital IQ identifiers from the STDIN") - Identifiers.loadFromStdin() + IO.stdin(_.getLines.toVector) match + case Success(ids) => Identifiers(ids: _*) + case Failure(err) => + logger.error("Could not load the Capital IQ identifiers from the STDIN. Aborting.") + throw err } - private def loadIdentifiersFromConfig(config: RevenueReportConfigDef, configPath: os.Path) = + private def loadIdentifiersFromConfig(config: RevenueReportConfigDef, configPath: Path) = config.identifiers .flatMap(_.dataset) - .map(loadIdentifiersFromDataset(config.orgId, _)) + .map(loadIdentifiersFromDataset(config.orgId)) .sequence .map { remoteIds => val inlineIds = config.identifiers.flatMap(_.inline) - val localIds = config.identifiers.flatMap(_.local).map { rawPath => - Identifiers( - os.read.lines(getPath(rawPath, configPath / os.up)): _* - ) - } + val localIds = config.identifiers + .flatMap(_.local) + .flatMap(loadIdentifiersFromLocalFile(configPath)) (inlineIds ++ localIds ++ remoteIds).reduceOption(_ ++ _) } - private def loadIdentifiersFromDataset(orgId: String, datasetId: String) = + private def loadIdentifiersFromLocalFile(configPath: Path)(rawPath: String) = + // assuming the config path is a base for resolving the file path of identifiers + val idsPath = configPath.getParent.resolve(IO.toPath(rawPath)) + + IO.readLines(idsPath) + .recover { + case err: Throwable => + logger.warn("Could not load the Capital IQ identifiers from the local file {}", err) + Vector.empty[String] + } + .toOption + .map(Identifiers(_: _*)) + + + private def loadIdentifiersFromDataset(datasetId: String)(orgId: String) = qtService.downloadDataset(orgId, datasetId) .map(Identifiers.loadFromCsvString) + .map { ids => + logger.info("Loaded CapitalIQ identifiers from remote dataset") + ids + } .recover { case e: Throwable => - logger.warn("Could not load the identifiers from the remote dataset") - println(("err", e)) - e.printStackTrace() - + logger.warn("Could not load the CapitalIQ identifiers from the remote dataset. Defaulting to local ones.", e) Vector.empty[CapitalIQ.Identifier] } diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala index 03f11f4..f696fe5 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportConfigDef.scala @@ -27,7 +27,7 @@ case class RevenueReportConfigDef( ) object RevenueReportConfigDef: - given Decoder[RevenueReportConfigDef] = Decoder.instance[RevenueReportConfigDef] { c => + given Decoder[RevenueReportConfigDef] = Decoder { c => ( c.get[String]("orgId"), c.get[String]("datasetId"), @@ -45,7 +45,7 @@ object RevenueReportConfigDef: ) object Identifiers: - given Decoder[Identifiers] = Decoder.instance[Identifiers] { c => + given Decoder[Identifiers] = Decoder { c => ( c.get[Option[String]]("local"), c.get[Option[String]]("dataset"), diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala index ab47388..c5f5b5f 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/IO.scala @@ -1,8 +1,26 @@ package com.quantemplate.capitaliq.common -import java.nio.file.Paths +import java.nio.file.Path +import java.nio.file.FileSystems +import scala.util.{Try, Using} +import scala.io.{Source, BufferedSource} -def getPath(rawPath: String, base: os.Path = os.pwd) = - if Paths.get(rawPath).isAbsolute - then os.Path(rawPath) - else os.Path(rawPath, base = base) +object IO: + def pwd = System.getProperty("user.dir") + + def stdin[A](fn: (BufferedSource => A)) = Using(Source.stdin)(fn) + + def toPath(rawPath: String) = + FileSystems.getDefault.getPath(rawPath).normalize + + def absolutePath(rawPath: String) = + toPath(rawPath).toAbsolutePath + + def readLines(path: Path) = + read(path)(_.getLines.toVector) + + def readAll(path: Path) = + read(path)(_.getLines.mkString("\n")) + + def read[A](path: Path): (BufferedSource => A) => Try[A] = + Using(Source.fromFile(path.toFile)) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala index 73faa5a..f7b272e 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/CapitalIQ.scala @@ -147,15 +147,15 @@ object CapitalIQ: case class MnemonicResponse(error: String, rows: Option[Rows]) object MnemonicResponse: - given Decoder[RawResponse.MnemonicResponse] = Decoder.instance[RawResponse.MnemonicResponse] { cursor => + given Decoder[RawResponse.MnemonicResponse] = Decoder { c => for - error <- cursor.get[String]("ErrMsg") - jsonRows <- cursor.get[Option[Vector[Json]]]("Rows") + error <- c.get[String]("ErrMsg") + jsonRows <- c.get[Option[Vector[Json]]]("Rows") rows <- jsonRows.traverse(_.traverse(_.hcursor.get[Vector[String]]("Row"))) yield MnemonicResponse(error, rows) } - given Decoder[RawResponse] = Decoder.instance[RawResponse]( + given Decoder[RawResponse] = Decoder( _.get[Vector[RawResponse.MnemonicResponse]]("GDSSDKResponse").map(RawResponse(_)) ) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala index 2901348..bf593aa 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala @@ -20,6 +20,3 @@ object Identifiers: .toVector .map(_.split(',').headOption.getOrElse("")): _* ) - - def loadFromStdin() = - Identifiers(Source.stdin.getLines.toVector: _*) diff --git a/data/revReport.yml b/data/revReport.yml index efa1eb2..de08f56 100644 --- a/data/revReport.yml +++ b/data/revReport.yml @@ -6,11 +6,12 @@ params: from: 1988-12-31 to: 2018-12-31 identifiers: + # local: /Users/gbielski/Projects/qt/data-ingress/data/capitaliq-identifiers.txt # local: ./capitaliq-identifiers.txt # dataset: d-pjuq6lofrnbszq5zntfy4rvv - inline: - - IQ121238 - - IQ598807 - - IQ723344 - - IQ956057 + # inline: + # - IQ121238 + # - IQ598807 + # - IQ723344 + # - IQ956057 # - InvalidID From 8667fe0db253d9e3cda98c006c0367034344ebb8 Mon Sep 17 00:00:00 2001 From: Grzegorz Bielski <gbielski@quantemplate.com> Date: Mon, 19 Apr 2021 12:39:06 +0200 Subject: [PATCH 4/9] include demo account flag in config --- README.md | 2 ++ capitaliq/src/main/resources/application.conf | 8 +++++--- .../scala/com/quantemplate/capitaliq/common/Config.scala | 4 +++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 426704e..48f47e8 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ An example set of identifiers could be found in the `./data/capitaliq-identifier ``` CAPITALIQ_API_USERNAME=<api username you got with Capital IQ API license> CAPITALIQ_API_PASSWORD=<password for the corresponding Capital IQ API user> + CAPITALIQ_DEMO_ACCOUNT=<indicates whether the Capital IQ demo account is used> QT_ENV=<name of the Quantemplate environment, should be `prod` for any consumers> QT_AUTH_REALM=<name of the Quantemplate auth ream, should be `qt` for any consumers> QT_CLIENT_ID=<id of the api-client user generated by the Quantemplate team> @@ -85,6 +86,7 @@ An example set of identifiers could be found in the `./data/capitaliq-identifier ``` CAPITALIQ_API_USERNAME=apiadmin@quantemplate.com CAPITALIQ_API_PASSWORD=<password from dev credentials> + CAPITALIQ_DEMO_ACCOUNT=true QT_ENV=dev QT_AUTH_REALM=test QT_CLIENT_ID=<id of the api-client user> diff --git a/capitaliq/src/main/resources/application.conf b/capitaliq/src/main/resources/application.conf index 6e96510..626b14e 100644 --- a/capitaliq/src/main/resources/application.conf +++ b/capitaliq/src/main/resources/application.conf @@ -2,9 +2,11 @@ capitaliq { endpoint = "https://api-ciq.marketintelligence.spglobal.com/gdsapi/rest/v3/clientservice.json" # number of mnemonics / identifiers used in a single HTTP request - # 500 for prod account - # 100 for trial account - mnemonicsPerRequest = 100 + mnemonicsPerRequestInDemoAccount = 100 + mnemonicsPerRequestInProdAccount = 500 + + demoAccount = true + demoAccount = $?CAPITALIQ_DEMO_ACCOUNT} credentials { username = ${CAPITALIQ_API_USERNAME} diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/Config.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/Config.scala index 70a04e4..772df1e 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/Config.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/Config.scala @@ -35,7 +35,9 @@ object Config: conf.getString("capitaliq.credentials.username"), conf.getString("capitaliq.credentials.password"), ), - mnemonicsPerRequest = conf.getInt("capitaliq.mnemonicsPerRequest") + mnemonicsPerRequest = if conf.getBoolean("capitaliq.demoAccount") + then conf.getInt("capitaliq.mnemonicsPerRequestInDemoAccount") + else conf.getInt("capitaliq.mnemonicsPerRequestInProdAccount") ), Quantemplate( auth = Quantemplate.Auth( From 681035a140c1f6180ccc7ac79e52441d5c5b9d6a Mon Sep 17 00:00:00 2001 From: Grzegorz Bielski <gbielski@quantemplate.com> Date: Mon, 19 Apr 2021 14:55:03 +0200 Subject: [PATCH 5/9] update scala version, make simple test setup --- build.sbt | 20 ++++-- capitaliq/src/main/resources/application.conf | 3 +- .../capitaliq/common/HttpService.scala | 1 - .../capitaliq/CapitalIQServiceSpec.scala | 62 +++++++++++++++++++ data/revReport.yml | 4 +- project/build.properties | 2 +- project/plugins.sbt | 1 - 7 files changed, 81 insertions(+), 12 deletions(-) create mode 100644 capitaliq/src/test/scala/com/quantemplate/domain/capitaliq/CapitalIQServiceSpec.scala diff --git a/build.sbt b/build.sbt index 83c5165..818f800 100644 --- a/build.sbt +++ b/build.sbt @@ -1,7 +1,7 @@ name := "data-ingress" ThisBuild / organization := "com.quantemplate" -ThisBuild / scalaVersion := "3.0.0-RC1" +ThisBuild / scalaVersion := "3.0.0-RC2" ThisBuild / version := "1.0" val AkkaVersion = "2.6.12" @@ -27,13 +27,23 @@ lazy val capitaliq = (project in file("capitaliq")) "io.circe" %% "circe-parser" % CirceVersion, "io.circe" %% "circe-yaml" % "0.13.1", // misc - "org.scalatest" %% "scalatest" % "3.1.0" % Test, - "ch.qos.logback" % "logback-classic" % "1.2.3", - "com.typesafe" % "config" % "1.4.1", "org.typelevel" %% "cats-core" % "2.4.2", "com.norbitltd" %% "spoiwo" % "1.7.0", "com.github.scopt" %% "scopt" % "4.0.1", - ).map(_.withDottyCompat(scalaVersion.value)), + ).map(_.cross(CrossVersion.for3Use2_13)), + + // java deps + libraryDependencies ++= Seq( + "ch.qos.logback" % "logback-classic" % "1.2.3", + "com.typesafe" % "config" % "1.4.1", + "org.mockito" % "mockito-core" % "3.9.0" + ), + + // native scala 3 deps + libraryDependencies ++= Seq( + "org.scalameta" %% "munit" % "0.7.23" % Test + ), + assembly / mainClass := Some("com.quantemplate.capitaliq.Main") ) diff --git a/capitaliq/src/main/resources/application.conf b/capitaliq/src/main/resources/application.conf index 626b14e..685cabb 100644 --- a/capitaliq/src/main/resources/application.conf +++ b/capitaliq/src/main/resources/application.conf @@ -5,8 +5,7 @@ capitaliq { mnemonicsPerRequestInDemoAccount = 100 mnemonicsPerRequestInProdAccount = 500 - demoAccount = true - demoAccount = $?CAPITALIQ_DEMO_ACCOUNT} + demoAccount = ${CAPITALIQ_DEMO_ACCOUNT} credentials { username = ${CAPITALIQ_API_USERNAME} diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala index 70fb808..efcfff7 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/common/HttpService.scala @@ -25,7 +25,6 @@ class HttpService(using system: ActorSystem[_]): for res <- GET(endpoint, auth) body <- getResponseBody(res) - // result <- Unmarshal(body).to[B] yield body.utf8String def post[B: Decoder]( diff --git a/capitaliq/src/test/scala/com/quantemplate/domain/capitaliq/CapitalIQServiceSpec.scala b/capitaliq/src/test/scala/com/quantemplate/domain/capitaliq/CapitalIQServiceSpec.scala new file mode 100644 index 0000000..875e1d9 --- /dev/null +++ b/capitaliq/src/test/scala/com/quantemplate/domain/capitaliq/CapitalIQServiceSpec.scala @@ -0,0 +1,62 @@ +package com.quantemplate.capitaliq.domain + +import munit.FunSuite +import org.mockito.Mockito.* +import org.mockito.ArgumentMatchers.* +import scala.concurrent.Future +import scala.concurrent.ExecutionContext +import akka.actor.typed.ActorSystem +import akka.actor.typed.scaladsl.Behaviors +import cats.syntax.option.* + +import com.quantemplate.capitaliq.common.{Config, HttpService} +import com.quantemplate.capitaliq.domain.CapitalIQ.* +import com.quantemplate.capitaliq.domain.CapitalIQService.* + +class CapitalIQServiceSpec extends FunSuite: + test("the response adaptation") { + new Context(): + run { + val mockHttpService = mock(classOf[HttpService]) + + val mnemonic = Mnemonic.IQ_COMPANY_NAME_LONG(Identifier("IQ12334")) + val rows = Vector(Vector("PZU")) + + val rawResponse = RawResponse( + Vector(RawResponse.MnemonicResponse("", rows.some)) + ) + + when( + mockHttpService.post[Request, RawResponse](any(), any(), any())(any(), any()) + ).thenReturn(Future.successful(rawResponse)) + + val service = CapitalIQService(mockHttpService) + + for + result <- service.sendRequest(Request(Vector(mnemonic))) + _ = assertEquals(result, Vector(Response(mnemonic, rows))) + yield () + } + } + +class Context: + given system: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "test-system") + given ExecutionContext = system.executionContext + given Config = Config( + capitaliq = Config.CapitalIQ( + endpoint = "http://", + mnemonicsPerRequest = 100, + credentials = Config.CapitalIQ.Credentials("username", "password"), + ), + quantemplate = Config.Quantemplate( + auth = Config.Quantemplate.Auth("http://", "clientid", "clientSecret"), + api = Config.Quantemplate.Api("http://") + ) + ) + + def run[T](code: => T) = + try + code + finally + system.terminate() + diff --git a/data/revReport.yml b/data/revReport.yml index de08f56..af897d3 100644 --- a/data/revReport.yml +++ b/data/revReport.yml @@ -9,8 +9,8 @@ params: # local: /Users/gbielski/Projects/qt/data-ingress/data/capitaliq-identifiers.txt # local: ./capitaliq-identifiers.txt # dataset: d-pjuq6lofrnbszq5zntfy4rvv - # inline: - # - IQ121238 + inline: + - IQ121238 # - IQ598807 # - IQ723344 # - IQ956057 diff --git a/project/build.properties b/project/build.properties index dbae93b..e67343a 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.4.9 +sbt.version=1.5.0 diff --git a/project/plugins.sbt b/project/plugins.sbt index 54dee10..b6a6320 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,2 @@ -addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % "0.5.3") addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6") From 5967e449ae25e13e665324280918555a4bec29f0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bielski <gbielski@quantemplate.com> Date: Mon, 19 Apr 2021 16:15:34 +0200 Subject: [PATCH 6/9] remove junk files --- data/capitaliq-identifiers.csv | 850 --------------------------------- 1 file changed, 850 deletions(-) delete mode 100644 data/capitaliq-identifiers.csv diff --git a/data/capitaliq-identifiers.csv b/data/capitaliq-identifiers.csv deleted file mode 100644 index 6d5b48c..0000000 --- a/data/capitaliq-identifiers.csv +++ /dev/null @@ -1,850 +0,0 @@ -identifiers, -IQ121238, -IQ598807, -IQ723344, -IQ956057, -IQ250388, -IQ5471113, -IQ876215, -IQ162456, -IQ8201008, -IQ22517019, -IQ22663157, -IQ875826, -IQ992287, -IQ8248315, -IQ20551569, -IQ874488, -IQ341546, -IQ876731, -IQ8210142, -IQ1082017, -IQ698550, -IQ876955, -IQ5467549, -IQ8475394, -IQ882901, -IQ7605761, -IQ875328, -IQ36739404, -IQ878160, -IQ2969167, -IQ2159401, -IQ877802, -IQ3680446, -IQ5477675, -IQ303566889, -IQ8125089, -IQ882957, -IQ4512131, -IQ883670, -IQ295539, -IQ22510313, -IQ4026111, -IQ22729379, -IQ26471214, -IQ8475467, -IQ37612389, -IQ4481454, -IQ8474885, -IQ215098026, -IQ2482281, -IQ5491822, -IQ27515847, -IQ5478545, -IQ883047, -IQ12732661, -IQ876952, -IQ3561426, -IQ874863, -IQ240824094, -IQ1051073, -IQ199668, -IQ20742105, -IQ5470931, -IQ874600, -IQ41356562, -IQ22533749, -IQ875917, -IQ250307, -IQ4497069, -IQ9235665, -IQ966841, -IQ883901, -IQ20970264, -IQ965915, -IQ8221280, -IQ5495596, -IQ8105207, -IQ1542889, -IQ882762, -IQ1882861, -IQ13335251, -IQ13483916, -IQ12933390, -IQ22039549, -IQ8952000, -IQ8237245, -IQ8642193, -IQ26972175, -IQ259879432, -IQ8292712, -IQ225687232, -IQ877323, -IQ5476832, -IQ22547601, -IQ7884005, -IQ26326530, -IQ22134956, -IQ5525514, -IQ5566600, -IQ10245092, -IQ318898, -IQ22548905, -IQ877527, -IQ307183867, -IQ878130, -IQ5569241, -IQ12205230, -IQ20527977, -IQ5473193, -IQ8982543, -IQ26934166, -IQ5604059, -IQ5686144, -IQ30921662, -IQ10639513, -IQ875073, -IQ53474567, -IQ32671961, -IQ873776, -IQ5533523, -IQ615970073, -IQ9514473, -IQ59372417, -IQ32777643, -IQ261155221, -IQ11519044, -IQ26841815, -IQ5569256, -IQ11461103, -IQ20394473, -IQ20457561, -IQ138540308, -IQ20357747, -IQ7683448, -IQ310010418, -IQ245593332, -IQ9450216, -IQ38010476, -IQ8751533, -IQ224448138, -IQ20929816, -IQ28526399, -IQ21790041, -IQ879281, -IQ22709629, -IQ233447435, -IQ9003966, -IQ22830143, -IQ53474451, -IQ20644228, -IQ6462568, -IQ128394122, -IQ5554951, -IQ5525485, -IQ10580089, -IQ20406261, -IQ20334327, -IQ32879143, -IQ59290204, -IQ35622138, -IQ22530726, -IQ2395875, -IQ9285238, -IQ20968703, -IQ20411372, -IQ36066453, -IQ20405257, -IQ41542793, -IQ6499589, -IQ24351707, -IQ254768850, -IQ35045462, -IQ876389, -IQ252124, -IQ699678, -IQ20376386, -IQ30777099, -IQ36070234, -IQ20442970, -IQ5705453, -IQ20357911, -IQ8951913, -IQ20875991, -IQ34360884, -IQ45364909, -IQ20439959, -IQ419608830, -IQ20388152, -IQ142687475, -IQ25603857, -IQ20401153, -IQ44731152, -IQ9638335, -IQ880319, -IQ37530273, -IQ8237517, -IQ45364901, -IQ9104794, -IQ118758025, -IQ23136532, -IQ20385336, -IQ47019555, -IQ280102994, -IQ6182739, -IQ24389424, -IQ98911479, -IQ118529610, -IQ34360875, -IQ6096740, -IQ206594311, -IQ20351433, -IQ20387499, -IQ20357133, -IQ25853564, -IQ884289, -IQ26840491, -IQ9446689, -IQ52657639, -IQ22817884, -IQ105309130, -IQ36824712, -IQ20342548, -IQ41542810, -IQ8863946, -IQ26425108, -IQ9317591, -IQ9648039, -IQ5435672, -IQ9898706, -IQ4509080, -IQ20392649, -IQ25601530, -IQ11251263, -IQ13338102, -IQ20333219, -IQ23956851, -IQ9383541, -IQ20928564, -IQ33059622, -IQ10837576, -IQ39420520, -IQ49756433, -IQ20386271, -IQ20387422, -IQ22519534, -IQ13389347, -IQ9616853, -IQ582343023, -IQ47286427, -IQ212327509, -IQ30585041, -IQ11354999, -IQ26539734, -IQ129791710, -IQ54412624, -IQ98910795, -IQ11459490, -IQ30746047, -IQ4493353, -IQ36070207, -IQ36218630, -IQ20356961, -IQ233903518, -IQ142039532, -IQ47938242, -IQ20388408, -IQ105289664, -IQ30612714, -IQ13483021, -IQ5449547, -IQ5514776, -IQ53474509, -IQ34430293, -IQ109965608, -IQ30468440, -IQ9648367, -IQ59676977, -IQ28506082, -IQ29785842, -IQ9648084, -IQ38928712, -IQ49943055, -IQ20385355, -IQ51446644, -IQ47285586, -IQ250801799, -IQ20406066, -IQ30746764, -IQ20440724, -IQ20406489, -IQ46949286, -IQ49273432, -IQ9648044, -IQ208360255, -IQ36308773, -IQ20376809, -IQ20384809, -IQ10167110, -IQ47733696, -IQ20379938, -IQ34360936, -IQ9265738, -IQ20387321, -IQ9264708, -IQ20405282, -IQ30446153, -IQ7958370, -IQ30589185, -IQ12004130, -IQ107792708, -IQ20387682, -IQ30864618, -IQ20384756, -IQ142681075, -IQ9663375, -IQ29226678, -IQ104919719, -IQ20357359, -IQ20406865, -IQ879972, -IQ117920629, -IQ83672799, -IQ3448881, -IQ9648086, -IQ11311924, -IQ23019707, -IQ217249500, -IQ20399470, -IQ225000580, -IQ20333227, -IQ236534039, -IQ28093936, -IQ321937608, -IQ30445681, -IQ13199091, -IQ552051031, -IQ22249494, -IQ9648296, -IQ21927946, -IQ317398442, -IQ22453364, -IQ11900969, -IQ83586150, -IQ20405278, -IQ34579314, -IQ138470937, -IQ4462958, -IQ112208069, -IQ46950019, -IQ20823232, -IQ20376805, -IQ30607633, -IQ20356260, -IQ27142992, -IQ22929324, -IQ46910621, -IQ416203171, -IQ12913444, -IQ20407950, -IQ42114936, -IQ29383141, -IQ106886296, -IQ114576425, -IQ84209473, -IQ31093, -IQ169008, -IQ250309, -IQ310818, -IQ632483, -IQ678651, -IQ722875, -IQ846304, -IQ872445, -IQ874020, -IQ875292, -IQ879079, -IQ880335, -IQ881280, -IQ881579, -IQ918690, -IQ973824, -IQ1028109, -IQ1056331, -IQ1511109, -IQ1515993, -IQ1529888, -IQ1549703, -IQ2745726, -IQ2759623, -IQ2897241, -IQ2994821, -IQ3649650, -IQ4090862, -IQ4096555, -IQ4163031, -IQ4165794, -IQ4168338, -IQ4183677, -IQ4205936, -IQ4210427, -IQ4217488, -IQ4222324, -IQ4224615, -IQ4249211, -IQ4251241, -IQ4279638, -IQ4303310, -IQ4332750, -//IQ4347511 invalid, -IQ4460048, -IQ4472553, -IQ4481550, -IQ4494248, -IQ4497162, -IQ4559043, -IQ4561438, -IQ4586814, -IQ4662943, -IQ4717183, -IQ4730735, -IQ4762761, -IQ4772278, -IQ5068910, -IQ5375495, -IQ5383311, -IQ5410293, -IQ5423523, -IQ5466287, -IQ5477916, -IQ5484075, -IQ5491169, -IQ5496125, -IQ5507600, -IQ5510525, -IQ5525297, -IQ5538187, -IQ5538747, -IQ5542158, -IQ5547143, -IQ5547613, -IQ5554373, -IQ5569286, -IQ5579426, -IQ5647001, -IQ5704985, -IQ5718690, -IQ5723278, -IQ5726675, -IQ5774517, -IQ5885638, -IQ5901538, -IQ5924809, -IQ6139502, -IQ6439926, -IQ6463183, -IQ6469667, -IQ6475664, -IQ6491233, -IQ6564312, -IQ6565183, -IQ6613221, -IQ6688088, -IQ6699370, -IQ6700017, -IQ6812156, -IQ6819792, -IQ6921281, -IQ6948694, -IQ6949267, -IQ7603981, -IQ7918016, -IQ7960416, -IQ8024164, -IQ8100801, -IQ8101330, -IQ8142882, -IQ8190935, -IQ8264274, -IQ8359879, -IQ9377948, -IQ9450267, -IQ9460690, -IQ9461042, -IQ9550032, -IQ9560354, -IQ9587298, -IQ9638878, -IQ9794004, -IQ9899048, -IQ9916947, -IQ10052670, -IQ11212063, -IQ11323005, -IQ11712739, -IQ12058656, -IQ12455886, -IQ12456448, -IQ12466795, -IQ12784598, -IQ12785981, -IQ12868386, -IQ12913479, -IQ12932470, -IQ13011212, -IQ13154195, -IQ13408154, -IQ13610052, -IQ20417852, -IQ20475401, -IQ20551258, -IQ20670757, -IQ20703772, -IQ20724600, -IQ20757724, -IQ20784162, -IQ20882165, -IQ20986395, -IQ21813186, -IQ21854837, -IQ21935579, -IQ22095148, -IQ22239048, -IQ22383525, -IQ22383606, -IQ22431794, -IQ22498272, -IQ22511361, -IQ22515611, -IQ22515844, -IQ22515964, -IQ22516125, -IQ22517048, -IQ22534319, -IQ22534512, -IQ22547744, -IQ22580222, -IQ22581143, -IQ22581472, -IQ22631649, -IQ22638720, -IQ22728611, -IQ22783832, -IQ22798804, -IQ22799264, -IQ22905959, -IQ22926403, -IQ22931810, -IQ23018346, -IQ23021162, -IQ23021452, -IQ23035756, -IQ23048633, -IQ23063990, -IQ23076171, -IQ23185132, -IQ23221640, -IQ23294481, -IQ23313835, -IQ23335868, -IQ23336901, -IQ23631262, -IQ23751882, -IQ23756872, -IQ23780053, -IQ23958224, -IQ24393771, -IQ24861410, -IQ25473977, -IQ25525075, -IQ25746444, -IQ25813840, -IQ25833297, -IQ25835239, -IQ25870800, -IQ25878705, -IQ25912594, -IQ25913389, -IQ25913460, -IQ25915583, -IQ25935856, -IQ26023649, -IQ26079324, -IQ26307837, -IQ26321017, -IQ26442394, -IQ26502798, -IQ26554716, -IQ26671175, -IQ26763923, -IQ26935576, -IQ26935957, -IQ26937924, -IQ26954558, -IQ26954965, -IQ26955671, -IQ26956943, -IQ26962223, -IQ26971203, -IQ26971884, -IQ26971963, -IQ26971979, -IQ26972138, -IQ26972384, -IQ26972531, -IQ26972634, -IQ26972671, -IQ27129412, -IQ27204765, -IQ27400044, -IQ27484055, -IQ27490077, -IQ27580696, -IQ27580703, -IQ27580715, -IQ27606887, -IQ27717292, -IQ27868361, -IQ27903956, -IQ28100928, -IQ28368577, -IQ28387463, -IQ28389647, -IQ28942515, -IQ29037395, -IQ29183054, -IQ29291881, -IQ29601835, -IQ30343267, -IQ30441904, -IQ30456609, -IQ30465921, -IQ30490816, -IQ30512591, -IQ30515896, -IQ30601911, -IQ30613257, -IQ30637238, -IQ30683814, -IQ30705424, -IQ30739286, -IQ30789560, -IQ30795690, -IQ30884638, -IQ30915913, -IQ30964685, -IQ31047931, -IQ31065816, -IQ31108425, -IQ31131565, -IQ31134863, -IQ31136645, -IQ31136770, -IQ32589022, -IQ32907389, -IQ33350617, -IQ33404477, -IQ33597720, -IQ34032519, -IQ34360580, -IQ34360903, -IQ34399111, -IQ34594700, -IQ34648845, -IQ34791159, -IQ34791294, -IQ35101458, -IQ35134316, -IQ35144642, -IQ35392019, -IQ35412230, -IQ35424160, -IQ35674202, -IQ35749702, -IQ35960436, -IQ36621656, -IQ36837622, -IQ37262203, -IQ37507853, -IQ37782812, -IQ38919390, -IQ38991075, -IQ39313436, -IQ39601472, -IQ39636404, -IQ39637638, -IQ39642860, -IQ40153601, -IQ40254139, -IQ40311435, -IQ40311458, -IQ40332961, -IQ40453340, -IQ41087044, -IQ42434220, -IQ42439675, -IQ42680296, -IQ42746052, -IQ43188556, -IQ44459504, -IQ44519103, -IQ45978451, -IQ46239777, -IQ46367675, -IQ46511782, -IQ46638310, -IQ46692499, -IQ46771171, -IQ46855213, -IQ46865535, -IQ46883648, -IQ46884369, -IQ46894216, -IQ46895330, -IQ46904183, -IQ46954509, -IQ46959086, -IQ47014971, -IQ47017185, -IQ47064062, -IQ47149593, -IQ47439243, -IQ47468586, -IQ47475952, -IQ47691832, -IQ47838075, -IQ47938173, -IQ48970871, -IQ49060929, -IQ49380472, -IQ49410153, -IQ49411434, -IQ49444150, -IQ49516174, -IQ49518798, -IQ49545850, -IQ49548668, -IQ49650376, -IQ49685718, -IQ50015159, -IQ50842667, -IQ50876655, -IQ51614649, -IQ52034375, -IQ52037418, -IQ52139112, -IQ52292994, -IQ52324618, -IQ52350790, -IQ52414484, -IQ52514348, -IQ52715887, -IQ53039417, -IQ54075352, -IQ54192726, -IQ54408712, -IQ54697432, -IQ54838173, -IQ54914944, -IQ58106132, -IQ58138359, -IQ58258316, -IQ58258443, -IQ58859461, -IQ59060203, -IQ59322303, -IQ59404618, -IQ59404714, -IQ59677595, -IQ60229152, -IQ60422576, -IQ61350252, -IQ62202991, -IQ62315508, -IQ79692443, -IQ83478970, -IQ83970849, -IQ83984401, -IQ95096703, -IQ98741050, -IQ99084162, -IQ99937902, -IQ100600351, -IQ105289685, -IQ106531222, -IQ106631150, -IQ109106005, -IQ109134014, -IQ109530569, -IQ111222612, -IQ112943576, -IQ113534309, -IQ113589908, -IQ115789121, -IQ116848128, -IQ116959809, -IQ117217007, -IQ118031862, -IQ118799766, -IQ127913018, -IQ129566475, -IQ129642709, -IQ132919045, -IQ133764203, -IQ134031773, -IQ138304877, -IQ139949085, -IQ140722841, -IQ140731306, -IQ140786833, -IQ141069742, -IQ143995662, -IQ144213439, -IQ144398172, -IQ144934211, -IQ145637033, -IQ166398410, -IQ171518824, -IQ207031891, -IQ214891946, -IQ216123401, -IQ216542411, -IQ216716050, -IQ222661302, -IQ225107913, -IQ225463127, -IQ232486210, -IQ232732478, -IQ241498873, -IQ244663058, -IQ245183346, -IQ248535154, -IQ248701560, -IQ254045238, -IQ256361384, -IQ264279640, -IQ264545174, -IQ266784727, -IQ267454529, -IQ275369484, -IQ279664808, -IQ281069021, -IQ306614602, -IQ318869095, -IQ320329041, -IQ383989486, -IQ420215945, -IQ530308476, -IQ47732041, -IQ545930207, From f0b88954edb91857505a5473e049ddb83f8f4d0f Mon Sep 17 00:00:00 2001 From: Grzegorz Bielski <gbielski@quantemplate.com> Date: Mon, 19 Apr 2021 17:06:53 +0200 Subject: [PATCH 7/9] update readme --- README.md | 17 ++++++++++---- build.sbt | 4 ++-- .../revenuereport/RevenueReportCmd.scala | 8 ++++++- .../capitaliq/domain/Identifiers.scala | 2 +- data/revReport.yml | 23 +++++++++---------- 5 files changed, 34 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 48f47e8..6338a40 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Integrations with third-party data providers such as [Capital IQ](https://www.capitaliq.com) leveraging the [Quantemplate Data Ingress API](https://quantemplate.readme.io/docs/getting-started#-data-ingress). -If you need help please contact as at support@quantemplate.com +If you need help please contact us at support@quantemplate.com # Capital IQ @@ -26,9 +26,18 @@ The diagram below describes a potential integration pattern where CapitalIQ data - Generating a total revenue report from the CapitalIQ data and uploading it to the Quantemplate dataset - ```sh - cat ./data/capitaliq-identifiers.txt | java -jar ./capitaliq/target/scala-3.0.0-RC1/capitaliq-assembly-1.0.jar generateRevenueReport --orgId c-my-small-insuranc-ltdzfd --datasetId d-e4tf3yyxerabcvicidv5oyey --currency USD --from 1988-12-31 --to 2018-12-31 - ``` + - with yaml config: + ```sh + java -jar ./capitaliq/target/scala-3.0.0-RC2/capitaliq-assembly-1.0.jar apply ./data/revReport.yml + ``` + + Check out the [config file](./data/revReport.yml) + + + - with CLI args: + ```sh + cat ./data/capitaliq-identifiers.txt | java -jar ./capitaliq/target/scala-3.0.0-RC2/capitaliq-assembly-1.0.jar generateRevenueReport --orgId c-my-small-insuranc-ltdzfd --datasetId d-e4tf3yyxerabcvicidv5oyey --currency USD --from 1988-12-31 --to 2018-12-31 + ``` diff --git a/build.sbt b/build.sbt index 818f800..95b5fef 100644 --- a/build.sbt +++ b/build.sbt @@ -36,7 +36,7 @@ lazy val capitaliq = (project in file("capitaliq")) libraryDependencies ++= Seq( "ch.qos.logback" % "logback-classic" % "1.2.3", "com.typesafe" % "config" % "1.4.1", - "org.mockito" % "mockito-core" % "3.9.0" + "org.mockito" % "mockito-core" % "3.9.0" % Test ), // native scala 3 deps @@ -44,7 +44,7 @@ lazy val capitaliq = (project in file("capitaliq")) "org.scalameta" %% "munit" % "0.7.23" % Test ), - assembly / mainClass := Some("com.quantemplate.capitaliq.Main") + assembly / mainClass := Some("com.quantemplate.capitaliq.Main"), ) Compile / run / fork := true diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala index 13981e8..186f34a 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala @@ -88,9 +88,15 @@ class RevenueReportCmd: } private def run(config: CmdConfig) = + val ids = config.identifiers.distinct + + if ids.isEmpty then + logger.error("No valid CapitalIQ identifiers were provided. Aborting") + Runtime.getRuntime.halt(1) + revenueReport .generateSpreadSheet( - ids = config.identifiers.distinct, + ids = ids, range = ( config.from, config.to diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala index bf593aa..8a608e6 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/domain/Identifiers.scala @@ -5,7 +5,7 @@ import scala.io.Source import CapitalIQ.Identifier object Identifiers: - def apply(ids: String*) = + def apply(ids: String*) = ids .filter(Identifier.isValid) .map(Identifier(_)) diff --git a/data/revReport.yml b/data/revReport.yml index af897d3..c95a0a0 100644 --- a/data/revReport.yml +++ b/data/revReport.yml @@ -1,16 +1,15 @@ command: generateRevenueReport -params: - orgId: c-my-small-insuranc-ltdzfd - datasetId: d-e4tf3yyxerabcvicidv5oyey - currency: USD - from: 1988-12-31 - to: 2018-12-31 - identifiers: - # local: /Users/gbielski/Projects/qt/data-ingress/data/capitaliq-identifiers.txt - # local: ./capitaliq-identifiers.txt - # dataset: d-pjuq6lofrnbszq5zntfy4rvv - inline: - - IQ121238 +params: + orgId: c-my-small-insuranc-ltdzfd # Quantemplate organisationId + datasetId: d-e4tf3yyxerabcvicidv5oyey # Quantemplate datasetId + currency: USD # Currency supported by the CapitalIQ + from: 1988-12-31 # Starting date in yyy-mm-dd format for the CapitalIQ query + to: 2018-12-31 # Ending date in yyy-mm-dd format for the CapitalIQ query + identifiers: # CapitalIQ identifiers, that could be downloaded from: + # dataset: d-pjuq6lofrnbszq5zntfy4rvv # - remote Quantemplate `dataset` (currently it only takes data from the first column) + # local: ./capitaliq-identifiers.txt # - `local` .txt file, (relative file path will be resolved against the config file) + inline: # - defined `inline` in the config file + # - IQ121238 # Multiple sources of identifiers will be merged together # - IQ598807 # - IQ723344 # - IQ956057 From 21fc08b5d3106bada39e1b2734912efac3743cd6 Mon Sep 17 00:00:00 2001 From: Grzegorz Bielski <gbielski@quantemplate.com> Date: Tue, 20 Apr 2021 10:41:05 +0200 Subject: [PATCH 8/9] update repo name --- README.md | 8 ++++---- build.sbt | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6338a40..a3b03ca 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# data-ingress +# api-integrations -Integrations with third-party data providers such as [Capital IQ](https://www.capitaliq.com) leveraging the [Quantemplate Data Ingress API](https://quantemplate.readme.io/docs/getting-started#-data-ingress). +Integrations with third-party data providers such as [Capital IQ](https://www.capitaliq.com) leveraging the [Quantemplate API](https://quantemplate.readme.io/docs/getting-started). If you need help please contact us at support@quantemplate.com @@ -28,7 +28,7 @@ The diagram below describes a potential integration pattern where CapitalIQ data - with yaml config: ```sh - java -jar ./capitaliq/target/scala-3.0.0-RC2/capitaliq-assembly-1.0.jar apply ./data/revReport.yml + java -jar ./capitaliq/target/scala-3.0.0-RC2/capitaliq-assembly-0.1.0.jar apply ./data/revReport.yml ``` Check out the [config file](./data/revReport.yml) @@ -36,7 +36,7 @@ The diagram below describes a potential integration pattern where CapitalIQ data - with CLI args: ```sh - cat ./data/capitaliq-identifiers.txt | java -jar ./capitaliq/target/scala-3.0.0-RC2/capitaliq-assembly-1.0.jar generateRevenueReport --orgId c-my-small-insuranc-ltdzfd --datasetId d-e4tf3yyxerabcvicidv5oyey --currency USD --from 1988-12-31 --to 2018-12-31 + cat ./data/capitaliq-identifiers.txt | java -jar ./capitaliq/target/scala-3.0.0-RC2/capitaliq-assembly-0.1.0.jar generateRevenueReport --orgId c-my-small-insuranc-ltdzfd --datasetId d-e4tf3yyxerabcvicidv5oyey --currency USD --from 1988-12-31 --to 2018-12-31 ``` diff --git a/build.sbt b/build.sbt index 95b5fef..1810d0a 100644 --- a/build.sbt +++ b/build.sbt @@ -1,8 +1,8 @@ -name := "data-ingress" +name := "api-integrations" ThisBuild / organization := "com.quantemplate" ThisBuild / scalaVersion := "3.0.0-RC2" -ThisBuild / version := "1.0" +ThisBuild / version := "0.1.0" val AkkaVersion = "2.6.12" val AkkaHttpVersion = "10.2.4" From 87b1e06159374d5de25ff081cbf00ba1fe148f3f Mon Sep 17 00:00:00 2001 From: Grzegorz Bielski <gbielski@quantemplate.com> Date: Tue, 20 Apr 2021 11:50:09 +0200 Subject: [PATCH 9/9] make revenue report cmd members private --- .../commands/revenuereport/RevenueReportCmd.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala index 186f34a..9a799f7 100644 --- a/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala +++ b/capitaliq/src/main/scala/com/quantemplate/capitaliq/commands/revenuereport/RevenueReportCmd.scala @@ -15,15 +15,15 @@ import com.quantemplate.capitaliq.qt.* class RevenueReportCmd: - given Config = Config.load() - given system: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "capitaliq") - given ExecutionContext = system.executionContext + private given Config = Config.load() + private given system: ActorSystem[Nothing] = ActorSystem(Behaviors.empty, "capitaliq") + private given ExecutionContext = system.executionContext - lazy val logger = LoggerFactory.getLogger(getClass) + private lazy val logger = LoggerFactory.getLogger(getClass) - val httpService = HttpService() - val qtService = QTService(httpService) - val revenueReport = RevenueReport(CapitalIQService(httpService), qtService) + private val httpService = HttpService() + private val qtService = QTService(httpService) + private val revenueReport = RevenueReport(CapitalIQService(httpService), qtService) def fromCli(args: Array[String]) = RevenueReportArgsParser.parse(args)