diff --git a/.github/workflows/jacoco_report.yml b/.github/workflows/jacoco_report.yml index bc629100..7830b727 100644 --- a/.github/workflows/jacoco_report.yml +++ b/.github/workflows/jacoco_report.yml @@ -69,7 +69,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{ env.coverage-overall }} min-coverage-changed-files: ${{ env.coverage-changed-files }} - title: JaCoCo `model` module code coverage report - scala ${{ env.scalaLong }} + title: JaCoCo `core` module code coverage report - scala ${{ env.scalaLong }} update-comment: true - name: Add coverage to PR (doobie) if: steps.jacocorun.outcome == 'success' @@ -80,7 +80,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{ env.coverage-overall }} min-coverage-changed-files: ${{ env.coverage-changed-files }} - title: JaCoCo `agent` module code coverage report - scala ${{ env.scalaLong }} + title: JaCoCo `doobie` module code coverage report - scala ${{ env.scalaLong }} update-comment: true - name: Add coverage to PR (slick) if: steps.jacocorun.outcome == 'success' diff --git a/demo_database/src/main/postgres/integration/V1.2.15__actors_json_seq.ddl b/demo_database/src/main/postgres/integration/V1.2.15__actors_json_seq.ddl new file mode 100644 index 00000000..de4c6c91 --- /dev/null +++ b/demo_database/src/main/postgres/integration/V1.2.15__actors_json_seq.ddl @@ -0,0 +1,21 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE TABLE IF NOT EXISTS integration.actors_json_seq ( + id SERIAL PRIMARY KEY, + actors_json JSON[], + actors_jsonb JSONB[] +); diff --git a/demo_database/src/main/postgres/integration/V1.2.16__insert_actors_json.sql b/demo_database/src/main/postgres/integration/V1.2.16__insert_actors_json.sql new file mode 100644 index 00000000..b59c5ab7 --- /dev/null +++ b/demo_database/src/main/postgres/integration/V1.2.16__insert_actors_json.sql @@ -0,0 +1,23 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE OR REPLACE FUNCTION integration.insert_actors_json(actorsJson JSON[], actorsJsonb JSONB[]) + RETURNS void AS $$ +BEGIN + INSERT INTO integration.actors_json_seq (actors_json, actors_jsonb) + VALUES (actorsJson, actorsJsonb); +END; +$$ LANGUAGE plpgsql; diff --git a/demo_database/src/main/postgres/integration/V1.2.17__retrieve_actors_json.sql b/demo_database/src/main/postgres/integration/V1.2.17__retrieve_actors_json.sql new file mode 100644 index 00000000..a34a472a --- /dev/null +++ b/demo_database/src/main/postgres/integration/V1.2.17__retrieve_actors_json.sql @@ -0,0 +1,22 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE OR REPLACE FUNCTION integration.retrieve_actors_json(idUntil INT) + RETURNS TABLE(actors_json JSON[]) AS $$ +BEGIN + RETURN QUERY SELECT a.actors_json FROM integration.actors_json_seq AS a WHERE id <= idUntil; +END; +$$ LANGUAGE plpgsql; diff --git a/demo_database/src/main/postgres/integration/V1.2.18__retrieve_actors_jsonb.sql b/demo_database/src/main/postgres/integration/V1.2.18__retrieve_actors_jsonb.sql new file mode 100644 index 00000000..bb5e3e4f --- /dev/null +++ b/demo_database/src/main/postgres/integration/V1.2.18__retrieve_actors_jsonb.sql @@ -0,0 +1,22 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE OR REPLACE FUNCTION integration.retrieve_actors_jsonb(idUntil INT) + RETURNS TABLE(actors_jsonb JSONB[]) AS $$ +BEGIN + RETURN QUERY SELECT a.actors_jsonb FROM integration.actors_json_seq AS a WHERE id <= idUntil; +END; +$$ LANGUAGE plpgsql; diff --git a/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala new file mode 100644 index 00000000..baf7ddaf --- /dev/null +++ b/doobie/src/main/scala/za/co/absa/db/fadb/doobie/postgres/circe/implicits/package.scala @@ -0,0 +1,103 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.db.fadb.doobie.postgres.circe + +import cats.Show +import cats.data.NonEmptyList +import doobie.{Get, Put} +import io.circe.Json +import org.postgresql.jdbc.PgArray +import org.postgresql.util.PGobject +import io.circe.parser._ + +import scala.util.{Failure, Success, Try} + +package object implicits { + + private implicit val showPgArray: Show[PgArray] = Show.fromToString + + implicit val jsonPut: Put[Json] = doobie.postgres.circe.json.implicits.jsonPut + implicit val jsonbPut: Put[Json] = doobie.postgres.circe.jsonb.implicits.jsonbPut + + implicit val jsonGet: Get[Json] = doobie.postgres.circe.json.implicits.jsonGet + implicit val jsonbGet: Get[Json] = doobie.postgres.circe.jsonb.implicits.jsonbGet + + implicit val jsonArrayPut: Put[List[Json]] = { + Put.Advanced + .other[PGobject]( + NonEmptyList.of("json[]") + ) + .tcontramap { a => + val o = new PGobject + o.setType("json[]") + o.setValue(jsonListToPGJsonArrayString(a)) + o + } + } + + implicit val jsonbArrayPut: Put[List[Json]] = { + Put.Advanced + .other[PGobject]( + NonEmptyList.of("jsonb[]") + ) + .tcontramap { a => + val o = new PGobject + o.setType("jsonb[]") + o.setValue(jsonListToPGJsonArrayString(a)) + o + } + } + + // to be used for both json[] and jsonb[] as it handles well both + // and we want to avoid collision when resolving implicits + implicit val jsonOrJsonbArrayGet: Get[List[Json]] = { + Get.Advanced + .other[PgArray]( + NonEmptyList.of("json[]") + ) + .temap(pgArray => pgArrayToListOfJson(pgArray)) + } + + private def jsonListToPGJsonArrayString(jsonList: List[Json]): String = { + val arrayElements = jsonList.map { x => + // Convert to compact JSON string and escape inner quotes + val escapedJsonString = x.noSpaces.replace("\"", "\\\"") + // Wrap in double quotes for the array element + s""""$escapedJsonString"""" + } + + arrayElements.mkString("{", ",", "}") + } + + private def pgArrayToListOfJson(pgArray: PgArray): Either[String, List[Json]] = { + Try(Option(pgArray.getArray)) match { + case Success(Some(array: Array[_])) => + val results = array.toList.map { + case str: String => parse(str).left.map(_.getMessage) + case other => parse(other.toString).left.map(_.getMessage) + } + results.partition(_.isLeft) match { + case (Nil, rights) => Right(rights.collect { case Right(json) => json }) + case (lefts, _) => Left("Failed to parse JSON: " + lefts.collect { case Left(err) => err }.mkString(", ")) + } + case Success(Some(_)) => Left("Unexpected type encountered. Expected an Array.") + case Success(None) => Right(Nil) + case Failure(exception) => Left(exception.getMessage) + } + } + +} diff --git a/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala b/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala new file mode 100644 index 00000000..f65e3b42 --- /dev/null +++ b/doobie/src/test/scala/za/co/absa/db/fadb/doobie/JsonArrayIntegrationTests.scala @@ -0,0 +1,80 @@ +/* + * Copyright 2022 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.db.fadb.doobie + +import cats.effect.IO +import cats.effect.unsafe.implicits.global +import doobie.implicits.toSqlInterpolator +import io.circe.Json +import io.circe.syntax.EncoderOps +import org.scalatest.funsuite.AnyFunSuite +import za.co.absa.db.fadb.DBSchema +import za.co.absa.db.fadb.doobie.DoobieFunction.{DoobieMultipleResultFunction, DoobieSingleResultFunction} +import za.co.absa.db.fadb.testing.classes.DoobieTest +import io.circe.generic.auto._ + +import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonOrJsonbArrayGet + +class JsonArrayIntegrationTests extends AnyFunSuite with DoobieTest { + + class InsertActorsJson(implicit schema: DBSchema, dbEngine: DoobieEngine[IO]) + extends DoobieSingleResultFunction[List[Actor], Unit, IO] ( + values => { + val actorsAsJsonList = values.map(_.asJson) + Seq( + { + // has to be imported inside separate scope to avoid conflicts with the import below + // as both implicits are of the same type and this would cause ambiguity + import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonArrayPut + fr"$actorsAsJsonList" + }, + { + // has to be imported inside separate scope to avoid conflicts with the import above + // as both implicits are of the same type and this would cause ambiguity + import za.co.absa.db.fadb.doobie.postgres.circe.implicits.jsonbArrayPut + fr"$actorsAsJsonList" + } + ) + } + ) + + class RetrieveActorsJson(implicit schema: DBSchema, dbEngine: DoobieEngine[IO]) + extends DoobieMultipleResultFunction[Int, List[Json], IO] ( + values => Seq(fr"$values") + ) + + class RetrieveActorsJsonb(implicit schema: DBSchema, dbEngine: DoobieEngine[IO]) + extends DoobieMultipleResultFunction[Int, List[Json], IO] ( + values => Seq(fr"$values") + ) + + private val insertActorsJson = new InsertActorsJson()(Integration, new DoobieEngine(transactor)) + + test("Retrieve Actors from json[] and jsonb[] columns"){ + val expectedActors = List(Actor(1, "John", "Doe"), Actor(2, "Jane", "Doe")) + insertActorsJson(expectedActors).unsafeRunSync() + + val retrieveActorsJson = new RetrieveActorsJson()(Integration, new DoobieEngine(transactor)) + val actualActorsJson = retrieveActorsJson(2).unsafeRunSync() + assert(expectedActors == actualActorsJson.head.map(_.as[Actor]).map(_.toTry.get)) + + val retrieveActorsJsonb = new RetrieveActorsJsonb()(Integration, new DoobieEngine(transactor)) + val actualActorsJsonb = retrieveActorsJsonb(2).unsafeRunSync() + assert(expectedActors == actualActorsJsonb.head.map(_.as[Actor]).map(_.toTry.get)) + } + +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 256fc732..1ad183b8 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -14,7 +14,7 @@ * limitations under the License. */ -import sbt._ +import sbt.* object Dependencies { @@ -47,7 +47,9 @@ object Dependencies { commonDependencies(scalaVersion) ++ Seq( "org.tpolecat" %% "doobie-core" % "1.0.0-RC2", "org.tpolecat" %% "doobie-hikari" % "1.0.0-RC2", - "org.tpolecat" %% "doobie-postgres" % "1.0.0-RC2" + "org.tpolecat" %% "doobie-postgres" % "1.0.0-RC2", + "org.tpolecat" %% "doobie-postgres-circe" % "1.0.0-RC2", + "io.circe" %% "circe-generic" % "0.14.9" % Test ) } @@ -56,4 +58,5 @@ object Dependencies { Seq(postgresql) } + }