feat: CQDG-369 new apis for using ferload by resources ids

.github/workflows/build.yml

@@ -1,24 +1,18 @@
 name: Test and build
 on:
-  pull_request:
-    branches:
-      - main
+  push:
 
 jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v1
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+      - uses: actions/checkout@v3
+      - uses: actions/setup-java@v3
         with:
-          java-version: "[email protected]"
-      - name: Test
-        run: sbt -v -Dfile.encoding=UTF-8 +test
-      - name: Publish Test Report
-        uses: mikepenz/action-junit-report@v2
-        with:
-          report_paths: 'target/test-reports/TEST-*.xml'
-      - name: Build
-        run: sbt -v -Dfile.encoding=UTF-8 +universal:packageZipTarball
+          distribution: 'corretto'
+          java-version: '17'
+          cache: 'sbt'
+      - name: Run tests
+        run: sbt test
+      - name: Assembly
+        run: sbt assembly

.github/workflows/publish_with_latest.yml

@@ -10,20 +10,16 @@ jobs:
     name: Publish Image using latest tag
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+      - uses: actions/checkout@v3
+      - uses: actions/setup-java@v3
         with:
-          java-version: "[email protected]"
-      - name: Test
-        run: sbt -v -Dfile.encoding=UTF-8 +test
-      - name: Publish Test Report
-        uses: mikepenz/action-junit-report@v2
-        with:
-          report_paths: 'target/test-reports/TEST-*.xml'
-      - name: Build
-        run: sbt -v -Dfile.encoding=UTF-8 +universal:packageZipTarball
+          distribution: 'corretto'
+          java-version: '17'
+          cache: 'sbt'
+      - name: Run tests
+        run: sbt test
+      - name: Assembly
+        run: sbt assembly
       - name: Push the image on the registry
         uses: Ferlab-Ste-Justine/action-push-image@v2
         with:

.github/workflows/publish_with_semver_tag.yml

@@ -10,20 +10,15 @@ jobs:
     name: Publish Image using tags
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+      - uses: actions/setup-java@v3
         with:
-          java-version: "[email protected]"
-      - name: Test
-        run: sbt -v -Dfile.encoding=UTF-8 +test
-      - name: Publish Test Report
-        uses: mikepenz/action-junit-report@v2
-        with:
-          report_paths: 'target/test-reports/TEST-*.xml'
-      - name: Build
-        run: sbt -v -Dfile.encoding=UTF-8 +universal:packageZipTarball
+          distribution: 'corretto'
+          java-version: '17'
+          cache: 'sbt'
+      - name: Run tests
+        run: sbt test
+      - name: Assembly
+        run: sbt assembly
       - name: Push the image on the registry
         uses: Ferlab-Ste-Justine/action-push-image@v2
         with:

.github/workflows/publish_with_sha.yml

@@ -10,20 +10,15 @@ jobs:
     name: Publish Image using commit sha and timestamp
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Setup Scala
-        uses: olafurpg/setup-scala@v10
+      - uses: actions/setup-java@v3
         with:
-          java-version: "[email protected]"
-      - name: Test
-        run: sbt -v -Dfile.encoding=UTF-8 +test
-      - name: Publish Test Report
-        uses: mikepenz/action-junit-report@v2
-        with:
-          report_paths: 'target/test-reports/TEST-*.xml'
-      - name: Build
-        run: sbt -v -Dfile.encoding=UTF-8 +universal:packageZipTarball
+          distribution: 'corretto'
+          java-version: '17'
+          cache: 'sbt'
+      - name: Run tests
+        run: sbt test
+      - name: Assembly
+        run: sbt assembly
       - name: Push the image on the registry
         uses: Ferlab-Ste-Justine/action-push-image@v2
         with:

.gitignore

@@ -1,9 +1,11 @@
-logs
+# common IDEs to be ignored
+.idea/
+.metals/
+.vscode/
+
+# build tool specific entries
+.bloop
+.bsp
 target
-/.idea
-/.g8
-/.idea_modules
-/.classpath
-/.project
-/.settings
-/RUNNING_PID
+metals.sbt
+project/project

.scalafmt.conf

@@ -0,0 +1,3 @@
+version = 3.7.14
+maxColumn = 140
+runner.dialect = scala3

Dockerfile

@@ -1,9 +1,5 @@
-FROM adoptopenjdk/openjdk11:alpine-jre
+FROM amazoncorretto:17-alpine3.18
 
-RUN apk update && apk add bash ca-certificates openssl
+COPY target/scala-3.3.1/ferload.jar .
 
-COPY target/universal/ferload.tgz .
-
-RUN tar xvf ferload.tgz
-
-ENTRYPOINT ["/ferload/bin/ferload"]
+ENTRYPOINT ["java", "-jar", "ferload.jar"]

README.md

@@ -1,6 +1,91 @@
-# ferload
+# Ferload
 
-This service is used to dowload files stored in an object store s3 compliant. It will :
-- Verify if user has access to the object (validating jwt token)
-- Generate a presigned url for this object
+    [![Docker Pulls](https://img.shields.io/docker/pulls/ferlabcrsj/ferload)](https://hub.docker.com/r/ferlab/ferload)
+    [![Docker Image Size (latest SEMVER)](https://img.shields.io/docker/image-size/ferlabcrsj/ferload?sort=semver)](https://hub.docker.com/r/ferlab/ferload)
+    [![Docker Image Version (latest semver)](https://img.shields.io/docker/v/ferlabcrsj/ferload?sort=semver)](https://hub.docker.com/r/ferlab/ferload)
 
+Ferload is an api that allows to control access to files stored in any object store S3-compliant, and generate presigned url if user is granted.
+Ferload is compliant with [GA4GH Data Repository Service](https://ga4gh.github.io/data-repository-service-schemas/). For now, only Bearers tokens are supported (not Passport and Visas).
+
+
+## Technologies
+
+Ferload is developed in Scala3 and is based on [tapir](https://tapir.softwaremill.com/en/latest/) and [cats-effect](https://typelevel.org/cats-effect/).
+
+## Quick start
+
+If you don't have [sbt](https://www.scala-sbt.org) installed already, you can use the provided wrapper script:
+
+```shell
+./sbtx -h # shows an usage of a wrapper script
+./sbtx compile # build the project
+./sbtx test # run the tests
+./sbtx run # run the application (Main)
+```
+
+For more details check the [sbtx usage](https://github.com/dwijnand/sbt-extras#sbt--h) page.
+
+Otherwise, if sbt is already installed, you can use the standard commands:
+
+```shell
+sbt compile # build the project
+sbt test # run the tests
+sbt run # run the application (Main)
+```
+
+## Build
+
+```shell
+sbt asembly
+docker build -t ferload .
+```
+
+## Environment variables
+
+Keyckloak Authentication server information :
+
+- `AUTH_URL` : Keycloak URL
+- `AUTH_REALM` : Keycloak Realm
+- `AUTH_CLIENT_ID` : Id of the client that contains resource definition and permissions
+- `AUTH_SECRET_KEY` : Secret of the client that contains resource definition and permissions
+- `AUTH_AUDIENCE` : ???
+- `AUTH_RESOURCES_POLICY_GLOBAL_NAME` : Name of the resource a user should have access to be able to download all files.
+  Works only with endpoints that fetch files by urls. Can be empty.
+
+AWS S3 information :
+
+- `AWS_ACCESS_KEY` : Access key of the AWS account
+- `AWS_SECRET_KEY` : Secret key of the AWS account
+- `AWS_BUCKET` : Default bucket to use if objects are fetched by urls. Can be empty.
+- `AWS_ENDPOINT`: Endpoint to S3 service. Can be empty.
+- `AWS_PATH_ACCESS_STYLE` : Path access style to S3 service (true for minio, false for AWS). Default false.
+- `AWS_PRESIGNED_URL_EXPIRATION_IN_SECONDS` : Expiration time of presigned urls. Default 3600.
+- `AWS_REGION` : Region of the AWS account. Can be empty.
+
+DRS Information : 
+- `DRS_ID` : DRS Server ID.
+- `DRS_NAME` : DRS Name.
+- `DRS_ORGANIZATION_NAME` : DRS Organization.
+- `DRS_ORGANIZATION_URL` : DRS Organization url. 
+- `DRS_SELF_HOST` : DRS self host, used in `self_uri` property of `DrsObject`. 
+- `DRS_VERSION` : DRS Version. Default 1.3.0
+- `DRS_CONTACT_URL` : DRS contact url. Can be empty.
+- `DRS_DESCRIPTION` : DRS description. Can be empty.
+- `DRS_DOCUMENTATION_URL` : DRS documentation url. Can be empty.
+- `DRS_ENVIRONMENT` :  DRS environment. Can be empty.
+
+HTTP Server information :
+- `HTTP_HOST` : Address HTTP server should listen to. Default 0.0.0.0 (all interfaces)
+- `HTTP_PORT`: Port HTTP server should listen to. Default 9090
+
+## Endpoints :
+
+- Swagger : /docs
+- Status : /status
+- Prometheus : /metrics
+
+## Links:
+
+* [tapir documentation](https://tapir.softwaremill.com/en/latest/)
+* [tapir github](https://github.com/softwaremill/tapir)
+* [sbtx wrapper](https://github.com/dwijnand/sbt-extras#installation)

app/services/keycloak/PermsService.scala

@@ -2,11 +2,12 @@ package services.keycloak
 
 import auth.UserRequest
 import org.keycloak.authorization.client.util.HttpResponseException
-import org.keycloak.representations.idm.authorization.UmaPermissionRepresentation
+import org.keycloak.representations.idm.authorization.{ResourceRepresentation, ScopeRepresentation, UmaPermissionRepresentation}
 import play.api.Configuration
 import play.api.mvc.AnyContent
 
 import java.util
+import java.util.{Collections, OptionalInt}
 import javax.inject.{Inject, Singleton}
 import scala.jdk.CollectionConverters._
 
@@ -41,6 +42,24 @@ class PermsService @Inject()(config: Configuration, permsClient: PermsClient) {
     }
   }
 
+  def createResources(token: String, resources: Set[CreateResource]) = {
+    val protection = permsClient.authzClient.protection(token) // insure token has resource creation rights
+    val resource = protection.resource()
+    resources.map {
+      r =>
+        val scopes: java.util.Set[ScopeRepresentation] = r.scopes.map(s => new ScopeRepresentation(s)).asJava
+        val rr = new ResourceRepresentation(r.name.getOrElse(r.id), scopes, r.uri.orNull, r.`type`.orNull)
+        rr.setId(r.id)
+        val attributes = new util.HashMap[String, util.List[String]]()
+        attributes.put("location", Collections.singletonList(r.location))
+        attributes.put("checksum", r.checksums.toList.asJava)
+        rr.setAttributes(attributes)
+        rr
+    }.map { r =>
+      resource.create(r)
+    }
+  }
+
   def createPermissions(token: String, userName: String, files: Set[String]): (Set[String], Set[String]) = {
     val protection = permsClient.authzClient.protection(token) // insure token has resource creation rights
     val resources = protection.resource()
@@ -75,3 +94,5 @@ class PermsService @Inject()(config: Configuration, permsClient: PermsClient) {
     (createdPerms, files.diff(createdPerms))
   }
 }
+
+case class CreateResource(id: String, name: Option[String], `type`: Option[String], uri: Option[String], scopes: Set[String], checksums: Set[String], location: String, size: Option[Long])

build.sbt

@@ -1,28 +1,44 @@
-name := """ferload"""
-organization := "bio.ferlab"
 
-version := "1.0-SNAPSHOT"
 
-lazy val root = (project in file(".")).enablePlugins(PlayScala)
-val keycloakVersion = "17.0.1"
+resolvers += "Sonatype Releases" at "https://s01.oss.sonatype.org/content/repositories/releases/"
+val tapirVersion = "1.7.3"
+val http4sVersion = "0.23.23"
+val sstpVersion = "3.9.0"
+scalacOptions ++= Seq("-Xmax-inlines", "100")
+lazy val rootProject = (project in file(".")).settings(
+  Seq(
+    name := "ferload",
+    version := "0.1.0-SNAPSHOT",
+    organization := "bio.ferlab",
+    scalaVersion := "3.3.1",
+    assembly / assemblyJarName := "ferload.jar",
+    assembly / assemblyMergeStrategy:= {
+      case PathList("META-INF", "mailcap") => MergeStrategy.first
+      case PathList("META-INF", "io.netty.versions.properties") => MergeStrategy.first
+      case PathList("META-INF", "maven", "org.webjars", "swagger-ui", "pom.properties") => MergeStrategy.first
+      case PathList("module-info.class") => MergeStrategy.discard
+      case x => MergeStrategy.defaultMergeStrategy(x)
+    },
+    libraryDependencies ++= Seq(
+      "com.softwaremill.sttp.tapir" %% "tapir-http4s-server" % tapirVersion,
+      "org.http4s" %% "http4s-ember-server" % http4sVersion,
+      "org.http4s" %% "http4s-ember-client" % http4sVersion,
+      "org.http4s" %% "http4s-circe" % http4sVersion,
+      "com.softwaremill.sttp.tapir" %% "tapir-prometheus-metrics" % tapirVersion,
+      "com.softwaremill.sttp.tapir" %% "tapir-swagger-ui-bundle" % tapirVersion,
+      "com.softwaremill.sttp.tapir" %% "tapir-json-circe" % tapirVersion,
+      "ch.qos.logback" % "logback-classic" % "1.4.11",
+      "com.softwaremill.sttp.client3" %% "http4s-backend" % sstpVersion,
+      "com.softwaremill.sttp.client3" %% "circe" % sstpVersion,
+      "software.amazon.awssdk" % "s3" % "2.20.150",
+      "com.github.ben-manes.caffeine" % "caffeine" % "3.1.8",
+      "com.softwaremill.sttp.tapir" %% "tapir-sttp-stub-server" % tapirVersion % Test,
+      "com.softwaremill.sttp.tapir" %% "tapir-sttp-client" % "1.7.5" % Test,
+      "com.softwaremill.sttp.tapir" %% "tapir-http4s-client" % "1.7.5" % Test,
+      "org.scalatest" %% "scalatest" % "3.2.17" % Test,
+    )
 
-scalaVersion := "2.13.9"
+  )
+)
 
-libraryDependencies += guice
-libraryDependencies += caffeine
-libraryDependencies += ws
-libraryDependencies += "software.amazon.awssdk" % "s3" % "2.17.295"
-//libraryDependencies += "com.amazonaws" % "aws-java-sdk-s3" % "1.11.880"
-libraryDependencies += "org.keycloak" % "keycloak-core" % keycloakVersion
-libraryDependencies += "org.keycloak" % "keycloak-authz-client" % keycloakVersion
-//libraryDependencies += "org.keycloak" % "keycloak-adapter-core" % keycloakVersion
-libraryDependencies += "org.mockito" % "mockito-core" % "3.8.0" % Test
-libraryDependencies += "org.scalatestplus.play" %% "scalatestplus-play" % "5.0.0" % Test
-libraryDependencies += "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.12.3"
 
-packageName in Universal := name.value
-// Adds additional packages into Twirl
-//TwirlKeys.templateImports += "bio.ferlab.controllers._"
-
-// Adds additional packages into conf/routes
-// play.sbt.routes.RoutesKeys.routesImport += "bio.ferlab.binders._"

project/build.properties

@@ -1 +1 @@
-sbt.version=1.3.13
+sbt.version=1.9.4

project/plugins.sbt

@@ -1,2 +1,3 @@
-addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.7")
-addSbtPlugin("org.foundweekends.giter8" % "sbt-giter8-scaffold" % "0.11.0")
+addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6")
+addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.5.11")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.3")