Skip to content

Commit

Permalink
Merge branch 'release/3.1.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
To-om committed Oct 30, 2020
2 parents a0f9053 + 9be2418 commit da75a6e
Show file tree
Hide file tree
Showing 31 changed files with 478 additions and 243 deletions.
12 changes: 12 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,17 @@
# Change Log

## [3.1.0](https://github.com/TheHive-Project/Cortex/milestone/27) (2020-10-30)

**Implemented enhancements:**

- Improve Docker image [\#296](https://github.com/TheHive-Project/Cortex/issues/296)
- Impossible to load catalog through a proxy [\#297](https://github.com/TheHive-Project/Cortex/issues/297)
- Update login page design [\#303](https://github.com/TheHive-Project/Cortex/issues/303)

**Fixed bugs:**

- [Bug] Cortex and boolean ConfigurationItems [\#309](https://github.com/TheHive-Project/Cortex/issues/309)

## [3.1.0-RC1](https://github.com/TheHive-Project/Cortex/milestone/21) (2020-08-13)

**Implemented enhancements:**
Expand Down
107 changes: 58 additions & 49 deletions app/org/thp/cortex/controllers/AttachmentCtrl.scala
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,21 @@ package org.thp.cortex.controllers

import java.net.URLEncoder
import java.nio.file.Files
import javax.inject.{Inject, Singleton}

import play.api.http.HttpEntity
import play.api.libs.Files.DefaultTemporaryFileCreator
import play.api.mvc._
import play.api.{mvc, Configuration}

import akka.stream.scaladsl.FileIO
import javax.inject.{Inject, Singleton}
import net.lingala.zip4j.core.ZipFile
import net.lingala.zip4j.model.ZipParameters
import net.lingala.zip4j.util.Zip4jConstants
import org.thp.cortex.models.Roles

import org.elastic4play.Timed
import org.elastic4play.controllers.{Authenticated, Renderer}
import org.elastic4play.controllers.Authenticated
import org.elastic4play.models.AttachmentAttributeFormat
import org.elastic4play.services.AttachmentSrv
import org.elastic4play.services.{AttachmentSrv, ExecutionContextSrv}
import org.thp.cortex.models.Roles
import play.api.http.HttpEntity
import play.api.libs.Files.DefaultTemporaryFileCreator
import play.api.mvc._
import play.api.{mvc, Configuration}

/**
* Controller used to access stored attachments (plain or zipped)
Expand All @@ -30,7 +28,7 @@ class AttachmentCtrl(
attachmentSrv: AttachmentSrv,
authenticated: Authenticated,
components: ControllerComponents,
renderer: Renderer
executionContextSrv: ExecutionContextSrv
) extends AbstractController(components) {

@Inject() def this(
Expand All @@ -39,9 +37,9 @@ class AttachmentCtrl(
attachmentSrv: AttachmentSrv,
authenticated: Authenticated,
components: ControllerComponents,
renderer: Renderer
executionContextSrv: ExecutionContextSrv
) =
this(configuration.get[String]("datastore.attachment.password"), tempFileCreator, attachmentSrv, authenticated, components, renderer)
this(configuration.get[String]("datastore.attachment.password"), tempFileCreator, attachmentSrv, authenticated, components, executionContextSrv)

/**
* Download an attachment, identified by its hash, in plain format
Expand All @@ -50,16 +48,25 @@ class AttachmentCtrl(
*/
@Timed("controllers.AttachmentCtrl.download")
def download(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { _ =>
if (hash.startsWith("{{")) // angularjs hack
NoContent
else if (!name.getOrElse("").intersect(AttachmentAttributeFormat.forbiddenChar).isEmpty)
mvc.Results.BadRequest("File name is invalid")
else
Result(
header = ResponseHeader(200, Map("Content-Disposition" -> s"""attachment; filename="${URLEncoder
.encode(name.getOrElse(hash), "utf-8")}"""", "Content-Transfer-Encoding" -> "binary")),
body = HttpEntity.Streamed(attachmentSrv.source(hash), None, None)
)
executionContextSrv.withDefault { implicit ec =>
if (hash.startsWith("{{")) // angularjs hack
NoContent
else if (!name.getOrElse("").intersect(AttachmentAttributeFormat.forbiddenChar).isEmpty)
mvc.Results.BadRequest("File name is invalid")
else
Result(
header = ResponseHeader(
200,
Map(
"Content-Disposition" ->
s"""attachment; filename="${URLEncoder
.encode(name.getOrElse(hash), "utf-8")}"""",
"Content-Transfer-Encoding" -> "binary"
)
),
body = HttpEntity.Streamed(attachmentSrv.source(hash), None, None)
)
}
}

/**
Expand All @@ -69,33 +76,35 @@ class AttachmentCtrl(
*/
@Timed("controllers.AttachmentCtrl.downloadZip")
def downloadZip(hash: String, name: Option[String]): Action[AnyContent] = authenticated(Roles.read) { _ =>
if (!name.getOrElse("").intersect(AttachmentAttributeFormat.forbiddenChar).isEmpty)
BadRequest("File name is invalid")
else {
val f = tempFileCreator.create("zip", hash).path
Files.delete(f)
val zipFile = new ZipFile(f.toFile)
val zipParams = new ZipParameters
zipParams.setCompressionLevel(Zip4jConstants.DEFLATE_LEVEL_FASTEST)
zipParams.setEncryptFiles(true)
zipParams.setEncryptionMethod(Zip4jConstants.ENC_METHOD_STANDARD)
zipParams.setPassword(password)
zipParams.setFileNameInZip(name.getOrElse(hash))
zipParams.setSourceExternalStream(true)
zipFile.addStream(attachmentSrv.stream(hash), zipParams)
executionContextSrv.withDefault { implicit ec =>
if (!name.getOrElse("").intersect(AttachmentAttributeFormat.forbiddenChar).isEmpty)
BadRequest("File name is invalid")
else {
val f = tempFileCreator.create("zip", hash).path
Files.delete(f)
val zipFile = new ZipFile(f.toFile)
val zipParams = new ZipParameters
zipParams.setCompressionLevel(Zip4jConstants.DEFLATE_LEVEL_FASTEST)
zipParams.setEncryptFiles(true)
zipParams.setEncryptionMethod(Zip4jConstants.ENC_METHOD_STANDARD)
zipParams.setPassword(password)
zipParams.setFileNameInZip(name.getOrElse(hash))
zipParams.setSourceExternalStream(true)
zipFile.addStream(attachmentSrv.stream(hash), zipParams)

Result(
header = ResponseHeader(
200,
Map(
"Content-Disposition" -> s"""attachment; filename="${URLEncoder.encode(name.getOrElse(hash), "utf-8")}.zip"""",
"Content-Type" -> "application/zip",
"Content-Transfer-Encoding" -> "binary",
"Content-Length" -> Files.size(f).toString
)
),
body = HttpEntity.Streamed(FileIO.fromPath(f), Some(Files.size(f)), Some("application/zip"))
)
Result(
header = ResponseHeader(
200,
Map(
"Content-Disposition" -> s"""attachment; filename="${URLEncoder.encode(name.getOrElse(hash), "utf-8")}.zip"""",
"Content-Type" -> "application/zip",
"Content-Transfer-Encoding" -> "binary",
"Content-Length" -> Files.size(f).toString
)
),
body = HttpEntity.Streamed(FileIO.fromPath(f), Some(Files.size(f)), Some("application/zip"))
)
}
}
}
}
1 change: 1 addition & 0 deletions app/org/thp/cortex/models/Migration.scala
Original file line number Diff line number Diff line change
Expand Up @@ -84,5 +84,6 @@ class Migration @Inject() (userSrv: UserSrv, organizationSrv: OrganizationSrv, w
("sequenceCounter" -> counter)
}
)
case DatabaseState(4) => Nil
}
}
2 changes: 1 addition & 1 deletion app/org/thp/cortex/models/package.scala
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
package org.thp.cortex

package object models {
val modelVersion = 4
val modelVersion = 5
}
118 changes: 118 additions & 0 deletions app/org/thp/cortex/services/CustomWSAPI.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
package org.thp.cortex.services

import scala.util.control.NonFatal

import javax.inject.{Inject, Singleton}
import play.api.inject.ApplicationLifecycle
import play.api.libs.ws._
import play.api.libs.ws.ahc.{AhcWSClient, AhcWSClientConfig, AhcWSClientConfigParser}
import play.api.{Configuration, Environment, Logger}

import akka.stream.Materializer
import com.typesafe.sslconfig.ssl.TrustStoreConfig

object CustomWSAPI {
private[CustomWSAPI] lazy val logger = Logger(getClass)

def parseWSConfig(config: Configuration): AhcWSClientConfig =
new AhcWSClientConfigParser(new WSConfigParser(config.underlying, getClass.getClassLoader).parse(), config.underlying, getClass.getClassLoader)
.parse()

def parseProxyConfig(config: Configuration): Option[WSProxyServer] =
config.getOptional[Configuration]("play.ws.proxy").map { proxyConfig
DefaultWSProxyServer(
proxyConfig.get[String]("host"),
proxyConfig.get[Int]("port"),
proxyConfig.getOptional[String]("protocol"),
proxyConfig.getOptional[String]("user"),
proxyConfig.getOptional[String]("password"),
proxyConfig.getOptional[String]("ntlmDomain"),
proxyConfig.getOptional[String]("encoding"),
proxyConfig.getOptional[Seq[String]]("nonProxyHosts")
)
}

def getWS(config: Configuration)(implicit mat: Materializer): AhcWSClient = {
val clientConfig = parseWSConfig(config)
val clientConfigWithTruststore = config.getOptional[String]("play.cert") match {
case Some(p)
logger.warn("""Use of "cert" parameter in configuration file is deprecated. Please use:
| ws.ssl {
| trustManager = {
| stores = [
| { type = "PEM", path = "/path/to/cacert.crt" },
| { type = "JKS", path = "/path/to/truststore.jks" }
| ]
| }
| }
""".stripMargin)
clientConfig.copy(
wsClientConfig = clientConfig
.wsClientConfig
.copy(
ssl = clientConfig
.wsClientConfig
.ssl
.withTrustManagerConfig(
clientConfig
.wsClientConfig
.ssl
.trustManagerConfig
.withTrustStoreConfigs(
clientConfig.wsClientConfig.ssl.trustManagerConfig.trustStoreConfigs :+ TrustStoreConfig(
filePath = Some(p),
data = None
)
)
)
)
)
case None clientConfig
}
AhcWSClient(clientConfigWithTruststore, None)
}

def getConfig(config: Configuration, path: String): Configuration =
Configuration(
config
.getOptional[Configuration](s"play.$path")
.getOrElse(Configuration.empty)
.underlying
.withFallback(config.getOptional[Configuration](path).getOrElse(Configuration.empty).underlying)
)
}

@Singleton
class CustomWSAPI(
ws: AhcWSClient,
val proxy: Option[WSProxyServer],
config: Configuration,
environment: Environment,
lifecycle: ApplicationLifecycle,
mat: Materializer
) extends WSClient {
private[CustomWSAPI] lazy val logger = Logger(getClass)

@Inject() def this(config: Configuration, environment: Environment, lifecycle: ApplicationLifecycle, mat: Materializer) =
this(CustomWSAPI.getWS(config)(mat), CustomWSAPI.parseProxyConfig(config), config, environment, lifecycle, mat)

override def close(): Unit = ws.close()

override def url(url: String): WSRequest = {
val req = ws.url(url)
proxy.fold(req)(req.withProxyServer)
}

override def underlying[T]: T = ws.underlying[T]

def withConfig(subConfig: Configuration): CustomWSAPI = {
logger.debug(s"Override WS configuration using $subConfig")
try {
new CustomWSAPI(Configuration(subConfig.underlying.atKey("play").withFallback(config.underlying)), environment, lifecycle, mat)
} catch {
case NonFatal(e)
logger.error(s"WSAPI configuration error, use default values", e)
this
}
}
}
12 changes: 10 additions & 2 deletions app/org/thp/cortex/services/DockerJobRunnerSrv.scala
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,13 @@ import org.thp.cortex.models._
import org.elastic4play.utils.RichFuture

@Singleton
class DockerJobRunnerSrv(client: DockerClient, autoUpdate: Boolean, implicit val system: ActorSystem) {
class DockerJobRunnerSrv(
client: DockerClient,
autoUpdate: Boolean,
jobBaseDirectory: Path,
dockerJobBaseDirectory: Path,
implicit val system: ActorSystem
) {

@Inject()
def this(config: Configuration, system: ActorSystem) =
Expand All @@ -37,6 +43,8 @@ class DockerJobRunnerSrv(client: DockerClient, autoUpdate: Boolean, implicit val
.useProxy(config.getOptional[Boolean]("docker.useProxy").getOrElse(false))
.build(),
config.getOptional[Boolean]("docker.autoUpdate").getOrElse(true),
Paths.get(config.get[String]("job.directory")),
Paths.get(config.get[String]("job.dockerDirectory")),
system: ActorSystem
)

Expand All @@ -60,7 +68,7 @@ class DockerJobRunnerSrv(client: DockerClient, autoUpdate: Boolean, implicit val
.builder()
.appendBinds(
Bind
.from(jobDirectory.toAbsolutePath.toString)
.from(dockerJobBaseDirectory.resolve(jobBaseDirectory.relativize(jobDirectory)).toAbsolutePath.toString)
.to("/job")
.readOnly(false)
.build()
Expand Down
3 changes: 2 additions & 1 deletion app/org/thp/cortex/services/JobRunnerSrv.scala
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ class JobRunnerSrv @Inject() (
val logger = Logger(getClass)
lazy val analyzerExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer")
lazy val responderExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("responder")
val jobDirectory: Path = Paths.get(config.get[String]("job.directory"))

private val runners: Seq[String] = config
.getOptional[Seq[String]]("job.runners")
Expand Down Expand Up @@ -89,7 +90,7 @@ class JobRunnerSrv @Inject() (
}

private def prepareJobFolder(worker: Worker, job: Job): Future[Path] = {
val jobFolder = Files.createTempDirectory(Paths.get(System.getProperty("java.io.tmpdir")), s"cortex-job-${job.id}-")
val jobFolder = Files.createTempDirectory(jobDirectory, s"cortex-job-${job.id}-")
val inputJobFolder = Files.createDirectories(jobFolder.resolve("input"))
Files.createDirectories(jobFolder.resolve("output"))

Expand Down
6 changes: 3 additions & 3 deletions app/org/thp/cortex/services/OAuth2Srv.scala
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ case class OAuth2Config(
authorizationUrl: String,
tokenUrl: String,
userUrl: String,
scope: String,
scope: Seq[String],
authorizationHeader: String,
autoupdate: Boolean,
autocreate: Boolean
Expand All @@ -41,7 +41,7 @@ object OAuth2Config {
authorizationUrl <- configuration.getOptional[String]("auth.oauth2.authorizationUrl")
tokenUrl <- configuration.getOptional[String]("auth.oauth2.tokenUrl")
userUrl <- configuration.getOptional[String]("auth.oauth2.userUrl")
scope <- configuration.getOptional[String]("auth.oauth2.scope")
scope <- configuration.getOptional[Seq[String]]("auth.oauth2.scope")
authorizationHeader = configuration.getOptional[String]("auth.oauth2.authorizationHeader").getOrElse("Bearer")
autocreate = configuration.getOptional[Boolean]("auth.sso.autocreate").getOrElse(false)
autoupdate = configuration.getOptional[Boolean]("auth.sso.autoupdate").getOrElse(false)
Expand Down Expand Up @@ -109,7 +109,7 @@ class OAuth2Srv(
private def authRedirect(oauth2Config: OAuth2Config): Result = {
val state = UUID.randomUUID().toString
val queryStringParams = Map[String, Seq[String]](
"scope" -> Seq(oauth2Config.scope),
"scope" -> Seq(oauth2Config.scope.mkString(" ")),
"response_type" -> Seq(oauth2Config.responseType),
"redirect_uri" -> Seq(oauth2Config.redirectUri),
"client_id" -> Seq(oauth2Config.clientId),
Expand Down
Loading

0 comments on commit da75a6e

Please sign in to comment.