Skip to content

Commit

Permalink
Merge branch 'release/2.1.0-RC1'
Browse files Browse the repository at this point in the history
  • Loading branch information
To-om committed Jul 31, 2018
2 parents 87aa619 + 057a054 commit e4ee899
Show file tree
Hide file tree
Showing 92 changed files with 2,707 additions and 953 deletions.
30 changes: 30 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,35 @@
# Change Log

## [2.1.0-RC1](https://github.com/TheHive-Project/Cortex/tree/2.1.0-RC1) (2018-07-31)
[Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.0.4...2.1.0-RC1)

**Implemented enhancements:**

- New TheHive-Project repository [\#112](https://github.com/TheHive-Project/Cortex/issues/112)

**Fixed bugs:**

- First analyze of a "file" always fail, must re-run the analyze a second time [\#117](https://github.com/TheHive-Project/Cortex/issues/117)
- Analyzers filter in Jobs History view is limited to 25 analyzers [\#116](https://github.com/TheHive-Project/Cortex/issues/116)
- Fix redirection from Migration page to login on 401 error [\#114](https://github.com/TheHive-Project/Cortex/issues/114)
- Analyzer Configuration Only Showing Global Configuration [\#104](https://github.com/TheHive-Project/Cortex/issues/104)

**Closed issues:**

- Automatic observables extraction from analysis reports. [\#111](https://github.com/TheHive-Project/Cortex/issues/111)
- Automated response via Cortex [\#110](https://github.com/TheHive-Project/Cortex/issues/110)
- Consider providing checksums for the release files [\#105](https://github.com/TheHive-Project/Cortex/issues/105)
- ImportError: No module named 'cortexutils' on V2.0.4 [\#102](https://github.com/TheHive-Project/Cortex/issues/102)
- Error occur from thehive project request to cortex project [\#101](https://github.com/TheHive-Project/Cortex/issues/101)
- Analyzers disappear after deactivation and can not get enabled [\#98](https://github.com/TheHive-Project/Cortex/issues/98)
- PAP as an analyzer restriction [\#65](https://github.com/TheHive-Project/Cortex/issues/65)
- Application.conf doesn't have Yeti config nor allows for API Auth [\#54](https://github.com/TheHive-Project/Cortex/issues/54)
- endless loop of cortex analyser call [\#36](https://github.com/TheHive-Project/Cortex/issues/36)

**Merged pull requests:**

- Update GitHub path [\#100](https://github.com/TheHive-Project/Cortex/pull/100) ([saadkadhi](https://github.com/saadkadhi))

## [2.0.4](https://github.com/TheHive-Project/Cortex/tree/2.0.4) (2018-04-13)
[Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.0.3...2.0.4)

Expand Down
7 changes: 3 additions & 4 deletions app/org/thp/cortex/controllers/AnalyzerConfigCtrl.scala
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
package org.thp.cortex.controllers

import javax.inject.{ Inject, Singleton }

import scala.concurrent.{ ExecutionContext, Future }

import play.api.libs.json.JsObject
import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents }

import org.thp.cortex.models.Roles
import org.thp.cortex.services.{ AnalyzerConfigSrv, BaseConfig, UserSrv }
import org.thp.cortex.models.{ BaseConfig, Roles }
import org.thp.cortex.services.{ AnalyzerConfigSrv, UserSrv }

import org.elastic4play.BadRequestError
import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer }
Expand All @@ -29,7 +28,7 @@ class AnalyzerConfigCtrl @Inject() (
}

def list(): Action[AnyContent] = authenticated(Roles.orgAdmin).async { request
analyzerConfigSrv.listForUser(request.userId)
analyzerConfigSrv.listConfigForUser(request.userId)
.map { bc
renderer.toOutput(OK, bc.sortWith {
case (BaseConfig("global", _, _, _), _) true
Expand Down
75 changes: 39 additions & 36 deletions app/org/thp/cortex/controllers/AnalyzerCtrl.scala
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
package org.thp.cortex.controllers

import javax.inject.{ Inject, Singleton }

import scala.concurrent.{ ExecutionContext, Future }

import play.api.libs.json.{ JsObject, Json }
import play.api.libs.json.{ JsNumber, JsObject, JsString, Json }
import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents }

import akka.stream.Materializer
import akka.stream.scaladsl.Sink
import org.thp.cortex.models.{ Analyzer, AnalyzerDefinition, Roles }
import org.thp.cortex.services.{ AnalyzerSrv, UserSrv }
import org.thp.cortex.models.{ Roles, Worker, WorkerDefinition }
import org.thp.cortex.services.{ UserSrv, WorkerSrv }

import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer }
import org.elastic4play.models.JsonFormat.baseModelEntityWrites
Expand All @@ -19,7 +18,7 @@ import org.elastic4play.services.{ QueryDSL, QueryDef }

@Singleton
class AnalyzerCtrl @Inject() (
analyzerSrv: AnalyzerSrv,
workerSrv: WorkerSrv,
userSrv: UserSrv,
authenticated: Authenticated,
fieldsBodyParser: FieldsBodyParser,
Expand All @@ -33,54 +32,56 @@ class AnalyzerCtrl @Inject() (
val range = request.body.getString("range")
val sort = request.body.getStrings("sort").getOrElse(Nil)
val isAdmin = request.roles.contains(Roles.orgAdmin)
val (analyzers, analyzerTotal) = analyzerSrv.findForUser(request.userId, query, range, sort)
val (analyzers, analyzerTotal) = workerSrv.findAnalyzersForUser(request.userId, query, range, sort)
val enrichedAnalyzers = analyzers.mapAsync(2)(analyzerJson(isAdmin))
renderer.toOutput(OK, enrichedAnalyzers, analyzerTotal)
}

def get(analyzerId: String): Action[AnyContent] = authenticated(Roles.read).async { request
val isAdmin = request.roles.contains(Roles.orgAdmin)
analyzerSrv.getForUser(request.userId, analyzerId)
workerSrv.getForUser(request.userId, analyzerId)
.flatMap(analyzerJson(isAdmin))
.map(renderer.toOutput(OK, _))
}

private val emptyAnalyzerDefinitionJson = Json.obj(
"version" -> "0.0",
"description" -> "unknown",
"dataTypeList" -> Nil,
"author" -> "unknown",
"url" -> "unknown",
"license" -> "unknown")

private def analyzerJson(analyzer: Analyzer, analyzerDefinition: Option[AnalyzerDefinition]) = {
"version" "0.0",
"description" "unknown",
"dataTypeList" Nil,
"author" "unknown",
"url" "unknown",
"license" "unknown")

private def analyzerJson(analyzer: Worker, analyzerDefinition: Option[WorkerDefinition]) = {
analyzer.toJson ++ analyzerDefinition.fold(emptyAnalyzerDefinitionJson) { ad
Json.obj(
"version" -> ad.version,
"description" -> ad.description,
"author" -> ad.author,
"url" -> ad.url,
"license" -> ad.license,
"baseConfig" -> ad.baseConfiguration)
}
"maxTlp" (analyzer.config \ "max_tlp").asOpt[JsNumber],
"maxPap" (analyzer.config \ "max_pap").asOpt[JsNumber],
"version" ad.version,
"description" ad.description,
"author" ad.author,
"url" ad.url,
"license" ad.license,
"baseConfig" ad.baseConfiguration)
} + ("analyzerDefinitionId" JsString(analyzer.workerDefinitionId())) // For compatibility reason
}

private def analyzerJson(isAdmin: Boolean)(analyzer: Analyzer): Future[JsObject] = {
analyzerSrv.getDefinition(analyzer.analyzerDefinitionId())
private def analyzerJson(isAdmin: Boolean)(analyzer: Worker): Future[JsObject] = {
workerSrv.getDefinition(analyzer.workerDefinitionId())
.map(analyzerDefinition analyzerJson(analyzer, Some(analyzerDefinition)))
.recover { case _ analyzerJson(analyzer, None) }
.map {
case a if isAdmin a + ("configuration" -> Json.parse(analyzer.configuration()))
case a if isAdmin a + ("configuration" Json.parse(analyzer.configuration()))
case a a
}
}

def listForType(dataType: String): Action[AnyContent] = authenticated(Roles.read).async { request
import org.elastic4play.services.QueryDSL._
analyzerSrv.findForUser(request.userId, "dataTypeList" ~= dataType, Some("all"), Nil)
workerSrv.findAnalyzersForUser(request.userId, "dataTypeList" ~= dataType, Some("all"), Nil)
._1
.mapAsyncUnordered(2) { analyzer
analyzerSrv.getDefinition(analyzer.analyzerDefinitionId())
workerSrv.getDefinition(analyzer.workerDefinitionId())
.map(ad analyzerJson(analyzer, Some(ad)))
}
.runWith(Sink.seq)
Expand All @@ -90,31 +91,33 @@ class AnalyzerCtrl @Inject() (
def create(analyzerDefinitionId: String): Action[Fields] = authenticated(Roles.orgAdmin).async(fieldsBodyParser) { implicit request
for {
organizationId userSrv.getOrganizationId(request.userId)
analyzer analyzerSrv.create(organizationId, analyzerDefinitionId, request.body)
} yield renderer.toOutput(CREATED, analyzer)
workerDefinition workerSrv.getDefinition(analyzerDefinitionId)
analyzer workerSrv.create(organizationId, workerDefinition, request.body)
} yield renderer.toOutput(CREATED, analyzerJson(analyzer, Some(workerDefinition)))
}

def listDefinitions: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { implicit request
val (analyzers, analyzerTotal) = analyzerSrv.listDefinitions
val (analyzers, analyzerTotal) = workerSrv.listAnalyzerDefinitions
renderer.toOutput(OK, analyzers, analyzerTotal)
}

def scan: Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin) { implicit request
analyzerSrv.rescan()
workerSrv.rescan()
NoContent
}

def delete(analyzerId: String): Action[AnyContent] = authenticated(Roles.orgAdmin, Roles.superAdmin).async { implicit request
for {
analyzer analyzerSrv.getForUser(request.userId, analyzerId)
_ analyzerSrv.delete(analyzer)
analyzer workerSrv.getForUser(request.userId, analyzerId)
_ workerSrv.delete(analyzer)
} yield NoContent
}

def update(analyzerId: String): Action[Fields] = authenticated(Roles.orgAdmin).async(fieldsBodyParser) { implicit request
for {
analyzer analyzerSrv.getForUser(request.userId, analyzerId)
updatedAnalyzer analyzerSrv.update(analyzer, request.body)
} yield renderer.toOutput(OK, updatedAnalyzer)
analyzer workerSrv.getForUser(request.userId, analyzerId)
updatedAnalyzer workerSrv.update(analyzer, request.body)
updatedAnalyzerJson analyzerJson(isAdmin = true)(updatedAnalyzer)
} yield renderer.toOutput(OK, updatedAnalyzerJson)
}
}
64 changes: 38 additions & 26 deletions app/org/thp/cortex/controllers/JobCtrl.scala
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ class JobCtrl @Inject() (
implicit val mat: Materializer,
implicit val actorSystem: ActorSystem) extends AbstractController(components) with Status {

def list(dataTypeFilter: Option[String], dataFilter: Option[String], analyzerFilter: Option[String], range: Option[String]): Action[AnyContent] = authenticated(Roles.read).async { implicit request
val (jobs, jobTotal) = jobSrv.listForUser(request.userId, dataTypeFilter, dataFilter, analyzerFilter, range)
def list(dataTypeFilter: Option[String], dataFilter: Option[String], workerFilter: Option[String], range: Option[String]): Action[AnyContent] = authenticated(Roles.read).async { implicit request
val (jobs, jobTotal) = jobSrv.listForUser(request.userId, dataTypeFilter, dataFilter, workerFilter, range)
renderer.toOutput(OK, jobs, jobTotal)
}

Expand All @@ -63,8 +63,20 @@ class JobCtrl @Inject() (
.map(_ NoContent)
}

def create(analyzerId: String): Action[Fields] = authenticated(Roles.analyze).async(fieldsBodyParser) { implicit request
jobSrv.create(analyzerId, request.body)
def createResponderJob(workerId: String): Action[Fields] = authenticated(Roles.analyze).async(fieldsBodyParser) { implicit request
val fields = request.body
val fieldsWithStringData = fields.getValue("data") match {
case Some(d) fields.set("data", d.toString)
case None fields
}
jobSrv.create(workerId, fieldsWithStringData)
.map { job
renderer.toOutput(OK, job)
}
}

def createAnalyzerJob(workerId: String): Action[Fields] = authenticated(Roles.analyze).async(fieldsBodyParser) { implicit request
jobSrv.create(workerId, request.body)
.map { job
renderer.toOutput(OK, job)
}
Expand All @@ -84,42 +96,44 @@ class JobCtrl @Inject() (
.collect {
case artifact if artifact.data().isDefined
Json.obj(
"data" -> artifact.data(),
"dataType" -> artifact.dataType(),
"message" -> artifact.message(),
"tags" -> artifact.tags(),
"tlp" -> artifact.tlp())
"data" artifact.data(),
"dataType" artifact.dataType(),
"message" artifact.message(),
"tags" artifact.tags(),
"tlp" artifact.tlp())
case artifact if artifact.attachment().isDefined
artifact.attachment().fold(JsObject.empty) { a
Json.obj(
"attachment" ->
"attachment"
Json.obj(
"contentType" -> a.contentType,
"id" -> a.id,
"name" -> a.name,
"size" -> a.size),
"message" -> artifact.message(),
"tags" -> artifact.tags(),
"tlp" -> artifact.tlp())
"contentType" a.contentType,
"id" a.id,
"name" a.name,
"size" a.size),
"message" artifact.message(),
"tags" artifact.tags(),
"tlp" artifact.tlp())
}
}
.runWith(Sink.seq)
} yield Json.obj(
"summary" -> Json.parse(report.summary()),
"full" -> Json.parse(report.full()),
"success" -> true,
"artifacts" -> artifacts)
case JobStatus.InProgress Future.successful(JsString("Running"))
"summary" Json.parse(report.summary()),
"full" Json.parse(report.full()),
"success" true,
"artifacts" artifacts,
"operations" Json.parse(report.operations()))
case JobStatus.Failure
val errorMessage = job.errorMessage().getOrElse("")
Future.successful(Json.obj(
"errorMessage" errorMessage,
"input" job.input(),
"success" false))
case JobStatus.Waiting Future.successful(JsString("Waiting"))
case JobStatus.InProgress Future.successful(JsString("Running"))
case JobStatus.Waiting Future.successful(JsString("Waiting"))
case JobStatus.Deleted Future.successful(JsString("Deleted"))
})
.map { report
Json.toJson(job).as[JsObject] + ("report" -> report)
Json.toJson(job).as[JsObject] + ("report" report)
}
}

Expand All @@ -131,7 +145,6 @@ class JobCtrl @Inject() (
jobSrv.getForUser(request.userId, jobId)
.flatMap {
case job if job.status() == JobStatus.InProgress || job.status() == JobStatus.Waiting
println(s"job status is ${job.status()} => wait")
val duration = Duration(atMost).asInstanceOf[FiniteDuration]
implicit val timeout: Timeout = Timeout(duration)
(auditActor ? Register(jobId, duration))
Expand All @@ -140,7 +153,6 @@ class JobCtrl @Inject() (
.withTimeout(duration, ())
.flatMap(_ getJobWithReport(request.userId, jobId))
case job
println(s"job status is ${job.status()} => send it directly")
getJobWithReport(request.userId, job)
}
.map(Ok(_))
Expand Down
4 changes: 2 additions & 2 deletions app/org/thp/cortex/controllers/MispCtrl.scala
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package org.thp.cortex.controllers
import javax.inject.Inject
import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer }
import org.thp.cortex.models.Roles
import org.thp.cortex.services.{ AnalyzerSrv, MispSrv }
import org.thp.cortex.services.{ WorkerSrv, MispSrv }
import play.api.Logger
import play.api.libs.json.{ JsObject, JsValue }
import play.api.mvc._
Expand All @@ -12,7 +12,7 @@ import scala.concurrent.{ ExecutionContext, Future }

class MispCtrl @Inject() (
mispSrv: MispSrv,
analyzerSrv: AnalyzerSrv,
analyzerSrv: WorkerSrv,
authenticated: Authenticated,
fieldsBodyParser: FieldsBodyParser,
renderer: Renderer,
Expand Down
48 changes: 48 additions & 0 deletions app/org/thp/cortex/controllers/ResponderConfigCtrl.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package org.thp.cortex.controllers

import scala.concurrent.{ ExecutionContext, Future }

import play.api.libs.json.JsObject
import play.api.mvc.{ AbstractController, Action, AnyContent, ControllerComponents }

import javax.inject.{ Inject, Singleton }
import org.thp.cortex.models.{ BaseConfig, Roles }
import org.thp.cortex.services.{ ResponderConfigSrv, UserSrv }

import org.elastic4play.BadRequestError
import org.elastic4play.controllers.{ Authenticated, Fields, FieldsBodyParser, Renderer }

@Singleton
class ResponderConfigCtrl @Inject() (
responderConfigSrv: ResponderConfigSrv,
userSrv: UserSrv,
authenticated: Authenticated,
fieldsBodyParser: FieldsBodyParser,
renderer: Renderer,
components: ControllerComponents,
implicit val ec: ExecutionContext) extends AbstractController(components) {

def get(analyzerConfigName: String): Action[AnyContent] = authenticated(Roles.orgAdmin).async { request
responderConfigSrv.getForUser(request.userId, analyzerConfigName)
.map(renderer.toOutput(OK, _))
}

def list(): Action[AnyContent] = authenticated(Roles.orgAdmin).async { request
responderConfigSrv.listConfigForUser(request.userId)
.map { bc
renderer.toOutput(OK, bc.sortWith {
case (BaseConfig("global", _, _, _), _) true
case (_, BaseConfig("global", _, _, _)) false
case (BaseConfig(a, _, _, _), BaseConfig(b, _, _, _)) a.compareTo(b) < 0
})
}
}

def update(analyzerConfigName: String): Action[Fields] = authenticated(Roles.orgAdmin).async(fieldsBodyParser) { implicit request
request.body.getValue("config").flatMap(_.asOpt[JsObject]) match {
case Some(config) responderConfigSrv.updateOrCreate(request.userId, analyzerConfigName, config)
.map(renderer.toOutput(OK, _))
case None Future.failed(BadRequestError("attribute config has invalid format"))
}
}
}
Loading

0 comments on commit e4ee899

Please sign in to comment.