diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a1a27d68..c72cfd5c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,16 @@ # Change Log -## [3.0.0-RC1](https://github.com/TheHive-Project/Cortex/tree/3.0.0-RC1) (2019-04-05) +## [3.0.0-RC2](https://github.com/TheHive-Project/Cortex/tree/3.0.0-RC2) (2019-05-03) + +[Full Changelog](https://github.com/TheHive-Project/Cortex/compare/3.0.0-RC1...3.0.0-RC2) + +**Fixed bugs:** + +- Unable to load Analyzers with 3.0.0 [\#185](https://github.com/TheHive-Project/Cortex/issues/185) +- Cortex will fail to run analyzers [\#182](https://github.com/TheHive-Project/Cortex/issues/182) +- Docker container exposes tcp/9000 instead of tcp/9001 [\#166](https://github.com/TheHive-Project/Cortex/issues/166) +## [3.0.0-RC1](https://github.com/TheHive-Project/Cortex/tree/3.0.0-RC1) (2019-04-05) [Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.1.3...3.0.0-RC1) **Implemented enhancements:** diff --git a/app/org/thp/cortex/models/Job.scala b/app/org/thp/cortex/models/Job.scala index 183949676..9dd9a2978 100644 --- a/app/org/thp/cortex/models/Job.scala +++ b/app/org/thp/cortex/models/Job.scala @@ -36,6 +36,7 @@ trait JobAttributes { val fromCache = optionalAttribute("fromCache", F.booleanFmt, "Indicates if cache is used", O.form) val tpe = attribute("type", F.enumFmt(WorkerType), "", O.readonly) val lbel = optionalAttribute("label", F.stringFmt, "Label of the job") + val cacheTag = optionalAttribute("cacheTag", F.stringFmt, "hash of job discriminant, used for cache", O.readonly) } @Singleton diff --git a/app/org/thp/cortex/services/JobSrv.scala b/app/org/thp/cortex/services/JobSrv.scala index fe568c2c5..7d6074c5e 100644 --- a/app/org/thp/cortex/services/JobSrv.scala +++ b/app/org/thp/cortex/services/JobSrv.scala @@ -20,6 +20,7 @@ import org.thp.cortex.models._ import org.elastic4play._ import org.elastic4play.controllers._ import org.elastic4play.services._ +import org.elastic4play.utils.Hasher @Singleton class JobSrv( @@ -243,11 +244,10 @@ class JobSrv( parameters: JsObject, label: Option[String], force: Boolean)(implicit authContext: AuthContext): Future[Job] = { - val previousJob = if (force) Future.successful(None) - else findSimilarJob(worker, dataType, dataAttachment, tlp, parameters) + val previousJob = findSimilarJob(worker, dataType, dataAttachment, tlp, parameters, force) previousJob.flatMap { - case Some(job) ⇒ Future.successful(job) - case None ⇒ isUnderRateLimit(worker).flatMap { + case Right(job) ⇒ Future.successful(job) + case Left(cacheTag) ⇒ isUnderRateLimit(worker).flatMap { case true ⇒ val fields = Fields(Json.obj( "workerDefinitionId" → worker.workerDefinitionId(), @@ -260,7 +260,8 @@ class JobSrv( "pap" → pap, "message" → message, "parameters" → parameters.toString, - "type" → worker.tpe())) + "type" → worker.tpe(), + "cacheTag" → cacheTag)) .set("label", label.map(JsString.apply)) val fieldWithData = dataAttachment match { case Left(data) ⇒ fields.set("data", data) @@ -298,28 +299,27 @@ class JobSrv( .getOrElse(Future.successful(true)) } - def findSimilarJob(worker: Worker, dataType: String, dataAttachment: Either[String, Attachment], tlp: Long, parameters: JsObject): Future[Option[Job]] = { - val cache = worker.jobCache().fold(jobCache)(_.minutes) - if (cache.length == 0 || worker.tpe() == WorkerType.responder) { + def findSimilarJob(worker: Worker, dataType: String, dataAttachment: Either[String, Attachment], tlp: Long, parameters: JsObject, force: Boolean): Future[Either[String, Job]] = { + val cacheTag = Hasher("MD5").fromString(s"${worker.id}|$dataType|$tlp|${dataAttachment.fold(data ⇒ data, attachment ⇒ attachment.id)}|$parameters").head.toString() + lazy val cache = worker.jobCache().fold(jobCache)(_.minutes) + if (force || cache.length == 0 || worker.tpe() == WorkerType.responder) { logger.info("Job cache is disabled") - Future.successful(None) + Future.successful(Left(cacheTag)) } else { import org.elastic4play.services.QueryDSL._ logger.info(s"Looking for similar job in the last ${cache.toMinutes} minutes (worker=${worker.id}, dataType=$dataType, data=$dataAttachment, tlp=$tlp, parameters=$parameters)") + val now = new Date().getTime find(and( - "workerId" ~= worker.id, + "cacheTag" ~= cacheTag, "status" ~!= JobStatus.Failure, "status" ~!= JobStatus.Deleted, - "startDate" ~>= (now - cache.toMillis), - "dataType" ~= dataType, - "tlp" ~= tlp, - dataAttachment.fold(data ⇒ "data" ~= data, attachment ⇒ "attachment.id" ~= attachment.id), - "parameters" ~= parameters.toString), Some("0-1"), Seq("-createdAt")) + "startDate" ~>= (now - cache.toMillis)), Some("0-1"), Seq("-createdAt")) ._1 .map(j ⇒ new Job(jobModel, j.attributes + ("fromCache" → JsBoolean(true)))) .runWith(Sink.headOption) + .map(_.toRight(cacheTag)) } } diff --git a/app/org/thp/cortex/services/WorkerSrv.scala b/app/org/thp/cortex/services/WorkerSrv.scala index d5ca255c8..457256964 100644 --- a/app/org/thp/cortex/services/WorkerSrv.scala +++ b/app/org/thp/cortex/services/WorkerSrv.scala @@ -160,6 +160,7 @@ class WorkerSrv @Inject() ( def readDirectory(path: Path, workerType: WorkerType.Type): Seq[WorkerDefinition] = { for { workerDir ← Files.newDirectoryStream(path).asScala.toSeq + if Files.isDirectory(workerDir) infoFile ← Files.newDirectoryStream(workerDir, "*.json").asScala workerDefinition ← readFile(infoFile, workerType) } yield workerDefinition diff --git a/docker.sbt b/docker.sbt index 1a64fabbb..cc19a9f6e 100644 --- a/docker.sbt +++ b/docker.sbt @@ -11,9 +11,9 @@ version in Docker := { } defaultLinuxInstallLocation in Docker := "/opt/cortex" dockerRepository := Some("thehiveproject") -dockerUpdateLatest := !version.value.toUpperCase.contains("RC") +dockerUpdateLatest := !version.value.toUpperCase.contains("RC") && !version.value.contains("SNAPSHOT") dockerEntrypoint := Seq("/opt/cortex/entrypoint") -dockerExposedPorts := Seq(9000) +dockerExposedPorts := Seq(9001) mappings in Docker ++= Seq( file("package/docker/entrypoint") -> "/opt/cortex/entrypoint", file("package/logback.xml") -> "/etc/cortex/logback.xml", diff --git a/version.sbt b/version.sbt index b4332167a..bbf00929c 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "3.0.0-RC1" +version in ThisBuild := "3.0.0-RC2" diff --git a/www/package.json b/www/package.json index 18ef7fd27..892c0f20c 100755 --- a/www/package.json +++ b/www/package.json @@ -1,6 +1,6 @@ { "name": "cortex", - "version": "3.0.0-RC1", + "version": "3.0.0-RC2", "description": "A powerfull observable analysis engine", "license": "AGPL-v3", "homepage": "https://github.com/TheHive-Project/Cortex",