Merge branch 'hotfix/1.1.1'

TheHive-Project · May 17, 2017 · 3ce3ae2 · 3ce3ae2
2 parents 171d525 + 5857787
commit 3ce3ae2
Show file tree

Hide file tree

Showing 42 changed files with 334 additions and 1,629 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,14 @@
 # Change Log
 
+## [1.1.1](https://github.com/CERT-BDF/Cortex/tree/1.1.1) (2017-05-17)
+
+[Full Changelog](https://github.com/CERT-BDF/Cortex/compare/1.1.0...1.1.1)
+
+**Implemented enhancements:**
+
+- Missing logos and favicons [\#25](https://github.com/CERT-BDF/Cortex/issues/25)
+- MISP integration feature request [\#21](https://github.com/CERT-BDF/Cortex/issues/21)
+
 ## [1.1.0](https://github.com/CERT-BDF/Cortex/tree/1.1.0) (2017-05-12)
 
 [Full Changelog](https://github.com/CERT-BDF/Cortex/compare/1.0.2...1.1.0)
@@ -8,7 +17,6 @@
 
 - Add support to .deb and .rpm package generation [\#20](https://github.com/CERT-BDF/Cortex/issues/20)
 - Display analyzers metadata [\#18](https://github.com/CERT-BDF/Cortex/issues/18)
-- MISP integration feature request [\#21](https://github.com/CERT-BDF/Cortex/issues/21)
 
 **Closed issues:**
 

diff --git a/app/models/JsonFormat.scala b/app/models/JsonFormat.scala
@@ -18,7 +18,8 @@ object JsonFormat {
   implicit val fileArtifactWrites: OWrites[FileArtifact] = OWrites[FileArtifact](fileArtifact ⇒ Json.obj(
     "attributes" → fileArtifact.attributes))
 
-  implicit val dataArtifactWrites: OWrites[DataArtifact] = Json.writes[DataArtifact]
+  implicit val dataArtifactWrites: OWrites[DataArtifact] = OWrites[DataArtifact](artifact ⇒
+    artifact.attributes + ("data" → JsString(artifact.data)))
   implicit val dataActifactReads: Reads[DataArtifact] = Json.reads[DataArtifact]
 
   val artifactWrites: OWrites[Artifact] = OWrites[Artifact] {

diff --git a/app/models/MispModule.scala b/app/models/MispModule.scala
@@ -1,13 +1,15 @@
 package models
 
+import play.api.libs.json.JsObject
+
 case class MispModule(
     name: String,
     version: String,
     description: String,
     author: String,
     dataTypeList: Seq[String],
     inputAttributes: Seq[String],
-    config: Seq[String],
+    config: JsObject,
     loaderCommand: String) extends Analyzer {
   val license = "AGPL-3.0"
   val url = "https://github.com/MISP/misp-modules"

diff --git a/app/services/MispSrv.scala b/app/services/MispSrv.scala
@@ -1,12 +1,13 @@
 package services
 
 import java.io.{ ByteArrayInputStream, FileInputStream, InputStream, SequenceInputStream }
-import javax.inject.Inject
+import javax.inject.{ Inject, Singleton }
 
 import akka.actor.ActorSystem
 import models.JsonFormat._
 import models._
 import org.apache.commons.codec.binary.{ Base64, Base64InputStream }
+import util.JsonConfig
 import play.api.libs.json.{ Json, _ }
 import play.api.{ Configuration, Logger }
 
@@ -15,8 +16,11 @@ import scala.concurrent.{ ExecutionContext, Future }
 import scala.sys.process._
 import scala.util.{ Failure, Success, Try }
 
+@Singleton
 class MispSrv(
-    loaderCommandOption: Option[String],
+    mispModulesEnabled: Boolean,
+    loaderCommand: String,
+    mispModuleConfig: JsObject,
     externalAnalyzerSrv: ExternalAnalyzerSrv,
     jobSrv: JobSrv,
     akkaSystem: ActorSystem) {
@@ -26,33 +30,37 @@ class MispSrv(
     externalAnalyzerSrv: ExternalAnalyzerSrv,
     jobSrv: JobSrv,
     akkaSystem: ActorSystem) = this(
-    configuration.getString("misp.modules.loader"),
+    configuration.getBoolean("misp.modules.enabled").getOrElse(false),
+    configuration.getString("misp.modules.loader").get,
+    JsonConfig.configWrites.writes(configuration.getConfig("misp.modules.config").getOrElse(Configuration.empty)),
     externalAnalyzerSrv,
     jobSrv,
     akkaSystem)
 
   private[MispSrv] lazy val logger = Logger(getClass)
   private[MispSrv] lazy val analyzeExecutionContext: ExecutionContext = akkaSystem.dispatchers.lookup("analyzer")
 
-  lazy val list: Seq[MispModule] =
-    loaderCommandOption.fold(Seq.empty[MispModule]) { loaderCommand ⇒
-      Json.parse(s"$loaderCommand --list".!!)
-        .as[Seq[String]]
-        .map { moduleName ⇒
-          moduleName → (for {
-            moduleInfo ← Try(Json.parse(s"$loaderCommand --info $moduleName".!!))
-            module ← Try(moduleInfo.as[MispModule](reads(loaderCommand)))
-          } yield module)
-        }
-        .flatMap {
-          case (moduleName, Failure(error)) ⇒
-            logger.warn(s"Load MISP module $moduleName fails", error)
-            Nil
-          case (_, Success(module)) ⇒
-            logger.info(s"Register MISP module ${module.name} ${module.version}")
-            Seq(module)
-        }
-    }
+  logger.info(s"MISP modules is ${if (mispModulesEnabled) "enabled" else "disabled"}, loader is $loaderCommand")
+
+  lazy val list: Seq[MispModule] = if (mispModulesEnabled) {
+    Json.parse(s"$loaderCommand --list".!!)
+      .as[Seq[String]]
+      .map { moduleName ⇒
+        moduleName → (for {
+          moduleInfo ← Try(Json.parse(s"$loaderCommand --info $moduleName".!!))
+          module ← Try(moduleInfo.as[MispModule](reads(loaderCommand, mispModuleConfig)))
+        } yield module)
+      }
+      .flatMap {
+        case (moduleName, Failure(error)) ⇒
+          logger.warn(s"Load MISP module $moduleName fails", error)
+          Nil
+        case (_, Success(module)) ⇒
+          logger.info(s"Register MISP module ${module.name} ${module.version}")
+          Seq(module)
+      }
+  }
+  else Nil
 
   def get(moduleName: String): Option[MispModule] = list.find(_.name == moduleName)
 
@@ -89,30 +97,31 @@ class MispSrv(
   }
 
   def query(module: String, mispType: String, data: String)(implicit ec: ExecutionContext): Future[JsObject] = {
-    loaderCommandOption
-      .flatMap { loaderCommand ⇒
-        val artifact = toArtifact(mispType, data)
-        get(module)
-          .map { mispModule ⇒
-            val mispReport = Future {
-              val input = Json.obj(mispType → data)
-              val output = (s"$loaderCommand --run $module" #< input.toString).!!
-              Json.parse(output).as[JsObject]
-            }
-            jobSrv.create(mispModule, artifact, mispReport.map(toReport))
-            mispReport
-
+    val artifact = toArtifact(mispType, data)
+    val mispModule = if (mispModulesEnabled) {
+      get(module)
+        .map { mispModule ⇒
+          val mispReport = Future {
+            val input = Json.obj("config" → mispModule.config, mispType → data)
+            val output = (s"$loaderCommand --run $module" #< input.toString).!!
+            Json.parse(output).as[JsObject]
           }
-          .orElse {
-            externalAnalyzerSrv
-              .get(module)
-              .map { analyzer ⇒
-                externalAnalyzerSrv.analyze(analyzer, artifact)
-                  .map { report ⇒ toMispOutput(report) }
-              }
+          jobSrv.create(mispModule, artifact, mispReport.map(toReport))
+          mispReport
+
+        }
+    }
+    else None
+    mispModule
+      .orElse {
+        externalAnalyzerSrv
+          .get(module)
+          .map { analyzer ⇒
+            externalAnalyzerSrv.analyze(analyzer, artifact)
+              .map { report ⇒ toMispOutput(report) }
           }
       }
-      .getOrElse(Future.failed(new Exception(s"Module $module not found")))
+      .getOrElse(Future.failed(new Exception(s"Module $module not found"))) // TODO add appropriate exception
   }
 
   def analyze(module: MispModule, artifact: Artifact): Future[Report] = {
@@ -121,10 +130,13 @@ class MispSrv(
 
     val input = artifact match {
       case DataArtifact(data, _) ⇒
-        stringStream(Json.obj(dataType2mispType(artifact.dataType).head → data).toString)
+        val mispType = dataType2mispType(artifact.dataType)
+          .filter(module.inputAttributes.contains)
+          .head
+        stringStream((Json.obj("config" → module.config) + (mispType → JsString(data))).toString)
       case FileArtifact(data, _) ⇒
         new SequenceInputStream(Iterator(
-          stringStream("""{"attachment":""""),
+          stringStream(Json.obj("config" → module.config).toString.replaceFirst("}$", ""","attachment":"""")),
           new Base64InputStream(new FileInputStream(data), true),
           stringStream("\"}")).asJavaEnumeration)
     }
@@ -207,15 +219,26 @@ class MispSrv(
     else mispTypes
   }
 
-  private def reads(loaderCommand: String): Reads[MispModule] =
+  private def reads(loaderCommand: String, mispModuleConfig: JsObject): Reads[MispModule] =
     for {
       name ← (__ \ "name").read[String]
-      version ← (__ \ "meta" \ "version").read[String]
-      description ← (__ \ "meta" \ "description").read[String]
-      author ← (__ \ "meta" \ "author").read[String]
-      config ← (__ \ "meta" \ "config").read[Seq[String]]
+      version ← (__ \ "moduleinfo" \ "version").read[String]
+      description ← (__ \ "moduleinfo" \ "description").read[String]
+      author ← (__ \ "moduleinfo" \ "author").read[String]
+      config = (mispModuleConfig \ name).asOpt[JsObject].getOrElse(JsObject(Nil))
+      requiredConfig ← (__ \ "config").read[Set[String]]
+      missingConfig = requiredConfig -- config.keys
+      _ ← if (missingConfig.nonEmpty) {
+        val message = s"MISP module $name is disabled because the following configuration " +
+          s"item${if (missingConfig.size > 1) "s are" else " is"} missing: ${missingConfig.mkString(", ")}"
+        logger.warn(message)
+        Reads[Unit](_ ⇒ JsError(message))
+      }
+      else {
+        Reads[Unit](_ ⇒ JsSuccess(()))
+      }
       input ← (__ \ "mispattributes" \ "input").read[Seq[String]]
-      dataTypes = input.map(mispType2dataType)
+      dataTypes = input.map(mispType2dataType).distinct
     } yield MispModule(name, version, description, author, dataTypes, input, config, loaderCommand)
 
   private val typeLookup = Map(

diff --git a/build.sbt b/build.sbt
@@ -43,14 +43,21 @@ mappings in Universal ~= {
     file("package/cortex.service") -> "package/cortex.service",
     file("package/cortex.conf") -> "package/cortex.conf",
     file("package/cortex") -> "package/cortex",
-    file("package/logback.xml") -> "conf/logback.xml"
+    file("package/logback.xml") -> "conf/logback.xml",
+    file("contrib/misp-modules-loader.py") -> "contrib/misp-modules-loader.py"
   )
 }
 
 // Package //
-maintainer := "Thomas Franco <[email protected]"
-packageSummary := "-"
-packageDescription := """--""".stripMargin
+maintainer := "TheHive Project <[email protected]>"
+packageSummary := "Powerful Observable Analysis Engine"
+packageDescription := """Cortex tries to solve a common problem frequently encountered by SOCs, CSIRTs and security
+  | researchers in the course of threat intelligence, digital forensics and incident response: how to analyze
+  | observables they have collected, at scale, by querying a single tool instead of several?
+  |
+  | Cortex, an open source and free software, has been created by TheHive Project for this very purpose. Observables,
+  | such as IP and email addresses, URLs, domain names, files or hashes, can be analyzed one by one or in bulk mode
+  | using a Web interface. Analysts can also automate these operations thanks to the Cortex REST API. """.stripMargin
 defaultLinuxInstallLocation := "/opt"
 linuxPackageMappings ~= { _.map { pm =>
   val mappings = pm.mappings.filterNot {
@@ -62,7 +69,7 @@ linuxPackageMappings ~= { _.map { pm =>
   file("package/cortex.conf") -> "/etc/init/cortex.conf",
   file("package/cortex") -> "/etc/init.d/cortex",
   file("conf/application.sample") -> "/etc/cortex/application.conf",
-  file("conf/logback.xml") -> "/etc/cortex/logback.xml"
+  file("package/logback.xml") -> "/etc/cortex/logback.xml"
 ).withConfig()
 }
 
@@ -125,7 +132,11 @@ dockerCommands ~= { dc =>
           "apt-get install -y --no-install-recommends python-pip python2.7-dev ssdeep libfuzzy-dev libfuzzy2 libimage-exiftool-perl libmagic1 build-essential git && " +
           "cd /opt && " +
           "git clone https://github.com/CERT-BDF/Cortex-Analyzers.git && " +
-          "pip install $(sort -u Cortex-Analyzers/analyzers/*/requirements.txt)"),
+          "pip install $(sort -u Cortex-Analyzers/analyzers/*/requirements.txt) && " +
+          "apt-get install -y --no-install-recommends python3-setuptools python3-dev zlib1g-dev libxslt1-dev libxml2-dev libpq5 libjpeg-dev && git clone https://github.com/MISP/misp-modules.git && " +
+          "easy_install3 pip && " +
+          "(cd misp-modules && pip3 install -I -r REQUIREMENTS && pip3 install -I .) && " +
+          "rm -rf misp_modules /var/lib/apt/lists/* /tmp/*"),
       Cmd("ADD", "var", "/var"),
       Cmd("ADD", "etc", "/etc"),
       ExecCmd("RUN", "chown", "-R", "daemon:daemon", "/var/log/cortex")) ++