Merge branch 'release/2.1.0'

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Change Log
 
+## [Unreleased](https://github.com/TheHive-Project/Cortex/tree/HEAD)
+
+[Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.1.0-RC1...HEAD)
+
+**Implemented enhancements:**
+
+- Show PAP value in the Org \> Analyzers screen [\#124](https://github.com/TheHive-Project/Cortex/issues/124)
+- Display cache configuration in analyzer admin page [\#123](https://github.com/TheHive-Project/Cortex/issues/123)
+
+**Fixed bugs:**
+
+- Temporary files are not removed at the end of job [\#129](https://github.com/TheHive-Project/Cortex/issues/129)
+- MISP fails to run analyzers [\#128](https://github.com/TheHive-Project/Cortex/issues/128)
+- MISP API fails [\#109](https://github.com/TheHive-Project/Cortex/issues/109)
+- File\_Info issue [\#53](https://github.com/TheHive-Project/Cortex/issues/53)
+
+**Merged pull requests:**
+
+- Update resolvers in build.sbt to contain Maven as a dependency [\#130](https://github.com/TheHive-Project/Cortex/pull/130) ([adl1995](https://github.com/adl1995))
+
 ## [2.1.0-RC1](https://github.com/TheHive-Project/Cortex/tree/2.1.0-RC1) (2018-07-31)
 [Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.0.4...2.1.0-RC1)
 
@@ -9,22 +29,22 @@
 
 **Fixed bugs:**
 
+- Analyzer Configuration Only Showing Global Configuration [\#104](https://github.com/TheHive-Project/Cortex/issues/104)
 - First analyze of a "file" always fail, must re-run the analyze a second time [\#117](https://github.com/TheHive-Project/Cortex/issues/117)
 - Analyzers filter in Jobs History view is limited to 25 analyzers [\#116](https://github.com/TheHive-Project/Cortex/issues/116)
 - Fix redirection from Migration page to login on 401 error [\#114](https://github.com/TheHive-Project/Cortex/issues/114)
-- Analyzer Configuration Only Showing Global Configuration [\#104](https://github.com/TheHive-Project/Cortex/issues/104)
 
 **Closed issues:**
 
 - Automatic observables extraction from analysis reports. [\#111](https://github.com/TheHive-Project/Cortex/issues/111)
-- Automated response via Cortex [\#110](https://github.com/TheHive-Project/Cortex/issues/110)
-- Consider providing checksums for the release files [\#105](https://github.com/TheHive-Project/Cortex/issues/105)
 - ImportError: No module named 'cortexutils' on V2.0.4 [\#102](https://github.com/TheHive-Project/Cortex/issues/102)
 - Error occur from thehive project request to cortex project [\#101](https://github.com/TheHive-Project/Cortex/issues/101)
 - Analyzers disappear after deactivation and can not get enabled [\#98](https://github.com/TheHive-Project/Cortex/issues/98)
-- PAP as an analyzer restriction [\#65](https://github.com/TheHive-Project/Cortex/issues/65)
 - Application.conf doesn't have Yeti config nor allows for API Auth [\#54](https://github.com/TheHive-Project/Cortex/issues/54)
 - endless loop of cortex analyser call [\#36](https://github.com/TheHive-Project/Cortex/issues/36)
+- Automated response via Cortex [\#110](https://github.com/TheHive-Project/Cortex/issues/110)
+- Consider providing checksums for the release files [\#105](https://github.com/TheHive-Project/Cortex/issues/105)
+- PAP as an analyzer restriction [\#65](https://github.com/TheHive-Project/Cortex/issues/65)
 
 **Merged pull requests:**
 

app/org/thp/cortex/services/JobSrv.scala

@@ -1,10 +1,10 @@
 package org.thp.cortex.services
 
 import java.io.{ ByteArrayOutputStream, InputStream }
-import java.nio.file.Files
+import java.nio.file.{ Files, Paths }
 import java.util.Date
-import javax.inject.{ Inject, Singleton }
 
+import javax.inject.{ Inject, Singleton }
 import akka.NotUsed
 import akka.actor.ActorSystem
 import akka.stream.Materializer
@@ -403,6 +403,11 @@ class JobSrv(
             val errorMessage = (error + output).take(8192)
             endJob(job, JobStatus.Failure, Some(s"Invalid output\n$errorMessage"))
         }
+        finally {
+          (input \ "file").asOpt[String].foreach { filename ⇒
+            Files.deleteIfExists(Paths.get(filename))
+          }
+        }
       }(executionContext)
   }
 

app/org/thp/cortex/services/MispSrv.scala

@@ -17,6 +17,7 @@ import org.apache.commons.codec.binary.Base64
 import org.thp.cortex.models._
 import org.thp.cortex.services.AuditActor.Register
 
+import org.elastic4play.NotFoundError
 import org.elastic4play.services._
 
 @Singleton
@@ -58,12 +59,15 @@ class MispSrv @Inject() (
   }
 
   def query(module: String, mispType: String, data: String)(implicit authContext: AuthContext): Future[JsObject] = {
+    import org.elastic4play.services.QueryDSL._
+
     val artifact: Either[String, Attachment] = toArtifact(mispType, data)
     val duration = 20.minutes // TODO configurable
 
     for {
-      analyzer ← workerSrv.get(module)
-      job ← jobSrv.create(analyzer, mispType2dataType(mispType), artifact, 0, 0, "", JsObject.empty, None, force = false)
+      analyzer ← workerSrv.findAnalyzersForUser(authContext.userId, "name" ~= module, Some("0-1"), Nil)._1.runWith(Sink.headOption)
+      job ← analyzer.map(jobSrv.create(_, mispType2dataType(mispType), artifact, 0, 0, "", JsObject.empty, None, force = false))
+        .getOrElse(Future.failed(NotFoundError(s"Module $module not found")))
       _ ← auditActor.ask(Register(job.id, duration))(Timeout(duration))
       updatedJob ← jobSrv.getForUser(authContext.userId, job.id)
       mispOutput ← toMispOutput(authContext.userId, updatedJob)

build.sbt

@@ -18,6 +18,7 @@ libraryDependencies ++= Seq(
 
 resolvers += Resolver.sbtPluginRepo("releases")
 resolvers += "scalaz-bintray" at "http://dl.bintray.com/scalaz/releases"
+resolvers += "elasticsearch-releases" at "https://artifacts.elastic.co/maven"
 publishArtifact in (Compile, packageDoc) := false
 publishArtifact in packageDoc := false
 sources in (Compile,doc) := Seq.empty

docker.sbt

@@ -28,8 +28,9 @@ dockerCommands ~= { dc =>
       ExecCmd("RUN", "bash", "-c",
         "apt-get update && " +
           "apt-get install -y --no-install-recommends python-pip python2.7-dev python3-pip python3-dev ssdeep libfuzzy-dev libfuzzy2 libimage-exiftool-perl libmagic1 build-essential git libssl-dev && " +
-          "pip install -U pip setuptools && " +
+          "pip2 install -U pip setuptools && " +
           "pip3 install -U pip setuptools && " +
+          "hash -r && " +
           "cd /opt && " +
           "git clone https://github.com/TheHive-Project/Cortex-Analyzers.git && " +
           "for I in $(find Cortex-Analyzers -name 'requirements.txt'); do pip2 install -r $I; done && " +

project/Dependencies.scala

@@ -17,6 +17,6 @@ object Dependencies {
 
   val reflections = "org.reflections" % "reflections" % "0.9.11"
   val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2"
-  val elastic4play = "org.thehive-project" %% "elastic4play" % "1.6.0"
+  val elastic4play = "org.thehive-project" %% "elastic4play" % "1.6.2"
 }
 

version.sbt

@@ -1 +1 @@
-version in ThisBuild := "2.1.0-RC1"
+version in ThisBuild := "2.1.0"

www/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortex",
-  "version": "2.0.4",
+  "version": "2.1.0",
   "description": "A powerfull observable analysis engine",
   "license": "AGPL-v3",
   "homepage": "https://github.com/TheHive-Project/Cortex",

www/src/app/core/directives/tlp/tlp.directive.js

@@ -5,7 +5,7 @@ import _ from 'lodash/core';
 import tpl from './tlp.html';
 import './tlp.scss';
 
-export default function(app) {
+export default function (app) {
   app.directive('tlp', tlp);
 
   function tlp(Tlps) {
@@ -14,7 +14,8 @@ export default function(app) {
     return {
       templateUrl: tpl,
       scope: {
-        value: '='
+        value: '=',
+        namespace: '@'
       },
       replace: true,
       link: linkFn
@@ -26,12 +27,14 @@ export default function(app) {
           scope.tlpClass = 'label-none';
           scope.tlp = 'None';
         } else {
-          const temp = (_.find(Tlps, { value: v }) || {}).key;
+          const temp = (_.find(Tlps, {
+            value: v
+          }) || {}).key;
 
           scope.tlpClass = `label-${(temp || '').toLowerCase()}`;
-          scope.tlp = `TLP:${temp}`;
+          scope.tlp = `${scope.namespace || 'TLP'}:${temp}`;
         }
       });
     }
   }
-}
+}

www/src/app/pages/admin/organizations/components/analyzers/analyzers-list.html

@@ -42,8 +42,10 @@ <h4 class="media-heading">
     <div class="col-sm-12 flex-table">
       <div class="flex-header media">
         <div class="flex-col flex-1">Analyzer</div>
-        <div class="flex-col flex-w-200 text-center">Max TLP</div>
-        <div class="flex-col flex-w-200 text-center">Rate Limit</div>
+        <div class="flex-col flex-w-100 text-center">Max TLP</div>
+        <div class="flex-col flex-w-100 text-center">Max PAP</div>
+        <div class="flex-col flex-w-100 text-center">Rate Limit</div>
+        <div class="flex-col flex-w-100 text-center">Cache</div>
         <div class="flex-col flex-w-100"></div>
         <div class="flex-col flex-w-100"></div>
       </div>
@@ -66,15 +68,22 @@ <h4 class="media-heading">
           </h4>
           <div class="mt-xs text-muted">{{definition.description}}</div>
         </div>
-        <div class="flex-col flex-w-200 text-center vertical centered" ng-if="$ctrl.activeAnalyzers[def]">
+        <div class="flex-col flex-w-100 text-center vertical centered" ng-if="$ctrl.activeAnalyzers[def]">
           <tlp value="$ctrl.activeAnalyzers[def].configuration.check_tlp ? $ctrl.activeAnalyzers[def].configuration.max_tlp : undefined"></tlp>
         </div>
-        <div class="flex-col flex-w-200 text-center vertical centered" ng-if="$ctrl.activeAnalyzers[def]">
+        <div class="flex-col flex-w-100 text-center vertical centered" ng-if="$ctrl.activeAnalyzers[def]">
+          <tlp value="$ctrl.activeAnalyzers[def].configuration.check_pap ? $ctrl.activeAnalyzers[def].configuration.max_pap : undefined" namepace="PAP"></tlp>
+        </div>
+        <div class="flex-col flex-w-100 text-center vertical centered" ng-if="$ctrl.activeAnalyzers[def]">
           <span ng-show="$ctrl.activeAnalyzers[def].rate && $ctrl.activeAnalyzers[def].rateUnit" class="label label-lg label-primary">
             {{$ctrl.activeAnalyzers[def].rate}} per {{$ctrl.activeAnalyzers[def].rateUnit}}
           </span>
           <span ng-show="!$ctrl.activeAnalyzers[def].rate || !$ctrl.activeAnalyzers[def].rateUnit" class="label label-lg label-default">None</span>
         </div>
+        <div class="flex-col flex-w-100 text-center vertical centered" ng-if="$ctrl.activeAnalyzers[def]">
+          <span ng-show="$ctrl.activeAnalyzers[def].jobCache === null" class="label label-lg label-default">Default</span>
+          <span ng-show="$ctrl.activeAnalyzers[def].jobCache !== null" class="label label-lg label-primary">{{$ctrl.activeAnalyzers[def].jobCache}} Minutes</span>
+        </div>
         <div class="flex-col flex-icon flex-w-100 text-center" ng-if="$ctrl.activeAnalyzers[def]" ng-click="$ctrl.edit('edit', $ctrl.analyzerDefinitions[def], $ctrl.activeAnalyzers[def])">
           <a href class="text-primary">
             <i class="fa fa-edit"></i>Edit</a>

www/src/app/pages/admin/organizations/components/responders/responders-list.html

@@ -47,8 +47,9 @@ <h4 class="media-heading">
     <div class="col-sm-12 flex-table">
       <div class="flex-header media">
         <div class="flex-col flex-1">Responders</div>
-        <div class="flex-col flex-w-200 text-center">Max TLP</div>
-        <div class="flex-col flex-w-200 text-center">Rate Limit</div>
+        <div class="flex-col flex-w-100 text-center">Max TLP</div>
+        <div class="flex-col flex-w-100 text-center">Max PAP</div>
+        <div class="flex-col flex-w-100 text-center">Rate Limit</div>        
         <div class="flex-col flex-w-100"></div>
         <div class="flex-col flex-w-100"></div>
       </div>
@@ -70,10 +71,13 @@ <h4 class="media-heading">
           </h4>
           <div class="mt-xs text-muted">{{definition.description}}</div>
         </div>
-        <div class="flex-col flex-w-200 text-center vertical centered" ng-if="$ctrl.activeResponders[def]">
+        <div class="flex-col flex-w-100 text-center vertical centered" ng-if="$ctrl.activeResponders[def]">
           <tlp value="$ctrl.activeResponders[def].configuration.check_tlp ? $ctrl.activeResponders[def].configuration.max_tlp : undefined"></tlp>
         </div>
-        <div class="flex-col flex-w-200 text-center vertical centered" ng-if="$ctrl.activeResponders[def]">
+        <div class="flex-col flex-w-100 text-center vertical centered" ng-if="$ctrl.activeResponders[def]">
+          <tlp value="$ctrl.activeResponders[def].configuration.check_pap ? $ctrl.activeResponders[def].configuration.max_pap : undefined" namespace="PAP"></tlp>
+        </div>
+        <div class="flex-col flex-w-100 text-center vertical centered" ng-if="$ctrl.activeResponders[def]">
           <span ng-show="$ctrl.activeResponders[def].rate && $ctrl.activeResponders[def].rateUnit" class="label label-lg label-primary">
             {{$ctrl.activeResponders[def].rate}} per {{$ctrl.activeResponders[def].rateUnit}}
           </span>

www/src/app/pages/jobs/components/job.details.html

@@ -31,7 +31,7 @@
 
     <strong>PAP</strong>
     <p>
-      <tlp value="$ctrl.job.pap"></tlp>
+      <tlp value="$ctrl.job.pap" namespace="PAP"></tlp>
     </p>
     <hr>