Merge branch 'hotfix/2.0.3'

CHANGELOG.md

@@ -1,13 +1,32 @@
 # Change Log
 
-## [2.0.2](https://github.com/TheHive-Project/Cortex/tree/2.02)
+## [2.0.3](https://github.com/TheHive-Project/Cortex/tree/2.0.3) (2018-04-09)
 
+[Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.0.2...2.0.3)
+
+**Implemented enhancements:**
+
+- Allow arbitrary parameters for a job [\#86](https://github.com/TheHive-Project/Cortex/issues/86)
+- Change of global config for proxy is not reflected in analyzer's configurations [\#81](https://github.com/TheHive-Project/Cortex/issues/81)
+
+**Fixed bugs:**
+
+- Refresh Analyzers button not working [\#83](https://github.com/TheHive-Project/Cortex/issues/83)
+- Version Upgrade of Analyzer makes all Analyzers invisible for TheHive \(Cortex2\) [\#75](https://github.com/TheHive-Project/Cortex/issues/75)
+
+**Closed issues:**
+
+- Allow specifying a cache period per analyzer [\#85](https://github.com/TheHive-Project/Cortex/issues/85)
+- Display existing analyzers with invalid definition [\#82](https://github.com/TheHive-Project/Cortex/issues/82)
+- Allow configuring auto artifacts extraction per analyzer [\#80](https://github.com/TheHive-Project/Cortex/issues/80)
+
+## [2.0.2](https://github.com/TheHive-Project/Cortex/tree/2.0.2) (2018-04-04)
 [Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.0.1...2.0.2)
 
 **Fixed bugs:**
 
-- Silently failure when ElasticSearch is unreachable [\#76](https://github.com/TheHive-Project/Cortex/issues/76)
 - Coretxutils and TypeError: argument of type 'bool' is not iterable [\#73](https://github.com/TheHive-Project/Cortex/issues/73)
+- Silently failure when ElasticSearch is unreachable [\#76](https://github.com/TheHive-Project/Cortex/issues/76)
 - Unable to disable analyzers [\#72](https://github.com/TheHive-Project/Cortex/issues/72)
 - Cortex 2 is not passing proxy variable to analyzers [\#71](https://github.com/TheHive-Project/Cortex/issues/71)
 - Session collision when TheHive & Cortex 2 share the same URL [\#70](https://github.com/TheHive-Project/Cortex/issues/70)
@@ -157,4 +176,4 @@
 ## [1.0.0](https://github.com/TheHive-Project/Cortex/tree/1.0.0) (2017-02-01)
 
 
-\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*
+\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*

app/org/thp/cortex/controllers/AnalyzerCtrl.scala

@@ -46,12 +46,12 @@ class AnalyzerCtrl @Inject() (
   }
 
   private val emptyAnalyzerDefinitionJson = Json.obj(
-    "version" -> JsNull,
-    "description" -> JsNull,
+    "version" -> "0.0",
+    "description" -> "unknown",
     "dataTypeList" -> Nil,
-    "author" -> JsNull,
-    "url" -> JsNull,
-    "license" -> JsNull)
+    "author" -> "unknown",
+    "url" -> "unknown",
+    "license" -> "unknown")
 
   private def analyzerJson(analyzer: Analyzer, analyzerDefinition: Option[AnalyzerDefinition]) = {
     analyzer.toJson ++ analyzerDefinition.fold(emptyAnalyzerDefinitionJson) { ad ⇒

app/org/thp/cortex/models/Analyzer.scala

@@ -27,6 +27,7 @@ trait AnalyzerAttributes { _: AttributeDef ⇒
   val configuration = attribute("configuration", F.textFmt, "Configuration of analyzer", O.sensitive)
   val rate = optionalAttribute("rate", F.numberFmt, "Number ")
   val rateUnit = optionalAttribute("rateUnit", F.enumFmt(RateUnit), "")
+  val jobCache = optionalAttribute("jobCache", F.numberFmt, "")
 }
 
 @Singleton

app/org/thp/cortex/services/JobSrv.scala

@@ -19,7 +19,7 @@ import org.scalactic.{ Bad, Good, One, Or }
 import org.thp.cortex.models._
 import play.api.libs.json._
 import play.api.{ Configuration, Logger }
-import scala.concurrent.duration.Duration
+import scala.concurrent.duration._
 import scala.concurrent.{ ExecutionContext, Future }
 import scala.sys.process.{ Process, ProcessIO }
 import scala.util.control.NonFatal
@@ -300,7 +300,8 @@ class JobSrv(
   }
 
   def findSimilarJob(analyzer: Analyzer, dataType: String, dataAttachment: Either[String, Attachment], tlp: Long, parameters: JsObject): Future[Option[Job]] = {
-    if (jobCache.length == 0) {
+    val cache = analyzer.jobCache().fold(jobCache)(_.minutes)
+    if (cache.length == 0) {
       logger.info("Job cache is disabled")
       Future.successful(None)
     }
@@ -312,7 +313,7 @@ class JobSrv(
         "analyzerId" ~= analyzer.id,
         "status" ~!= JobStatus.Failure,
         "status" ~!= JobStatus.Deleted,
-        "startDate" ~>= (now - jobCache.toMillis),
+        "startDate" ~>= (now - cache.toMillis),
         "dataType" ~= dataType,
         "tlp" ~= tlp,
         dataAttachment.fold(data ⇒ "data" ~= data, attachment ⇒ "attachment.id" ~= attachment.id),
@@ -410,19 +411,19 @@ class JobSrv(
             "data" -> job.data().get)
       }
       .map { artifact ⇒
-        val configAndParam = analyzer.config.deepMerge(job.params)
         (BaseConfig.global.items ++ BaseConfig.tlp.items ++ analyzerDefinition.configurationItems)
-          .validatedBy(_.read(configAndParam))
+          .validatedBy(_.read(analyzer.config))
           .map(cfg ⇒ Json.obj("config" -> JsObject(cfg).deepMerge(analyzerDefinition.configuration)))
           .map { cfg ⇒
             val proxy_http = (cfg \ "config" \ "proxy_http").asOpt[String].fold(JsObject.empty) { proxy ⇒ Json.obj("proxy" -> Json.obj("http" -> proxy)) }
             val proxy_https = (cfg \ "config" \ "proxy_https").asOpt[String].fold(JsObject.empty) { proxy ⇒ Json.obj("proxy" -> Json.obj("https" -> proxy)) }
-            cfg.deepMerge(Json.obj("config" -> (proxy_http.deepMerge(proxy_https))))
+            cfg.deepMerge(Json.obj("config" -> proxy_http.deepMerge(proxy_https)))
           }
           .map(_ deepMerge artifact +
             ("dataType" -> JsString(job.dataType())) +
             ("tlp" -> JsNumber(job.tlp())) +
-            ("message" -> JsString(job.message().getOrElse(""))))
+            ("message" -> JsString(job.message().getOrElse(""))) +
+            ("parameters" -> job.params))
           .badMap(e ⇒ AttributeCheckingError("job", e.toSeq))
           .toTry
       }

build.sbt

@@ -140,11 +140,13 @@ dockerCommands ~= { dc =>
       Cmd("USER", "root"),
       ExecCmd("RUN", "bash", "-c",
         "apt-get update && " +
-          "apt-get install -y --no-install-recommends python-pip python2.7-dev ssdeep libfuzzy-dev libfuzzy2 libimage-exiftool-perl libmagic1 build-essential git libssl-dev && " +
+          "apt-get install -y --no-install-recommends python-pip python2.7-dev python3-pip python3-dev ssdeep libfuzzy-dev libfuzzy2 libimage-exiftool-perl libmagic1 build-essential git libssl-dev && " +
           "pip install -U pip setuptools && " +
+          "pip3 install -U pip setuptools && " +
           "cd /opt && " +
           "git clone https://github.com/CERT-BDF/Cortex-Analyzers.git && " +
-          "pip install $(sort -u Cortex-Analyzers/analyzers/*/requirements.txt)"),
+          "pip install $(sort -u Cortex-Analyzers/analyzers/*/requirements.txt) && " +
+          "sort -u Cortex-Analyzers/analyzers/*/requirements.txt | grep -v ';python_version' | xargs -n 1 pip3 install || true"),
       Cmd("ADD", "var", "/var"),
       Cmd("ADD", "etc", "/etc"),
       ExecCmd("RUN", "chown", "-R", "daemon:root", "/var/log/cortex"),

package/docker/entrypoint

@@ -48,7 +48,7 @@ then
 			SECRET=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 64 | head -n 1)
 		fi
 		echo Using secret: $SECRET
-		echo play.crypto.secret=\"$SECRET\" >> $CONFIG_FILE
+		echo play.http.secret.key=\"$SECRET\" >> $CONFIG_FILE
 	fi
 
 	if test $CONFIG_ES = 1

version.sbt

@@ -1 +1 @@
-version in ThisBuild := "2.0.2"
+version in ThisBuild := "2.0.3"

www/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortex",
-  "version": "2.0.2",
+  "version": "2.0.3",
   "description": "A powerfull observable analysis engine",
   "license": "AGPL-v3",
   "homepage": "https://github.com/TheHive-Project/Cortex",
@@ -21,6 +21,7 @@
     "angular-bootstrap-multiselect": "git+https://github.com/bentorfs/angular-bootstrap-multiselect.git",
     "angular-clipboard": "^1.6.2",
     "angular-images-resizer": "^2.0.2",
+    "angular-input-masks": "^4.1.0",
     "angular-local-storage": "^0.7.1",
     "angular-messages": "1.6.8",
     "angular-moment": "^1.2.0",

www/src/app/index.module.js

@@ -33,6 +33,7 @@ const App = angular.module('cortex', [
   'btorfs.multiselect',
   'LocalStorageModule',
   'angularUtils.directives.dirPagination',
+  'ui.utils.masks',
 
   // core
   coreModule.name,

www/src/app/index.vendor.js

@@ -40,6 +40,8 @@ import 'angular-images-resizer';
 
 import 'angular-base64-upload';
 
+import 'angular-input-masks';
+
 // local scripts
 //import "../assets/js/...";
 

www/src/app/pages/admin/organizations/components/analyzer-config-form.controller.js

@@ -1,10 +1,33 @@
 'use strict';
 
+import _ from 'lodash/core';
+
 export default class AnalyzerConfigFormController {
-  constructor(Tlps) {
+  constructor($log, Tlps, AnalyzerService) {
     'ngInject';
 
+    this.AnalyzerService = AnalyzerService;
     this.Tlps = Tlps;
     this.rateUnits = ['Day', 'Month'];
   }
+
+  $onInit() {
+    this.useGlobalCache =
+      this.analyzer.jobCache === null || this.analyzer.jobCache === undefined;
+  }
+
+  applyConfig(config) {
+    _.forEach(
+      _.keys(config),
+      k => (this.analyzer.configuration[k] = config[k])
+    );
+  }
+
+  applyGlobalConfig() {
+    this.applyConfig(this.globalConfig.config);
+  }
+
+  applyBaseConfig() {
+    this.applyConfig(this.baseConfig.config);
+  }
 }

www/src/app/pages/admin/organizations/components/analyzer-config-form.html

@@ -10,12 +10,28 @@ <h4>Base details</h4>
 </section>
 
 <section ng-show="$ctrl.definition.configurationItems.length > 0">
-  <h4>Configuration</h4>  
+  <h4>
+    Configuration
+    <div class="clearfix pull-right">
+      <small>
+        <a href ng-click="$ctrl.applyBaseConfig()">
+          <i class="fa fa-copy"></i> Apply defaults</a>
+      </small>
+    </div>
+  </h4>  
   <configuration-form items="$ctrl.definition.configurationItems" configuration="$ctrl.analyzer.configuration"></configuration-form>
 </section>
 
 <section>
-  <h4>Options</h4>
+  <h4>
+    Options
+    <div class="clearfix pull-right">
+      <small>
+        <a href ng-click="$ctrl.applyGlobalConfig()">
+          <i class="fa fa-copy"></i> Apply defaults</a>
+      </small>
+    </div>    
+  </h4>
   <div class="form-group">
     <label class="col-sm-3 control-label">Enable TLP check</label>
     <div class="col-sm-9 form-inline">
@@ -42,10 +58,22 @@ <h4>Options</h4>
       <input class="form-control" type="text" ng-model="$ctrl.analyzer.configuration.proxy_https">
     </div>    
   </div>
+  <div class="form-group">
+    <label class="col-sm-3 control-label">Extract observables</label>
+    <div class="col-sm-9">
+      <div class="btn-group">
+        <label uib-btn-radio="true" class="btn btn-default" ng-model="$ctrl.analyzer.configuration.auto_extract_artifacts">True</label>
+        <label uib-btn-radio="false" class="btn btn-default" ng-model="$ctrl.analyzer.configuration.auto_extract_artifacts">False</label>
+      </div>
+    </div>
+    <div class="col-sm-offset-3 col-sm-9">
+      <p class="help-block">Set to True to enable automatic observables extraction from analysis reports.</p>
+    </div>
+  </div>  
   <div class="form-group">
     <label class="col-sm-3 control-label">Rate Limiting</label>
     <div class="col-sm-4">
-      <input class="form-control" type="number" ng-model="$ctrl.analyzer.rate">
+      <input class="form-control" type="text" ng-model="$ctrl.analyzer.rate" ui-number-mask="0" ui-hide-group-sep>
     </div>
     <div class="col-sm-5">      
       <select class="form-control" ng-model="$ctrl.analyzer.rateUnit" ng-options="unit for unit in $ctrl.rateUnits">
@@ -55,5 +83,21 @@ <h4>Options</h4>
     <div class="col-sm-offset-3 col-sm-9">
       <p class="help-block">Define the maximum number of requests and the associated unit if applicable.</p>
     </div>  
-  </div>  
+  </div>
+  <div class="form-group">
+    <label class="col-sm-3 control-label">Cache</label>
+    <div class="col-sm-9 form-inline">
+      <div class="btn-group">
+        <label uib-btn-radio="true" class="btn btn-default" ng-model="$ctrl.useGlobalCache" ng-click="$ctrl.analyzer.jobCache = null">Use global</label>
+        <label uib-btn-radio="false" class="btn btn-default" ng-model="$ctrl.useGlobalCache">Custom</label>
+      </div>
+      <div class="form-group" style="margin-left: 30px;" ng-show="$ctrl.useGlobalCache === false">
+        <label>Number of minutes</label>
+        <input class="form-control" style="width: 200px;" type="text" ng-model="$ctrl.analyzer.jobCache" ui-number-mask="0" ui-hide-group-sep>        
+      </div>
+    </div>
+    <div class="col-sm-offset-3 col-sm-9">
+      <p class="help-block">Define the number minutes for analysis report caching, or use the globally defined value.</p>
+    </div>
+  </div>
 </section>

www/src/app/pages/admin/organizations/components/analyzer.edit.controller.js

@@ -7,6 +7,8 @@ export default class AnalyzerEditController {
     $log,
     $uibModalInstance,
     definition,
+    globalConfig,
+    baseConfig,
     configuration,
     analyzer,
     mode
@@ -17,6 +19,8 @@ export default class AnalyzerEditController {
     this.$uibModalInstance = $uibModalInstance;
     this.mode = mode;
     this.definition = definition;
+    this.globalConfig = globalConfig;
+    this.baseConfig = baseConfig;
     this.configuration = configuration;
     this.analyzer = analyzer;
   }
@@ -27,7 +31,8 @@ export default class AnalyzerEditController {
         name: this.definition.id,
         configuration: {},
         rate: undefined,
-        rateUnit: undefined
+        rateUnit: undefined,
+        jobCache: null
       };
 
       _.forEach(this.definition.configurationItems, item => {
@@ -49,7 +54,9 @@ export default class AnalyzerEditController {
       _.forEach(globalConfig, cnf => {
         if (analyzer.configuration[cnf] === undefined) {
           analyzer.configuration[cnf] =
-            this.configuration.config[cnf] || undefined;
+            this.configuration.config[cnf] !== undefined
+              ? this.configuration.config[cnf]
+              : undefined;
         }
       });
 

www/src/app/pages/admin/organizations/components/analyzer.edit.modal.html

@@ -3,7 +3,11 @@
     <h3 class="modal-title">{{$modal.mode === 'create' ? 'Enable' : 'Edit'}} analyzer {{$modal.definition.id}}</h3>
   </div>
   <div class="modal-body">
-    <analyzer-config-form definition="$modal.definition" analyzer="$modal.analyzer"></analyzer-config-form>
+    <analyzer-config-form definition="$modal.definition"       
+      global-config="$modal.globalConfig"
+      base-config="$modal.baseConfig"
+      configuration="$modal.configuration"
+      analyzer="$modal.analyzer"></analyzer-config-form>
   </div>
   <div class="modal-footer">
     <button class="btn btn-default pull-left" type="button" ng-click="$modal.cancel()">Cancel</button>