Merge pull request #28 from CzechCyberTeam/feature-qol

Czech QOL changes

.dockerignore

@@ -0,0 +1 @@
+/traffic

.env.example

@@ -12,3 +12,6 @@ TICK_START="2018-06-27T13:00+02:00"
 # Tick length in ms
 TICK_LENGTH=180000
 
+# PCAP-over-IP connection
+PCAP_OVER_IP=
+#PCAP_OVER_IP=127.0.0.1:4242

.gitignore

@@ -131,4 +131,4 @@ venv.bak/
 workspace.xml
 
 
-traffic
+/traffic

docker-compose.yml

@@ -13,6 +13,7 @@ services:
  context: frontend
  dockerfile: Dockerfile-frontend
  image: tulip-frontend:latest
+ restart: unless-stopped
  ports:
  - "3000:3000"
  depends_on:
@@ -27,8 +28,7 @@ services:
  context: .
  dockerfile: Dockerfile-python
  image: tulip-api:latest
- ports:
- - "127.0.0.1:5000:5000"
+ restart: unless-stopped
  depends_on:
  - mongo
  networks:
@@ -47,6 +47,7 @@ services:
  context: services/go-importer
  dockerfile: Dockerfile-assembler
  image: tulip-assembler:latest
+ restart: unless-stopped
  depends_on:
  - mongo
  networks:
@@ -57,12 +58,14 @@ services:
  environment:
  TULIP_MONGO: ${TULIP_MONGO}
  FLAG_REGEX: ${FLAG_REGEX}
+ PCAP_OVER_IP: ${PCAP_OVER_IP}
 
  enricher:
  build:
  context: services/go-importer
  dockerfile: Dockerfile-enricher
  image: tulip-enricher:latest
+ restart: unless-stopped
  depends_on:
  - mongo
  networks:

services/data2req.py

@@ -8,12 +8,12 @@
 # Copyright ©2018 Brunello Simone
 # Copyright ©2018 Alessio Marotta
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-# 
+#
 # Flower is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
-# 
+#
 # Flower is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
@@ -36,7 +36,11 @@ def __init__(self, raw_http_request):
  self.error_code = self.error_message = None
  self.parse_request()
 
- self.headers = dict(self.headers)
+ try:
+ self.headers = dict(self.headers)
+ except AttributeError:
+ self.headers = {}
+
  # Data
  try:
  self.body = raw_http_request.split(b"\r\n\r\n", 1)[1].rstrip()
@@ -96,7 +100,7 @@ def decode_http_request(raw_request, tokenize):
 
 # tokenize used for automatically fill data param of request
 def convert_single_http_requests(raw_request, flow, tokenize=True, use_requests_session=False):
- 
+
  request, data, data_param_name, headers = decode_http_request(raw_request, tokenize)
  if not request.path.startswith('/'):
  raise Exception('request path must start with / to be a valid HTTP request')
@@ -151,7 +155,7 @@ def convert_flow_to_http_requests(flow, tokenize=True, use_requests_session=True
  if not request.path.startswith('/'):
  raise Exception('request path must start with / to be a valid HTTP request')
  request_path_repr = repr(request.path)
- 
+
  script += render("""
 {% if use_requests_session %}
 s.headers = {{headers}}

services/go-importer/cmd/assembler/http.go

@@ -43,24 +43,33 @@ func ParseHttpFlow(flow *db.FlowEntry) {
  if flowItem.From == "c" {
  // HTTP Request
  req, err := http.ReadRequest(reader)
- if err != nil {
- if *experimental {
- // Parse cookie and grab fingerprints
- AddFingerprints(req.Cookies(), fingerprintsSet)
- }
+ if err != nil || req == nil {
  continue
- //TODO; replace the HTTP data.
- // Remember to use a `LimitReader` when implementing this to prevent
- // decompressions bombs / DOS!
  }
 
+ if !containsTag(flow.Tags, "http") {
+ flow.Tags = append(flow.Tags, "http")
+ }
+
+ if *experimental {
+ // Parse cookie and grab fingerprints
+ AddFingerprints(req.Cookies(), fingerprintsSet)
+ }
+
+ //TODO; replace the HTTP data.
+ // Remember to use a `LimitReader` when implementing this to prevent
+ // decompressions bombs / DOS!
  } else if flowItem.From == "s" {
  // Parse HTTP Response
  res, err := http.ReadResponse(reader, nil)
- if err != nil {
+ if err != nil || res == nil {
  continue
  }
 
+ if !containsTag(flow.Tags, "http") {
+ flow.Tags = append(flow.Tags, "http")
+ }
+
  if *experimental {
  // Parse cookie and grab fingerprints
  AddFingerprints(res.Cookies(), fingerprintsSet)
@@ -79,20 +88,21 @@ func ParseHttpFlow(flow *db.FlowEntry) {
  // Failed to fully read the body. Bail out here
  continue
  }
+
  switch encoding[0] {
- case "gzip":
- newReader, err = handleGzip(res.Body)
- break
- case "br":
- newReader, err = handleBrotili(res.Body)
- break
- case "deflate":
- //TODO; verify this is correct
- newReader, err = handleGzip(res.Body)
- break
- default:
- // Skipped, unknown or identity encoding
- continue
+  case "gzip":
+  newReader, err = handleGzip(res.Body)
+  break
+  case "br":
+  newReader, err = handleBrotili(res.Body)
+  break
+  case "deflate":
+  //TODO; verify this is correct
+  newReader, err = handleGzip(res.Body)
+  break
+  default:
+  // Skipped, unknown or identity encoding
+  continue
  }
 
  // Replace the reader to allow for in-place decompression

services/go-importer/cmd/enricher/main.go

@@ -45,21 +45,16 @@ func main() {
 }
 
 func watchEve(eve_file string) {
- stat, err := os.Stat(eve_file)
- if err != nil {
- log.Fatal("Failed to open the eve file with error: ", err)
- }
-
- if stat.IsDir() {
- log.Fatal("eve file is not a file")
- }
-
  // Do the initial scan
  log.Println("Parsing initial eve contents...")
  ratchet := updateEve(eve_file, 0)
 
  log.Println("Monitoring eve file: ", eve_file)
- prevSize := stat.Size()
+ stat, err := os.Stat(eve_file)
+ prevSize := int64(0)
+ if err == nil {
+ prevSize = stat.Size()
+ }
 
  for {
  time.Sleep(time.Duration(*rescan_period) * time.Second)

services/webservice.py

@@ -8,12 +8,12 @@
 # Copyright ©2018 Brunello Simone
 # Copyright ©2018 Alessio Marotta
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-# 
+#
 # Flower is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
-# 
+#
 # Flower is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
@@ -22,6 +22,7 @@
 # You should have received a copy of the GNU General Public License
 # along with Flower. If not, see <https://www.gnu.org/licenses/>.
 
+import traceback
 from flask import Flask, Response, send_file
 
 from configurations import services, traffic_dir, start_date, tick_length
@@ -107,7 +108,7 @@ def convertToSingleRequest():
  try:
  converted = convert_single_http_requests(data, flow, tokenize, use_requests_session)
  except Exception as ex:
- return return_text_response("There was an error while converting the request:\n{}: {}".format(type(ex).__name__, ex))
+ return return_text_response("There was an error while converting the request:\n{}: {}".format(type(ex).__name__, traceback.format_exc()))
  return return_text_response(converted)
 
 @application.route('/to_python_request/<id>')
@@ -121,7 +122,7 @@ def convertToRequests(id):
  try:
  converted = convert_flow_to_http_requests(flow, tokenize, use_requests_session)
  except Exception as ex:
- return return_text_response("There was an error while converting the request:\n{}: {}".format(type(ex).__name__, ex))
+ return return_text_response("There was an error while converting the request:\n{}: {}".format(type(ex).__name__, traceback.format_exc()))
  return return_text_response(converted)
 
 @application.route('/to_pwn/<id>')
@@ -136,13 +137,13 @@ def downloadFile():
  if filepath is None:
  return return_text_response("There was an error while downloading the requested file:\n{}: {}".format("Invalid 'file'", "No 'file' given"))
  filepath = Path(filepath)
- 
+
  # Check for path traversal by resolving the file first.
  filepath = filepath.resolve()
  if not traffic_dir in filepath.parents:
  return return_text_response("There was an error while downloading the requested file:\n{}: {}".format("Invalid 'file'", "'file' was not in a subdirectory of traffic_dir"))
 
- try: 
+ try:
  return send_file(filepath, as_attachment=True)
  except FileNotFoundError:
  return return_text_response("There was an error while downloading the requested file:\n{}: {}".format("Invalid 'file'", "'file' not found"))