diff --git a/docker-compose.yml b/docker-compose.yml index 6e97cd0..f72fedb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,7 +65,7 @@ services: networks: - internal volumes: - - /traffic:/traffic:ro + - /home/qyn/tulip/traffic:/traffic:ro command: "./enricher -eve /traffic/eve.json" environment: TULIP_MONGO: mongo:27017 diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 1d9ade5..d50c298 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -14,8 +14,8 @@ export interface Flow { // TODO: get this from backend instead of hacky work around service_tag: string inx: number - starred: boolean - blocked: boolean + parent_id: Id + child_id: Id tags: string[] suricata: number[] filename: string @@ -76,12 +76,10 @@ class TulipApi { async _getFlows(query: FlowsQuery, destToService: any) { // HACK: make starred look like a tag - const starred = query.tags.includes("starred"); - let tags = query.tags.filter(tag => tag !== "starred") + let tags = query.tags; const hacky_query = { ...query, tags: tags.length > 0 ? tags : undefined, - starred } // END HACK @@ -110,8 +108,8 @@ class TulipApi { const response = await fetch(`${this.API_ENDPOINT}/tags`); const tags = await response.json(); - // HACK: make starred look like a tag - tags.push("starred") + // HACK: make connected flows look like a tag + tags.push("connected") // END HACK @@ -129,8 +127,8 @@ class TulipApi { return await response.text() } - async toPythonRequest(body: string, tokenize: boolean) { - const response = await fetch(`${this.API_ENDPOINT}/to_python_request?tokenize=${tokenize ? "1" : "0"}`, { + async toPythonRequest(body: string, id: string, tokenize: boolean) { + const response = await fetch(`${this.API_ENDPOINT}/to_python_request?tokenize=${tokenize ? "1" : "0"}&id=${id}`, { method: "POST", headers: { "Content-Type": "text/plain;charset=UTF-8" diff --git a/frontend/src/components/FlowList.tsx b/frontend/src/components/FlowList.tsx index ec36020..67514a9 100644 --- a/frontend/src/components/FlowList.tsx +++ b/frontend/src/components/FlowList.tsx @@ -4,7 +4,7 @@ import { useParams, useNavigate, } from "react-router-dom"; -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useState, useRef } from "react"; import { useAtom, useAtomValue } from "jotai"; import { Flow, FullFlow, useTulip } from "../api"; import { @@ -14,7 +14,7 @@ import { END_FILTER_KEY, } from "../App"; -import { HeartIcon, FilterIcon } from "@heroicons/react/solid"; +import { HeartIcon, FilterIcon, LinkIcon } from "@heroicons/react/solid"; import { HeartIcon as EmptyHeartIcon, FilterIcon as EmptyFilterIcon, @@ -23,7 +23,7 @@ import { import classes from "./FlowList.module.css"; import { format } from "date-fns"; import useDebounce from "../hooks/useDebounce"; -import { Virtuoso } from "react-virtuoso"; +import { Virtuoso, VirtuosoHandle } from "react-virtuoso"; import classNames from "classnames"; import { Tag } from "./Tag"; import { lastRefreshAtom } from "./Header"; @@ -36,6 +36,11 @@ export function FlowList() { const [flowList, setFlowList] = useState([]); + // Set default flowIndex to Infinity, so that initialTopMostItemIndex != 0 and therefore scrolledToInitialItem != true + const [flowIndex, setFlowIndex] = useState(Infinity); + + const virtuoso = useRef(null); + const service_name = searchParams.get(SERVICE_FILTER_KEY) ?? ""; const service = services.find((s) => s.name == service_name); @@ -73,8 +78,13 @@ export function FlowList() { service: "", // FIXME tags: selectedTags, }); + data.forEach((flow, index)=> { + if(flow._id.$oid === params.id){ setFlowIndex(index)} + }) + setFlowList(data); setLoading(false); + }; fetchData().catch(console.error); }, [ @@ -84,12 +94,14 @@ export function FlowList() { to_filter, selectedTags, lastRefresh, + params, + virtuoso ]); const onHeartHandler = useCallback(async (flow: Flow) => { - await api.starFlow(flow._id.$oid, !flow.starred); + await api.starFlow(flow._id.$oid, !flow.tags.includes("starred")); // optimistic update - const newFlow = { ...flow, starred: !flow.starred }; + const newFlow = { ...flow}; setFlowList((prev) => prev.map((f) => (f._id.$oid === flow._id.$oid ? newFlow : f)) ); @@ -147,6 +159,8 @@ export function FlowList() { "sidebar-loading": loading, })} data={flowList} + ref={virtuoso} + initialTopMostItemIndex={flowIndex} itemContent={(index, flow) => ( !["starred"].includes(t)); + const filtered_tag_list = flow.tags.filter((t) => t != "starred"); const duration = flow.duration > 10000 ? ( @@ -186,7 +200,6 @@ function FlowListEntry({ flow, isActive, onHeartClick }: FlowListEntryProps) { ) : (
{flow.duration}ms
); - return (
  • { onHeartClick(flow); }} > - {flow.starred ? ( + {flow.tags.includes("starred") ? ( ) : ( )}
    + +
    + {flow.child_id.$oid != "000000000000000000000000" || flow.parent_id.$oid != "000000000000000000000000" ? ( + + ) : undefined} +
    diff --git a/frontend/src/pages/FlowView.tsx b/frontend/src/pages/FlowView.tsx index d0965a5..46a14ba 100644 --- a/frontend/src/pages/FlowView.tsx +++ b/frontend/src/pages/FlowView.tsx @@ -1,4 +1,4 @@ -import { useParams } from "react-router-dom"; +import { useSearchParams, Link, useParams } from "react-router-dom"; import React, { useEffect, useState } from "react"; import { useTulip, FlowData, FullFlow } from "../api"; import { Buffer } from "buffer"; @@ -84,14 +84,14 @@ function WebFlow({ flow }: { flow: FlowData }) { ); } -function PythonRequestFlow({ flow }: { flow: FlowData }) { +function PythonRequestFlow({ full_flow, flow }: {full_flow:FullFlow, flow: FlowData }) { const [data, setData] = useState(""); const { api } = useTulip(); useEffect(() => { const fetchData = async () => { - const data = await api.toPythonRequest(btoa(flow.data), true); + const data = await api.toPythonRequest(btoa(flow.data), full_flow._id.$oid,true); setData(data); }; // TODO proper error handling @@ -102,6 +102,7 @@ function PythonRequestFlow({ flow }: { flow: FlowData }) { } interface FlowProps { + full_flow: FullFlow; flow: FlowData; delta_time: number; } @@ -140,7 +141,7 @@ function detectType(flow: FlowData) { return "Plain"; } -function Flow({ flow, delta_time }: FlowProps) { +function Flow({ full_flow, flow, delta_time }: FlowProps) { const formatted_time = format(new Date(flow.time), "HH:mm:ss:SSS"); const displayOptions = ["Plain", "Hex", "Web", "PythonRequest"]; @@ -183,7 +184,7 @@ function Flow({ flow, delta_time }: FlowProps) { {displayOption === "Plain" && } {displayOption === "Web" && } {displayOption === "PythonRequest" && ( - + )}
    @@ -263,6 +264,7 @@ function FlowOverview({ flow }: { flow: FullFlow }) { function Header() {} export function FlowView() { + let [searchParams] = useSearchParams(); const params = useParams(); const [flow, setFlow] = useState(); @@ -318,6 +320,12 @@ export function FlowView() {
    + {flow?.parent_id?.$oid != "000000000000000000000000"? Parent: undefined} + {flow ? : undefined} {flow?.flow.map((flow_data, i, a) => { const delta_time = a[i].time - (a[i - 1]?.time ?? a[i].time); @@ -325,10 +333,17 @@ export function FlowView() { ); })} + + {flow?.child_id?.$oid != "000000000000000000000000" ? Child: undefined} ); } diff --git a/services/README.md b/services/README.md index 4850854..51b4200 100755 --- a/services/README.md +++ b/services/README.md @@ -18,7 +18,6 @@ Each document will have: "dst_ip": "127.0.0.1", "dst_port": 1234, "contains_flag": //true if the importer have found that the flow contains a flag based on the env var regex - "starred": //if the flow is starred "flow": [ { "data": "...", //printable data @@ -59,9 +58,6 @@ Returns the all document with `flow_id` id, including the field `flow` ##### GET /star/(flow_id)/(0,1) Set the flow favourite (1) or not (0) -##### POST /starred -Returns a list of document like `/query` endpoint, but only with starred items. - ##### POST /to_python_request/(tokenize) convert the request to python syntax. Tokenize is used to toggle the auto-parsing of args. diff --git a/services/data2req.py b/services/data2req.py index 2cbcee8..592308a 100755 --- a/services/data2req.py +++ b/services/data2req.py @@ -48,7 +48,7 @@ def send_error(self, code, message): self.error_message = message # tokenize used for automatically fill data param of request -def convert_http_requests(raw_request, tokenize=True, use_requests_session=False): +def convert_http_requests(raw_request, flow, tokenize=True, use_requests_session=False): request = HTTPRequest(raw_request) data = {} @@ -96,8 +96,9 @@ def convert_http_requests(raw_request, tokenize=True, use_requests_session=False rtemplate = Environment(loader=BaseLoader()).from_string("""import os import requests +import sys -host = os.getenv("TARGET_IP") +host = sys.argv[1] {% if use_requests_session %} s = requests.Session() @@ -107,7 +108,7 @@ def convert_http_requests(raw_request, tokenize=True, use_requests_session=False {% endif %} data = {{data}} -{% if use_requests_session %}s{% else %}requests{% endif %}.{{request.command.lower()}}("http://{}{{request.path}}".format(host), {{data_param_name}}=data{% if not use_requests_session %}, headers=headers{% endif %})""") +{% if use_requests_session %}s{% else %}requests{% endif %}.{{request.command.lower()}}("http://{}:{{port}}{{request.path}}".format(host), {{data_param_name}}=data{% if not use_requests_session %}, headers=headers{% endif %})""") return rtemplate.render( headers=str(dict(headers)), @@ -115,4 +116,5 @@ def convert_http_requests(raw_request, tokenize=True, use_requests_session=False request=request, data_param_name=data_param_name, use_requests_session=use_requests_session, + port=flow["dst_port"] ) diff --git a/services/db.py b/services/db.py index 980de54..8fc29ca 100755 --- a/services/db.py +++ b/services/db.py @@ -59,10 +59,6 @@ def getFlowList(self, filters): if "from_time" in filters and "to_time" in filters: f["time"] = {"$gte": int(filters["from_time"]), "$lt": int(filters["to_time"])} - if "starred" in filters: - f["starred"] = bool(filters["starred"]) - if "blocked" in filters: - f["blocked"] = bool(filters["blocked"]) if "tags" in filters: f["tags"] = { "$all": [str(elem) for elem in filters["tags"]] @@ -92,7 +88,10 @@ def getFlowDetail(self, id): return ret def setStar(self, flow_id, star): - self.pcap_coll.find_one_and_update({"_id": ObjectId(flow_id)}, {"$set": {"starred": star}}) + if star: + self.pcap_coll.find_one_and_update({"_id": ObjectId(flow_id)}, {"$push": {"tags": "starred"}}) + else: + self.pcap_coll.find_one_and_update({"_id": ObjectId(flow_id)}, {"$pull": {"tags": "starred"}}) def isFileAlreadyImported(self, file_name): return self.file_coll.find({"file_name": file_name}).count() != 0 diff --git a/services/flow2pwn.py b/services/flow2pwn.py index 00cfcc9..7e3830d 100755 --- a/services/flow2pwn.py +++ b/services/flow2pwn.py @@ -40,8 +40,10 @@ def flow2pwn(flow): port = flow["dst_port"] script = """from pwn import * +import sys -proc = remote('{}', {}) +host = sys.argv[1] +proc = remote(host, {}) """.format(ip, port) for message in flow['flow']: diff --git a/services/go-importer/cmd/assembler/http.go b/services/go-importer/cmd/assembler/http.go index aa4766d..432c665 100644 --- a/services/go-importer/cmd/assembler/http.go +++ b/services/go-importer/cmd/assembler/http.go @@ -4,38 +4,66 @@ import ( "bufio" "bytes" "compress/gzip" + "fmt" "go-importer/internal/pkg/db" + "hash/crc32" "io" "io/ioutil" "net/http" "net/http/httputil" + "net/url" "strings" "github.com/andybalholm/brotli" ) +func AddFingerprints(cookies []*http.Cookie, fingerPrints map[uint32]bool) { + for _, cookie := range cookies { + + fmt.Println(cookie) + // Prevent exploitation by encoding :pray:, who cares about collisions + checksum := crc32.Checksum([]byte(url.QueryEscape(cookie.Name)), crc32.IEEETable) + + checksum = crc32.Update(checksum, crc32.IEEETable, []byte("=")) + + checksum = crc32.Update(checksum, crc32.IEEETable, []byte(url.QueryEscape(cookie.Value))) + fingerPrints[checksum] = true + //*fingerPrints = append(*fingerPrints, int(checksum)) + } +} + // Parse and simplify every item in the flow. Items that were not successfuly // parsed are left as-is. // // If we manage to simplify a flow, the new data is placed in flowEntry.data func ParseHttpFlow(flow *db.FlowEntry) { + // Use a set to get rid of duplicates + fingerprintsSet := make(map[uint32]bool) + for idx := 0; idx < len(flow.Flow); idx++ { flowItem := &flow.Flow[idx] // TODO; rethink the flowItem format to make this less clunky reader := bufio.NewReader(strings.NewReader(flowItem.Data)) if flowItem.From == "c" { // HTTP Request - _, err := http.ReadRequest(reader) - if err == nil { + req, err := http.ReadRequest(reader) + if err != nil { + continue //TODO; replace the HTTP data. } + // Parse cookie and grab fingerprints + AddFingerprints(req.Cookies(), fingerprintsSet) + } else if flowItem.From == "s" { // Parse HTTP Response res, err := http.ReadResponse(reader, nil) if err != nil { continue } + // Parse cookie and grab fingerprints + AddFingerprints(res.Cookies(), fingerprintsSet) + // Substitute body encoding := res.Header["Content-Encoding"] @@ -82,6 +110,12 @@ func ParseHttpFlow(flow *db.FlowEntry) { flowItem.Data = string(replacement) } } + + // Use maps.Keys(fingerprintsSet) in the future + flow.Fingerprints = make([]uint32, 0, len(fingerprintsSet)) + for k := range fingerprintsSet { + flow.Fingerprints = append(flow.Fingerprints, k) + } } func handleGzip(r io.Reader) (io.ReadCloser, error) { diff --git a/services/go-importer/cmd/assembler/tcp.go b/services/go-importer/cmd/assembler/tcp.go index 4cb85d8..74da472 100644 --- a/services/go-importer/cmd/assembler/tcp.go +++ b/services/go-importer/cmd/assembler/tcp.go @@ -18,6 +18,7 @@ import ( "github.com/google/gopacket" "github.com/google/gopacket/layers" "github.com/google/gopacket/reassembly" + "go.mongodb.org/mongo-driver/bson/primitive" ) var allowmissinginit = true @@ -174,7 +175,6 @@ func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool { "time": 1530098789655, "duration": 96, "inx": 0, - "starred": 0, } */ src, dst := t.net.Endpoints() @@ -189,19 +189,20 @@ func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool { duration = t.FlowItems[len(t.FlowItems)-1].Time - time entry := db.FlowEntry{ - Src_port: int(t.src_port), - Dst_port: int(t.dst_port), - Src_ip: src.String(), - Dst_ip: dst.String(), - Time: time, - Duration: duration, - Inx: 0, - Starred: false, - Blocked: false, - Tags: make([]string, 0), - Suricata: make([]int, 0), - Filename: t.source, - Flow: t.FlowItems, + Src_port: int(t.src_port), + Dst_port: int(t.dst_port), + Src_ip: src.String(), + Dst_ip: dst.String(), + Time: time, + Duration: duration, + Inx: 0, + Parent_id: primitive.NilObjectID, + Child_id: primitive.NilObjectID, + Blocked: false, + Tags: make([]string, 0), + Suricata: make([]int, 0), + Filename: t.source, + Flow: t.FlowItems, } t.reassemblyCallback(entry) diff --git a/services/go-importer/cmd/assembler/udp.go b/services/go-importer/cmd/assembler/udp.go new file mode 100644 index 0000000..06ab7d0 --- /dev/null +++ b/services/go-importer/cmd/assembler/udp.go @@ -0,0 +1 @@ +package main diff --git a/services/go-importer/internal/pkg/db/db.go b/services/go-importer/internal/pkg/db/db.go index c75e12a..83a843b 100644 --- a/services/go-importer/internal/pkg/db/db.go +++ b/services/go-importer/internal/pkg/db/db.go @@ -25,19 +25,21 @@ type FlowItem struct { } type FlowEntry struct { - Src_port int - Dst_port int - Src_ip string - Dst_ip string - Time int - Duration int - Inx int - Starred bool - Blocked bool - Filename string - Suricata []int - Flow []FlowItem - Tags []string + Src_port int + Dst_port int + Src_ip string + Dst_ip string + Time int + Duration int + Inx int + Blocked bool + Filename string + Parent_id primitive.ObjectID + Child_id primitive.ObjectID + Fingerprints []uint32 + Suricata []int + Flow []FlowItem + Tags []string } type Database struct { @@ -63,6 +65,8 @@ func (db Database) ConfigureDatabase() { db.InsertTag("flag-out") db.InsertTag("blocked") db.InsertTag("suricata") + db.InsertTag("starred") + db.InsertTag("blocked") db.ConfigureIndexes() } @@ -106,13 +110,51 @@ func (db Database) ConfigureIndexes() { func (db Database) InsertFlow(flow FlowEntry) { flowCollection := db.client.Database("pcap").Collection("pcap") + if len(flow.Fingerprints) > 0 { + query := bson.M{ + "fingerprints": bson.M{ + "$in": flow.Fingerprints, + }, + } + opts := options.FindOne().SetSort(bson.M{"time": -1}) + + // TODO does this return the first one? If multiple documents satisfy the given query expression, then this method will return the first document according to the natural order which reflects the order of documents on the disk. + connectedFlow := struct { + MongoID primitive.ObjectID `bson:"_id"` + }{} + err := flowCollection.FindOne(context.TODO(), query, opts).Decode(&connectedFlow) + + // There is a connected flow + if err == nil { + //TODO Maybe add the childs fingerprints to mine? + flow.Child_id = connectedFlow.MongoID + } + } + // TODO; use insertMany instead - _, err := flowCollection.InsertOne(context.TODO(), flow) + insertion, err := flowCollection.InsertOne(context.TODO(), flow) // check for errors in the insertion if err != nil { log.Println("Error occured while inserting record: ", err) log.Println("NO PCAP DATA WILL BE AVAILABLE FOR: ", flow.Filename) } + + if flow.Child_id == primitive.NilObjectID { + return + } + + query := bson.M{ + "_id": flow.Child_id, + } + + info := bson.M{ + "$set": bson.M{ + "parent_id": insertion.InsertedID, + }, + } + + _, err = flowCollection.UpdateOne(context.TODO(), query, info) + //TODO error handling } // Insert a new pcap uri, returns true if the pcap was not present yet, diff --git a/services/tests.py b/services/tests.py index 857c0a2..22f850d 100755 --- a/services/tests.py +++ b/services/tests.py @@ -42,9 +42,6 @@ def get_first_flow_id(): def do_request(path): return requests.get("{}/{}".format(WS_URL,path)) -def get_starred(): - return requests.post("{}/starred".format(WS_URL), json={}).json() - def test_services(): services = do_request("services").json() assert len(services) == 5 @@ -54,13 +51,6 @@ def test_query(): res = requests.post("{}/query".format(WS_URL), json={}).json() assert len(res) == 539 -def test_star(): - assert len(get_starred()) == 0 - requests.get("{}/star/{}/1".format(WS_URL,FLOW_ID)) - assert len(get_starred()) == 1 - requests.get("{}/star/{}/0".format(WS_URL,FLOW_ID)) - assert len(get_starred()) == 0 - def test_frontend(): assert "You need to enable JavaScript to run this app." in requests.get("{}".format(FE_URL)).text # todo find a better way to test this, maybe diff --git a/services/webservice.py b/services/webservice.py index f60fc08..45c28e9 100755 --- a/services/webservice.py +++ b/services/webservice.py @@ -67,16 +67,6 @@ def signature(id): result = db.getSignature(int(id)) return return_json_response(result) -@application.route('/starred', methods=['POST']) -def getStarred(): - json = request.get_json() - json["starred"] = True - result = db.getFlowList(json) - return return_json_response(result) - - - - @application.route('/star//') def setStar(flow_id, star_to_set): db.setStar(flow_id, star_to_set != "0") @@ -96,11 +86,18 @@ def getFlowDetail(id): @application.route('/to_python_request', methods=['POST']) def convertToRequests(): + flow_id = request.args.get("id", "") + if flow_id == "": + return return_text_response("There was an error while converting the request:\n{}: {}".format("No flow id", "No flow id param")) + #TODO check flow null or what + flow = db.getFlowDetail(flow_id) + if not flow: + return return_text_response("There was an error while converting the request:\n{}: {}".format("Invalid flow", "Invalid flow id")) data = b64decode(request.data) tokenize = request.args.get("tokenize", False) use_requests_session = request.args.get("use_requests_session", False) try: - converted = convert_http_requests(data, tokenize, use_requests_session) + converted = convert_http_requests(data, flow, tokenize, use_requests_session) except Exception as ex: return return_text_response("There was an error while converting the request:\n{}: {}".format(type(ex).__name__, ex)) return return_text_response(converted)