From 78a18b6c58018d8d9dc86624ef8440f5f9c9a186 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Sat, 21 Oct 2023 23:15:50 +0900 Subject: [PATCH 01/12] feat: add extract-scriptblocks command. --- src/takajo.nim | 13 ++- src/takajopkg/extractScriptblocks.nim | 145 ++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 src/takajopkg/extractScriptblocks.nim diff --git a/src/takajo.nim b/src/takajo.nim index 091708e5..4c391285 100644 --- a/src/takajo.nim +++ b/src/takajo.nim @@ -16,6 +16,7 @@ import os import std/enumerate import suru import takajopkg/general +include takajopkg/extractScriptblocks include takajopkg/listDomains include takajopkg/listIpAddresses include takajopkg/listUndetectedEvtxFiles @@ -34,6 +35,7 @@ include takajopkg/vtHashLookup when isMainModule: clCfg.version = "2.1.0-dev" const examples = "Examples:\p" + const example_extract_scriptblocks = " extract-scriptblocks -t ../hayabusa/timeline.jsonl -o scriptblock-logs\p" const example_list_domains = " list-domains -t ../hayabusa/timeline.jsonl -o domains.txt\p" const example_list_ip_addresses = " list-ip-addresses -t ../hayabusa/timeline.jsonl -o ipAddresses.txt\p" const example_list_undetected_evtx = " list-undetected-evtx -t ../hayabusa/timeline.csv -e ../hayabusa-sample-evtx\p" @@ -50,7 +52,7 @@ when isMainModule: const example_vt_ip_lookup = " vt-ip-lookup -a --ipList ipAddresses.txt -r 1000 -o results.csv --jsonOutput responses.json\p" clCfg.useMulti = "Version: 2.1.0-dev\pUsage: takajo.exe \p\pCommands:\p$subcmds\pCommand help: $command help \p\p" & - examples & example_list_domains & example_list_hashes & example_list_ip_addresses & example_list_undetected_evtx & example_list_unused_rules & + examples & example_extract_scriptblocks & example_list_domains & example_list_hashes & example_list_ip_addresses & example_list_undetected_evtx & example_list_unused_rules & example_split_csv_timeline & example_split_json_timeline & example_stack_logons & example_sysmon_process_tree & example_timeline_logon & example_timeline_suspicious_processes & example_vt_domain_lookup & example_vt_hash_lookup & example_vt_ip_lookup @@ -58,6 +60,15 @@ when isMainModule: if paramCount() == 0: styledEcho(fgGreen, outputLogo()) dispatchMulti( + [ + extractScriptblocks, cmdName = "extract-scriptblocks", + doc = "extract and reassemble PowerShell EID 4104 script block logs", + help = { + "output": "output directory (default: scriptblock-logs)", + "quiet": "do not display the launch banner", + "timeline": "Hayabusa JSONL timeline (profile: any)", + } + ], [ listDomains, cmdName = "list-domains", doc = "create a list of unique domains to be used with vt-domain-lookup", diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim new file mode 100644 index 00000000..cd50c6f8 --- /dev/null +++ b/src/takajopkg/extractScriptblocks.nim @@ -0,0 +1,145 @@ +type + Script = object + scriptBlockId: string + firstTimestamp: string + scriptBlocks: OrderedSet[string] + +proc outputScriptText(output: string, timestamp: string, computerName: string, + scriptObj: Script) = + var scriptText = "" + for text in scriptObj.scriptBlocks.items: + scriptText = scriptText & text.replace("\\r\\n", "\p").replace("\\\"", "\"") + let date = timestamp.replace(":", "_").replace(" ", "_") + let fileName = output & "/" & computerName & "-" & date & "-" & scriptObj.scriptBlockId & ".txt" + var outputFile = open(filename, fmWrite) + outputFile.write(scriptText) + flushFile(outputFile) + close(outputFile) + +proc buildSummaryRecord(path: string, messageTotal: int, + scriptObj: Script): array[5, string] = + let ts = scriptObj.firstTimestamp + let id = scriptObj.scriptBlockId + let count = scriptObj.scriptBlocks.len + var status = "Complete" + if count != messageTotal: + status = "Incomplete" + let records = $count & "/" & $messageTotal + return [ts, id, path, status, records] + +proc extractScriptblocks(output: string = "scriptblock-logs", + quiet: bool = false, timeline: string) = + let startTime = epochTime() + if not quiet: + styledEcho(fgGreen, outputLogo()) + + if not os.fileExists(timeline): + echo "The file '" & timeline & "' does not exist. Please specify a valid file path." + quit(1) + + if not isJsonConvertible(timeline): + quit(1) + + echo "Started the Extract ScriptBlock command." + echo "This command will extract PowerShell Script Block." + echo "" + + echo "Counting total lines. Please wait." + let totalLines = countLinesInTimeline(timeline) + echo "Total lines: ", totalLines + echo "" + echo "Extracting PowerShell ScriptBlocks. Please wait." + + if not dirExists(output): + echo "" + echo "The directory '" & output & "' does not exist so will be created." + createDir(output) + echo "" + + var + bar: SuruBar = initSuruBar() + stackedRecords = newTable[string, Script]() + summaryRecords = newOrderedTable[string, array[5, string]]() + + bar[0].total = totalLines + bar.setup() + + for line in lines(timeline): + inc bar + bar.update(1000000000) # refresh every second + let jsonLine = parseJson(line) + if jsonLine["EventID"].getInt(0) != 4104: + continue + + let + timestamp = jsonLine["Timestamp"].getStr() + computerName = jsonLine["Computer"].getStr() + scriptBlock = jsonLine["Details"]["ScriptBlock"].getStr() + scriptBlockId = jsonLine["ExtraFieldInfo"]["ScriptBlockId"].getStr() + messageNumber = jsonLine["ExtraFieldInfo"]["MessageNumber"].getInt() + messageTotal = jsonLine["ExtraFieldInfo"]["MessageTotal"].getInt() + var path = jsonLine["ExtraFieldInfo"].getOrDefault("Path").getStr() + if path == "": + path = "no-path" + + if scriptBlockId in stackedRecords: + stackedRecords[scriptBlockId].scriptBlocks.incl(scriptBlock) + else: + stackedRecords[scriptBlockId] = Script(scriptBlockId: scriptBlockId, + firstTimestamp: timestamp, scriptBlocks: toOrderedSet([scriptBlock])) + + if messageNumber == messageTotal: + if scriptBlockId in summaryRecords: + # Already outputted + continue + let scriptObj = stackedRecords[scriptBlockId] + outputScriptText(output, timestamp, computerName, scriptObj) + summaryRecords[scriptBlockId] = buildSummaryRecord(path, messageTotal, scriptObj) + bar.finish() + + let summaryFile = output & "/" & "summary.csv" + let header = ["Creation Time", "Script ID", "Script Name", "Results", "Extracted Records"] + var outputFile = open(summaryFile, fmWrite) + echo "" + stdout.styledWrite(fgGreen, header[0]) + stdout.styledWrite(fgWhite, " | ") + stdout.styledWrite(fgYellow, header[1]) + stdout.styledWrite(fgWhite, " | ") + stdout.styledWrite(fgBlue, header[2]) + stdout.styledWrite(fgWhite, " | ") + stdout.styledWrite(fgMagenta, header[3]) + stdout.styledWrite(fgWhite, " | ") + stdout.styledWrite(fgCyan, header[4] & "\p") + for i, val in header: + if i < 4: + outputFile.write(escapeCsvField(val) & ",") + else: + outputFile.write(escapeCsvField(val) & "\p") + for rec in summaryRecords.values: + stdout.styledWrite(fgGreen, rec[0]) + stdout.styledWrite(fgWhite, " | ") + stdout.styledWrite(fgYellow, rec[1]) + stdout.styledWrite(fgWhite, " | ") + stdout.styledWrite(fgBlue, rec[2]) + stdout.styledWrite(fgWhite, " | ") + stdout.styledWrite(fgMagenta, rec[3]) + stdout.styledWrite(fgWhite, " | ") + stdout.styledWrite(fgCyan, rec[4] & "\p") + for i, val in rec: + if i < 4: + outputFile.write(escapeCsvField(val) & ",") + else: + outputFile.write(escapeCsvField(val) & "\p") + let outputFileSize = getFileSize(outputFile) + outputFile.close() + echo "" + echo "Saved summary file: " & output & " (" & formatFileSize(outputFileSize) & ")" + + let endTime = epochTime() + let elapsedTime = int(endTime - startTime) + let hours = elapsedTime div 3600 + let minutes = (elapsedTime mod 3600) div 60 + let seconds = elapsedTime mod 60 + echo "" + echo "Elapsed time: ", $hours & " hours, " & $minutes & " minutes, " & $seconds & " seconds" + echo "" \ No newline at end of file From c77ca6078ab218f9e598f4c323d99bcf07fcd745 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Sat, 21 Oct 2023 23:28:02 +0900 Subject: [PATCH 02/12] chg: display output directory explicitly --- src/takajopkg/extractScriptblocks.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim index cd50c6f8..4e82fa12 100644 --- a/src/takajopkg/extractScriptblocks.nim +++ b/src/takajopkg/extractScriptblocks.nim @@ -133,7 +133,8 @@ proc extractScriptblocks(output: string = "scriptblock-logs", let outputFileSize = getFileSize(outputFile) outputFile.close() echo "" - echo "Saved summary file: " & output & " (" & formatFileSize(outputFileSize) & ")" + echo "The extracted PowerShell ScriptBlock is saved in the directory: " & output + echo "Saved summary file: " & summaryFile & " (" & formatFileSize(outputFileSize) & ")" let endTime = epochTime() let elapsedTime = int(endTime - startTime) From 798b84e9ff30b1602e5a3ff1f602e515d87921a5 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Sun, 22 Oct 2023 00:04:07 +0900 Subject: [PATCH 03/12] chg: replace \t to tab --- src/takajopkg/extractScriptblocks.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim index 4e82fa12..30e49179 100644 --- a/src/takajopkg/extractScriptblocks.nim +++ b/src/takajopkg/extractScriptblocks.nim @@ -8,7 +8,7 @@ proc outputScriptText(output: string, timestamp: string, computerName: string, scriptObj: Script) = var scriptText = "" for text in scriptObj.scriptBlocks.items: - scriptText = scriptText & text.replace("\\r\\n", "\p").replace("\\\"", "\"") + scriptText = scriptText & text.replace("\\r\\n", "\p").replace("\\\"", "\"").replace("\\t", "\t") let date = timestamp.replace(":", "_").replace(" ", "_") let fileName = output & "/" & computerName & "-" & date & "-" & scriptObj.scriptBlockId & ".txt" var outputFile = open(filename, fmWrite) From ecdb159bb96e292f5558253d787226d990f7d2cb Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Tue, 24 Oct 2023 09:44:17 +0900 Subject: [PATCH 04/12] chg: use nancy for terminal table output --- src/takajo.nim | 2 + src/takajopkg/extractScriptblocks.nim | 74 +++++++++++++++++---------- takajo.nimble | 4 +- 3 files changed, 52 insertions(+), 28 deletions(-) diff --git a/src/takajo.nim b/src/takajo.nim index 4c391285..b8f0669f 100644 --- a/src/takajo.nim +++ b/src/takajo.nim @@ -1,6 +1,7 @@ import algorithm import cligen import json +import nancy import puppy import re import sets @@ -9,6 +10,7 @@ import strformat import strutils import tables import terminal +import termstyle import times import threadpool import uri diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim index 30e49179..37695306 100644 --- a/src/takajopkg/extractScriptblocks.nim +++ b/src/takajopkg/extractScriptblocks.nim @@ -3,6 +3,8 @@ type scriptBlockId: string firstTimestamp: string scriptBlocks: OrderedSet[string] + levels: HashSet[string] + ruleTitles: HashSet[string] proc outputScriptText(output: string, timestamp: string, computerName: string, scriptObj: Script) = @@ -16,8 +18,22 @@ proc outputScriptText(output: string, timestamp: string, computerName: string, flushFile(outputFile) close(outputFile) + +proc calcMaxAlert(levels:HashSet):string = + if "crit" in levels: + return "crit" + if "high" in levels: + return "high" + if "med" in levels: + return "med" + if "low" in levels: + return "low" + if "info" in levels: + return "info" + return "N/A" + proc buildSummaryRecord(path: string, messageTotal: int, - scriptObj: Script): array[5, string] = + scriptObj: Script): array[7, string] = let ts = scriptObj.firstTimestamp let id = scriptObj.scriptBlockId let count = scriptObj.scriptBlocks.len @@ -25,7 +41,9 @@ proc buildSummaryRecord(path: string, messageTotal: int, if count != messageTotal: status = "Incomplete" let records = $count & "/" & $messageTotal - return [ts, id, path, status, records] + let ruleTitles = fmt"{scriptObj.ruleTitles}".replace("{", "").replace("}", "") + let maxLevel = calcMaxAlert(scriptObj.levels) + return [ts, id, path, status, records, maxLevel, ruleTitles] proc extractScriptblocks(output: string = "scriptblock-logs", quiet: bool = false, timeline: string) = @@ -59,7 +77,7 @@ proc extractScriptblocks(output: string = "scriptblock-logs", var bar: SuruBar = initSuruBar() stackedRecords = newTable[string, Script]() - summaryRecords = newOrderedTable[string, array[5, string]]() + summaryRecords = newOrderedTable[string, array[7, string]]() bar[0].total = totalLines bar.setup() @@ -74,6 +92,8 @@ proc extractScriptblocks(output: string = "scriptblock-logs", let timestamp = jsonLine["Timestamp"].getStr() computerName = jsonLine["Computer"].getStr() + level = jsonLine["Level"].getStr() + ruleTitle = jsonLine["RuleTitle"].getStr() scriptBlock = jsonLine["Details"]["ScriptBlock"].getStr() scriptBlockId = jsonLine["ExtraFieldInfo"]["ScriptBlockId"].getStr() messageNumber = jsonLine["ExtraFieldInfo"]["MessageNumber"].getInt() @@ -83,55 +103,55 @@ proc extractScriptblocks(output: string = "scriptblock-logs", path = "no-path" if scriptBlockId in stackedRecords: + stackedRecords[scriptBlockId].levels.incl(level) + stackedRecords[scriptBlockId].ruleTitles.incl(ruleTitle) stackedRecords[scriptBlockId].scriptBlocks.incl(scriptBlock) else: stackedRecords[scriptBlockId] = Script(scriptBlockId: scriptBlockId, - firstTimestamp: timestamp, scriptBlocks: toOrderedSet([scriptBlock])) + firstTimestamp: timestamp, scriptBlocks: toOrderedSet([scriptBlock]), + levels:toHashSet([level]), ruleTitles:toHashSet([ruleTitle])) if messageNumber == messageTotal: + let scriptObj = stackedRecords[scriptBlockId] if scriptBlockId in summaryRecords: + summaryRecords[scriptBlockId] = buildSummaryRecord(path, messageTotal, scriptObj) # Already outputted continue - let scriptObj = stackedRecords[scriptBlockId] outputScriptText(output, timestamp, computerName, scriptObj) summaryRecords[scriptBlockId] = buildSummaryRecord(path, messageTotal, scriptObj) bar.finish() let summaryFile = output & "/" & "summary.csv" - let header = ["Creation Time", "Script ID", "Script Name", "Results", "Extracted Records"] + let header = ["Creation Time", "Script ID", "Script Name", "Results", "Extracted Records", "Max alert", "Alerts"] var outputFile = open(summaryFile, fmWrite) - echo "" - stdout.styledWrite(fgGreen, header[0]) - stdout.styledWrite(fgWhite, " | ") - stdout.styledWrite(fgYellow, header[1]) - stdout.styledWrite(fgWhite, " | ") - stdout.styledWrite(fgBlue, header[2]) - stdout.styledWrite(fgWhite, " | ") - stdout.styledWrite(fgMagenta, header[3]) - stdout.styledWrite(fgWhite, " | ") - stdout.styledWrite(fgCyan, header[4] & "\p") + var table: TerminalTable + table.add header[0], header[1], header[2], header[3], header[4], header[5], header[6] for i, val in header: - if i < 4: + if i < 6: outputFile.write(escapeCsvField(val) & ",") else: outputFile.write(escapeCsvField(val) & "\p") for rec in summaryRecords.values: - stdout.styledWrite(fgGreen, rec[0]) - stdout.styledWrite(fgWhite, " | ") - stdout.styledWrite(fgYellow, rec[1]) - stdout.styledWrite(fgWhite, " | ") - stdout.styledWrite(fgBlue, rec[2]) - stdout.styledWrite(fgWhite, " | ") - stdout.styledWrite(fgMagenta, rec[3]) - stdout.styledWrite(fgWhite, " | ") - stdout.styledWrite(fgCyan, rec[4] & "\p") + if rec[5] == "crit": + table.add red rec[0], red rec[1], red rec[2], red rec[3], red rec[4], red rec[5], red rec[6] + elif rec[5] == "high": + table.add yellow rec[0], yellow rec[1], yellow rec[2], yellow rec[3], yellow rec[4], yellow rec[5], yellow rec[6] + elif rec[5] == "med": + table.add cyan rec[0], cyan rec[1], cyan rec[2], cyan rec[3], cyan rec[4], cyan rec[5], cyan rec[6] + elif rec[5] == "low": + table.add green rec[0], green rec[1], green rec[2], green rec[3], green rec[4], green rec[5], green rec[6] + elif rec[5] == "info": + table.add rec[0], rec[1], rec[2], rec[3], rec[4], rec[5], rec[6] + else: + table.add rec[0], rec[1], rec[2], rec[3], rec[4], rec[5], rec[6] for i, val in rec: - if i < 4: + if i < 6: outputFile.write(escapeCsvField(val) & ",") else: outputFile.write(escapeCsvField(val) & "\p") let outputFileSize = getFileSize(outputFile) outputFile.close() + table.echoTableSeps(seps = boxSeps) echo "" echo "The extracted PowerShell ScriptBlock is saved in the directory: " & output echo "Saved summary file: " & summaryFile & " (" & formatFileSize(outputFileSize) & ")" diff --git a/takajo.nimble b/takajo.nimble index e0dd3209..e4678288 100644 --- a/takajo.nimble +++ b/takajo.nimble @@ -14,4 +14,6 @@ bin = @["takajo"] requires "nim >= 1.6.12" requires "cligen >= 1.5" requires "suru#f6f1e607c585b2bc2f71309996643f0555ff6349" -requires "puppy >= 2.1.0" \ No newline at end of file +requires "puppy >= 2.1.0" +requires "termstyle" +requires "nancy" \ No newline at end of file From a12e39cb1c8d172d5f9d6fc44089dc713db38d19 Mon Sep 17 00:00:00 2001 From: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Date: Tue, 24 Oct 2023 14:17:24 +0900 Subject: [PATCH 05/12] changelog update --- CHANGELOG-Japanese.md | 4 ++++ CHANGELOG.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index 0c575af4..cc9ca193 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -2,6 +2,10 @@ ## x.x.x [xxxx/xx/xx] +**新機能:** + +- PowerShell EID 4104のScriptBlockログを元に戻す`extract-scriptblocks`コマンドを追加した。 (#47) (@fukusuket) + **改善:** - TakajoがNim 2.0.0でコンパイルできるようになった。(#31) (@fukusuket) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0eee8349..e62b797a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## x.x.x [xxxx/xx/xx] +**New Features:** + +- New `extract-scriptblocks` command to reassemble PowerShell EID 4104 ScriptBlock logs. (#47) (@fukusuket) + **Enhancements:** - Takajo now compiles with Nim 2.0.0. (#31) (@fukusuket) From bce67c73d71b8fffe62284b040f424b4ad283a7e Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Tue, 24 Oct 2023 17:53:32 +0900 Subject: [PATCH 06/12] feat: add --level option --- src/takajo.nim | 3 +- src/takajopkg/extractScriptblocks.nim | 96 ++++++++++++++++----------- 2 files changed, 59 insertions(+), 40 deletions(-) diff --git a/src/takajo.nim b/src/takajo.nim index b8f0669f..1883040a 100644 --- a/src/takajo.nim +++ b/src/takajo.nim @@ -37,7 +37,7 @@ include takajopkg/vtHashLookup when isMainModule: clCfg.version = "2.1.0-dev" const examples = "Examples:\p" - const example_extract_scriptblocks = " extract-scriptblocks -t ../hayabusa/timeline.jsonl -o scriptblock-logs\p" + const example_extract_scriptblocks = " extract-scriptblocks -t ../hayabusa/timeline.jsonl [--level low] -o scriptblock-logs\p" const example_list_domains = " list-domains -t ../hayabusa/timeline.jsonl -o domains.txt\p" const example_list_ip_addresses = " list-ip-addresses -t ../hayabusa/timeline.jsonl -o ipAddresses.txt\p" const example_list_undetected_evtx = " list-undetected-evtx -t ../hayabusa/timeline.csv -e ../hayabusa-sample-evtx\p" @@ -66,6 +66,7 @@ when isMainModule: extractScriptblocks, cmdName = "extract-scriptblocks", doc = "extract and reassemble PowerShell EID 4104 script block logs", help = { + "level": "specify the minimum alert level", "output": "output directory (default: scriptblock-logs)", "quiet": "do not display the launch banner", "timeline": "Hayabusa JSONL timeline (profile: any)", diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim index 37695306..247d202b 100644 --- a/src/takajopkg/extractScriptblocks.nim +++ b/src/takajopkg/extractScriptblocks.nim @@ -45,7 +45,7 @@ proc buildSummaryRecord(path: string, messageTotal: int, let maxLevel = calcMaxAlert(scriptObj.levels) return [ts, id, path, status, records, maxLevel, ruleTitles] -proc extractScriptblocks(output: string = "scriptblock-logs", +proc extractScriptblocks(level: string = "low", output: string = "scriptblock-logs", quiet: bool = false, timeline: string) = let startTime = epochTime() if not quiet: @@ -58,6 +58,11 @@ proc extractScriptblocks(output: string = "scriptblock-logs", if not isJsonConvertible(timeline): quit(1) + if level != "critical" and level != "high" and level != "medium" and level != "low" and level != "informational": + echo "You must specify a minimum level of critical, high, medium, low or informational. (default: low)" + echo "" + return + echo "Started the Extract ScriptBlock command." echo "This command will extract PowerShell Script Block." echo "" @@ -66,16 +71,20 @@ proc extractScriptblocks(output: string = "scriptblock-logs", let totalLines = countLinesInTimeline(timeline) echo "Total lines: ", totalLines echo "" - echo "Extracting PowerShell ScriptBlocks. Please wait." + if level == "critical": + echo "Extracting PowerShell ScriptBlocks with an alert level of critical. Please wait." + else: + echo "Extracting PowerShell ScriptBlocks with a minimal alert level of " & level & ". Please wait." + echo "" if not dirExists(output): - echo "" echo "The directory '" & output & "' does not exist so will be created." createDir(output) - echo "" + echo "" var bar: SuruBar = initSuruBar() + currentIndex = 0 stackedRecords = newTable[string, Script]() summaryRecords = newOrderedTable[string, array[7, string]]() @@ -83,16 +92,17 @@ proc extractScriptblocks(output: string = "scriptblock-logs", bar.setup() for line in lines(timeline): + inc currentIndex inc bar bar.update(1000000000) # refresh every second let jsonLine = parseJson(line) - if jsonLine["EventID"].getInt(0) != 4104: + let eventLevel = jsonLine["Level"].getStr() + if jsonLine["EventID"].getInt(0) != 4104 or isMinLevel(eventLevel, level) == false: continue let timestamp = jsonLine["Timestamp"].getStr() computerName = jsonLine["Computer"].getStr() - level = jsonLine["Level"].getStr() ruleTitle = jsonLine["RuleTitle"].getStr() scriptBlock = jsonLine["Details"]["ScriptBlock"].getStr() scriptBlockId = jsonLine["ExtraFieldInfo"]["ScriptBlockId"].getStr() @@ -103,58 +113,66 @@ proc extractScriptblocks(output: string = "scriptblock-logs", path = "no-path" if scriptBlockId in stackedRecords: - stackedRecords[scriptBlockId].levels.incl(level) + stackedRecords[scriptBlockId].levels.incl(eventLevel) stackedRecords[scriptBlockId].ruleTitles.incl(ruleTitle) stackedRecords[scriptBlockId].scriptBlocks.incl(scriptBlock) else: stackedRecords[scriptBlockId] = Script(scriptBlockId: scriptBlockId, firstTimestamp: timestamp, scriptBlocks: toOrderedSet([scriptBlock]), - levels:toHashSet([level]), ruleTitles:toHashSet([ruleTitle])) + levels:toHashSet([eventLevel]), ruleTitles:toHashSet([ruleTitle])) + let scriptObj = stackedRecords[scriptBlockId] if messageNumber == messageTotal: - let scriptObj = stackedRecords[scriptBlockId] if scriptBlockId in summaryRecords: summaryRecords[scriptBlockId] = buildSummaryRecord(path, messageTotal, scriptObj) # Already outputted continue outputScriptText(output, timestamp, computerName, scriptObj) summaryRecords[scriptBlockId] = buildSummaryRecord(path, messageTotal, scriptObj) + elif currentIndex + 1 == totalLines: + outputScriptText(output, timestamp, computerName, scriptObj) + summaryRecords[scriptBlockId] = buildSummaryRecord(path, messageTotal, scriptObj) + bar.finish() + echo "" - let summaryFile = output & "/" & "summary.csv" - let header = ["Creation Time", "Script ID", "Script Name", "Results", "Extracted Records", "Max alert", "Alerts"] - var outputFile = open(summaryFile, fmWrite) - var table: TerminalTable - table.add header[0], header[1], header[2], header[3], header[4], header[5], header[6] - for i, val in header: - if i < 6: - outputFile.write(escapeCsvField(val) & ",") - else: - outputFile.write(escapeCsvField(val) & "\p") - for rec in summaryRecords.values: - if rec[5] == "crit": - table.add red rec[0], red rec[1], red rec[2], red rec[3], red rec[4], red rec[5], red rec[6] - elif rec[5] == "high": - table.add yellow rec[0], yellow rec[1], yellow rec[2], yellow rec[3], yellow rec[4], yellow rec[5], yellow rec[6] - elif rec[5] == "med": - table.add cyan rec[0], cyan rec[1], cyan rec[2], cyan rec[3], cyan rec[4], cyan rec[5], cyan rec[6] - elif rec[5] == "low": - table.add green rec[0], green rec[1], green rec[2], green rec[3], green rec[4], green rec[5], green rec[6] - elif rec[5] == "info": - table.add rec[0], rec[1], rec[2], rec[3], rec[4], rec[5], rec[6] - else: - table.add rec[0], rec[1], rec[2], rec[3], rec[4], rec[5], rec[6] - for i, val in rec: + if summaryRecords.len == 0: + echo "No malicious powershell script block were found. There are either no malicious powershell script block or you need to change the level." + else: + let summaryFile = output & "/" & "summary.csv" + let header = ["Creation Time", "Script ID", "Script Name", "Results", "Extracted Records", "Max alert", "Alerts"] + var outputFile = open(summaryFile, fmWrite) + var table: TerminalTable + table.add header[0], header[1], header[2], header[3], header[4], header[5], header[6] + for i, val in header: if i < 6: outputFile.write(escapeCsvField(val) & ",") else: outputFile.write(escapeCsvField(val) & "\p") - let outputFileSize = getFileSize(outputFile) - outputFile.close() - table.echoTableSeps(seps = boxSeps) - echo "" - echo "The extracted PowerShell ScriptBlock is saved in the directory: " & output - echo "Saved summary file: " & summaryFile & " (" & formatFileSize(outputFileSize) & ")" + for v in summaryRecords.values: + if v[5] == "crit": + table.add red v[0], red v[1], red v[2], red v[3], red v[4], red v[5], red v[6] + elif v[5] == "high": + table.add yellow v[0], yellow v[1], yellow v[2], yellow v[3], yellow v[4], yellow v[5], yellow v[6] + elif v[5] == "med": + table.add cyan v[0], cyan v[1], cyan v[2], cyan v[3], cyan v[4], cyan v[5], cyan v[6] + elif v[5] == "low": + table.add green v[0], green v[1], green v[2], green v[3], green v[4], green v[5], green v[6] + elif v[5] == "info": + table.add v[0], v[1], v[2], v[3], v[4], v[5], v[6] + else: + table.add v[0], v[1], v[2], v[3], v[4], v[5], v[6] + for i, val in v: + if i < 6: + outputFile.write(escapeCsvField(val) & ",") + else: + outputFile.write(escapeCsvField(val) & "\p") + let outputFileSize = getFileSize(outputFile) + outputFile.close() + table.echoTableSeps(seps = boxSeps) + echo "" + echo "The extracted PowerShell ScriptBlock is saved in the directory: " & output + echo "Saved summary file: " & summaryFile & " (" & formatFileSize(outputFileSize) & ")" let endTime = epochTime() let elapsedTime = int(endTime - startTime) From a8ce71f545b6adb2e67f24a5fab14109d2938338 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Wed, 25 Oct 2023 13:11:36 +0900 Subject: [PATCH 07/12] chg: override echoTableSeps to use stdout.styledWrite instead of ansi escape --- src/takajopkg/extractScriptblocks.nim | 35 ++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim index 247d202b..ddf66e77 100644 --- a/src/takajopkg/extractScriptblocks.nim +++ b/src/takajopkg/extractScriptblocks.nim @@ -45,6 +45,39 @@ proc buildSummaryRecord(path: string, messageTotal: int, let maxLevel = calcMaxAlert(scriptObj.levels) return [ts, id, path, status, records, maxLevel, ruleTitles] +proc colorWrite(color: ForegroundColor, ansiEscape: string, txt: string) = + let replacedTxt = txt.replace(ansiEscape,"").replace(termClear,"") + if "│" in replacedTxt: + stdout.styledWrite(color, replacedTxt.replace("│ ","")) + stdout.write "│ " + else: + stdout.styledWrite(color, replacedTxt) + +proc stdoutStyledWrite(txt: string) = + if txt.startsWith(termRed): + colorWrite(fgRed, termRed,txt) + elif txt.startsWith(termGreen): + colorWrite(fgGreen, termGreen, txt) + elif txt.startsWith(termYellow): + colorWrite(fgYellow, termYellow, txt) + elif txt.startsWith(termCyan): + colorWrite(fgCyan, termCyan, txt) + else: + stdout.write txt + +proc echoTableSepsWithStyled*(table: TerminalTable, maxSize = terminalWidth(), seps = defaultSeps) = + let sizes = table.getColumnSizes(maxSize - 4, padding = 3) + printSeparator(top) + for k, entry in table.entries(sizes): + for _, row in entry(): + stdout.write seps.vertical & " " + for i, cell in row(): + stdoutStyledWrite cell & (if i != sizes.high: " " & seps.vertical & " " else: "") + stdout.write " " & seps.vertical & "\n" + if k != table.rows - 1: + printSeparator(center) + printSeparator(bottom) + proc extractScriptblocks(level: string = "low", output: string = "scriptblock-logs", quiet: bool = false, timeline: string) = let startTime = epochTime() @@ -169,7 +202,7 @@ proc extractScriptblocks(level: string = "low", output: string = "scriptblock-lo outputFile.write(escapeCsvField(val) & "\p") let outputFileSize = getFileSize(outputFile) outputFile.close() - table.echoTableSeps(seps = boxSeps) + table.echoTableSepsWithStyled(seps = boxSeps) echo "" echo "The extracted PowerShell ScriptBlock is saved in the directory: " & output echo "Saved summary file: " & summaryFile & " (" & formatFileSize(outputFileSize) & ")" From 8992efaf267bce227d9966aa7799fd5053321b72 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Wed, 25 Oct 2023 13:17:57 +0900 Subject: [PATCH 08/12] fix: remove termClear --- src/takajopkg/extractScriptblocks.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim index ddf66e77..166e41f2 100644 --- a/src/takajopkg/extractScriptblocks.nim +++ b/src/takajopkg/extractScriptblocks.nim @@ -63,7 +63,7 @@ proc stdoutStyledWrite(txt: string) = elif txt.startsWith(termCyan): colorWrite(fgCyan, termCyan, txt) else: - stdout.write txt + stdout.write txt.replace(termClear,"") proc echoTableSepsWithStyled*(table: TerminalTable, maxSize = terminalWidth(), seps = defaultSeps) = let sizes = table.getColumnSizes(maxSize - 4, padding = 3) From 24863a3dd420de4ed704f1aa3a45e3022d24f53b Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Wed, 25 Oct 2023 18:10:05 +0900 Subject: [PATCH 09/12] refactor: pass the array as is --- src/takajopkg/extractScriptblocks.nim | 59 ++++++++++++++------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim index 166e41f2..d8b06e74 100644 --- a/src/takajopkg/extractScriptblocks.nim +++ b/src/takajopkg/extractScriptblocks.nim @@ -46,6 +46,7 @@ proc buildSummaryRecord(path: string, messageTotal: int, return [ts, id, path, status, records, maxLevel, ruleTitles] proc colorWrite(color: ForegroundColor, ansiEscape: string, txt: string) = + # Remove ANSI escape sequences and use stdout.styledWrite instead let replacedTxt = txt.replace(ansiEscape,"").replace(termClear,"") if "│" in replacedTxt: stdout.styledWrite(color, replacedTxt.replace("│ ","")) @@ -65,18 +66,21 @@ proc stdoutStyledWrite(txt: string) = else: stdout.write txt.replace(termClear,"") -proc echoTableSepsWithStyled*(table: TerminalTable, maxSize = terminalWidth(), seps = defaultSeps) = - let sizes = table.getColumnSizes(maxSize - 4, padding = 3) - printSeparator(top) - for k, entry in table.entries(sizes): - for _, row in entry(): - stdout.write seps.vertical & " " - for i, cell in row(): - stdoutStyledWrite cell & (if i != sizes.high: " " & seps.vertical & " " else: "") - stdout.write " " & seps.vertical & "\n" - if k != table.rows - 1: - printSeparator(center) - printSeparator(bottom) +proc echoTableSepsWithStyled(table: TerminalTable, maxSize = terminalWidth(), seps = defaultSeps) = + # This function equivalent to echoTableSeps without using ANSI escape to avoid the following issue. + # https://github.com/PMunch/nancy/issues/4 + # https://github.com/PMunch/nancy/blob/9918716a563f64d740df6a02db42662781e94fc8/src/nancy.nim#L195C6-L195C19 + let sizes = table.getColumnSizes(maxSize - 4, padding = 3) + printSeparator(top) + for k, entry in table.entries(sizes): + for _, row in entry(): + stdout.write seps.vertical & " " + for i, cell in row(): + stdoutStyledWrite cell & (if i != sizes.high: " " & seps.vertical & " " else: "") + stdout.write " " & seps.vertical & "\n" + if k != table.rows - 1: + printSeparator(center) + printSeparator(bottom) proc extractScriptblocks(level: string = "low", output: string = "scriptblock-logs", quiet: bool = false, timeline: string) = @@ -176,30 +180,29 @@ proc extractScriptblocks(level: string = "low", output: string = "scriptblock-lo let header = ["Creation Time", "Script ID", "Script Name", "Results", "Extracted Records", "Max alert", "Alerts"] var outputFile = open(summaryFile, fmWrite) var table: TerminalTable - table.add header[0], header[1], header[2], header[3], header[4], header[5], header[6] + table.add header for i, val in header: if i < 6: outputFile.write(escapeCsvField(val) & ",") else: outputFile.write(escapeCsvField(val) & "\p") - for v in summaryRecords.values: - if v[5] == "crit": - table.add red v[0], red v[1], red v[2], red v[3], red v[4], red v[5], red v[6] - elif v[5] == "high": - table.add yellow v[0], yellow v[1], yellow v[2], yellow v[3], yellow v[4], yellow v[5], yellow v[6] - elif v[5] == "med": - table.add cyan v[0], cyan v[1], cyan v[2], cyan v[3], cyan v[4], cyan v[5], cyan v[6] - elif v[5] == "low": - table.add green v[0], green v[1], green v[2], green v[3], green v[4], green v[5], green v[6] - elif v[5] == "info": - table.add v[0], v[1], v[2], v[3], v[4], v[5], v[6] + for row in summaryRecords.values: + let level = row[5] + if level == "crit": + table.add red row + elif level == "high": + table.add yellow row + elif level == "med": + table.add cyan row + elif level == "low": + table.add green row else: - table.add v[0], v[1], v[2], v[3], v[4], v[5], v[6] - for i, val in v: + table.add row + for i, cell in row: if i < 6: - outputFile.write(escapeCsvField(val) & ",") + outputFile.write(escapeCsvField(cell) & ",") else: - outputFile.write(escapeCsvField(val) & "\p") + outputFile.write(escapeCsvField(cell) & "\p") let outputFileSize = getFileSize(outputFile) outputFile.close() table.echoTableSepsWithStyled(seps = boxSeps) From 3ddd301135cd0afc250000b4808ff12b1f02f4e4 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Wed, 25 Oct 2023 18:14:13 +0900 Subject: [PATCH 10/12] fix: revert to processing element by element --- src/takajopkg/extractScriptblocks.nim | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim index d8b06e74..0b56ed21 100644 --- a/src/takajopkg/extractScriptblocks.nim +++ b/src/takajopkg/extractScriptblocks.nim @@ -186,19 +186,19 @@ proc extractScriptblocks(level: string = "low", output: string = "scriptblock-lo outputFile.write(escapeCsvField(val) & ",") else: outputFile.write(escapeCsvField(val) & "\p") - for row in summaryRecords.values: - let level = row[5] - if level == "crit": - table.add red row - elif level == "high": - table.add yellow row - elif level == "med": - table.add cyan row - elif level == "low": - table.add green row + for v in summaryRecords.values: + let level = v[5] + if v[5] == "crit": + table.add red v[0], red v[1], red v[2], red v[3], red v[4], red v[5], red v[6] + elif v[5] == "high": + table.add yellow v[0], yellow v[1], yellow v[2], yellow v[3], yellow v[4], yellow v[5], yellow v[6] + elif v[5] == "med": + table.add cyan v[0], cyan v[1], cyan v[2], cyan v[3], cyan v[4], cyan v[5], cyan v[6] + elif v[5] == "low": + table.add green v[0], green v[1], green v[2], green v[3], green v[4], green v[5], green v[6] else: - table.add row - for i, cell in row: + table.add v + for i, cell in v: if i < 6: outputFile.write(escapeCsvField(cell) & ",") else: From 5d112be38155f2b20e99ac10053343e49b096170 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Wed, 25 Oct 2023 18:15:08 +0900 Subject: [PATCH 11/12] fix: remove unused variable --- src/takajopkg/extractScriptblocks.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim index 0b56ed21..50d99519 100644 --- a/src/takajopkg/extractScriptblocks.nim +++ b/src/takajopkg/extractScriptblocks.nim @@ -187,7 +187,6 @@ proc extractScriptblocks(level: string = "low", output: string = "scriptblock-lo else: outputFile.write(escapeCsvField(val) & "\p") for v in summaryRecords.values: - let level = v[5] if v[5] == "crit": table.add red v[0], red v[1], red v[2], red v[3], red v[4], red v[5], red v[6] elif v[5] == "high": From 62d4a5188aeb557ba1b111b5404f0766c4ca349b Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Thu, 26 Oct 2023 08:44:47 +0900 Subject: [PATCH 12/12] chg: add computerName column. --- src/takajopkg/extractScriptblocks.nim | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim index 50d99519..3dbfb34d 100644 --- a/src/takajopkg/extractScriptblocks.nim +++ b/src/takajopkg/extractScriptblocks.nim @@ -1,7 +1,8 @@ type Script = object - scriptBlockId: string firstTimestamp: string + computerName: string + scriptBlockId: string scriptBlocks: OrderedSet[string] levels: HashSet[string] ruleTitles: HashSet[string] @@ -35,15 +36,13 @@ proc calcMaxAlert(levels:HashSet):string = proc buildSummaryRecord(path: string, messageTotal: int, scriptObj: Script): array[7, string] = let ts = scriptObj.firstTimestamp + let cs = scriptObj.computerName let id = scriptObj.scriptBlockId let count = scriptObj.scriptBlocks.len - var status = "Complete" - if count != messageTotal: - status = "Incomplete" let records = $count & "/" & $messageTotal let ruleTitles = fmt"{scriptObj.ruleTitles}".replace("{", "").replace("}", "") let maxLevel = calcMaxAlert(scriptObj.levels) - return [ts, id, path, status, records, maxLevel, ruleTitles] + return [ts, cs, id, path, records, maxLevel, ruleTitles] proc colorWrite(color: ForegroundColor, ansiEscape: string, txt: string) = # Remove ANSI escape sequences and use stdout.styledWrite instead @@ -154,8 +153,10 @@ proc extractScriptblocks(level: string = "low", output: string = "scriptblock-lo stackedRecords[scriptBlockId].ruleTitles.incl(ruleTitle) stackedRecords[scriptBlockId].scriptBlocks.incl(scriptBlock) else: - stackedRecords[scriptBlockId] = Script(scriptBlockId: scriptBlockId, - firstTimestamp: timestamp, scriptBlocks: toOrderedSet([scriptBlock]), + stackedRecords[scriptBlockId] = Script(firstTimestamp: timestamp, + computerName: computerName, + scriptBlockId: scriptBlockId, + scriptBlocks: toOrderedSet([scriptBlock]), levels:toHashSet([eventLevel]), ruleTitles:toHashSet([ruleTitle])) let scriptObj = stackedRecords[scriptBlockId] @@ -177,7 +178,7 @@ proc extractScriptblocks(level: string = "low", output: string = "scriptblock-lo echo "No malicious powershell script block were found. There are either no malicious powershell script block or you need to change the level." else: let summaryFile = output & "/" & "summary.csv" - let header = ["Creation Time", "Script ID", "Script Name", "Results", "Extracted Records", "Max alert", "Alerts"] + let header = ["Creation Time", "Computer Name", "Script ID", "Script Name", "Records", "Level", "Alerts"] var outputFile = open(summaryFile, fmWrite) var table: TerminalTable table.add header