Just log analysis

.github/workflows/tests.yml

@@ -11,7 +11,7 @@ jobs:
  runs-on: ubuntu-latest
  strategy:
  matrix:
- benchmark: [nlp, file-enc, unix50]
+ benchmark: [nlp, file-enc, unix50, log-analysis]
 
  steps:
  - name: Checkout code

log-analysis/.gitignore

@@ -0,0 +1,2 @@
+input/
+results/

log-analysis/cleanup.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
-cd "$(realpath $(dirname "$0"))"
+REPO_TOP=$(git rev-parse --show-toplevel)
+results_dir="${REPO_TOP}/log-analysis/input"
+results_dir="${REPO_TOP}/log-analysis/results"
 
-rm -rf ./inputs
-rm -rf ./outputs
+rm -rf $input_dir
+rm -rf $results_dir

log-analysis/deps.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+sudo apt update 
+
+sudo apt install -y --no-install-recommends \
+ sudo \
+ tcpdump \
+ curl \
+ wget \
+ unzip \
+ samtools \
+ minimap2 \
+ bcftools \
+ python3-pip \
+ vim \
+ ffmpeg unrtf imagemagick libarchive-tools libncurses5-dev libncursesw5-dev zstd liblzma-dev libbz2-dev zip unzip nodejs tcpdump \
+ git
+

log-analysis/hashes/nginx.full.md5sum

@@ -0,0 +1,8 @@
+e1db42fe14aa938fb2f34bfc5735fd1b nginx.full/log0
+aaf7960c4ba2c7bf92db9590fdb38542 nginx.full/log1
+27633e1cfb60207d4ac901614f463a9f nginx.full/log2
+fd419b81767eda4cc9e5aaa1818b487c nginx.full/log3
+79450f4c4a79cb71ef96d7bc03943a98 nginx.full/log4
+db4185dd14527deb56e09f0099ad43b1 nginx.full/log5
+879fe7329abf549d56706f601f2a120f nginx.full/log6
+926b41adef30d4695d85cc12ece29334 nginx.full/log7

log-analysis/hashes/nginx.small.md5sum

@@ -0,0 +1,8 @@
+e1db42fe14aa938fb2f34bfc5735fd1b nginx.small/log0
+aaf7960c4ba2c7bf92db9590fdb38542 nginx.small/log1
+27633e1cfb60207d4ac901614f463a9f nginx.small/log2
+fd419b81767eda4cc9e5aaa1818b487c nginx.small/log3
+79450f4c4a79cb71ef96d7bc03943a98 nginx.small/log4
+db4185dd14527deb56e09f0099ad43b1 nginx.small/log5
+879fe7329abf549d56706f601f2a120f nginx.small/log6
+926b41adef30d4695d85cc12ece29334 nginx.small/log7

log-analysis/hashes/pcaps.full.md5sum

@@ -0,0 +1,69 @@
+4e489b35942d0c45d8b16d1cb1a06210 pcaps.full/4SICS-GeekLounge-151020.pcap.log
+890b35d7af03d13aab446e306ba60782 pcaps.full/4SICS-GeekLounge-151021.pcap.log
+1ebd451920d098aa4b7002054900a0b8 pcaps.full/4SICS-GeekLounge-151022.pcap.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/README.log
+f5b48dd8c8037efc43951efc3e1e16af pcaps.full/challenge101-0.pcapng.log
+6b2c4e89211bc977ac538ad218845794 pcaps.full/challenge101-1.pcapng.log
+765d385418861c272c081f85abee4448 pcaps.full/challenge101-3.pcapng.log
+6c5777e9a6a4601487f5bb0d637eaef0 pcaps.full/challenge101-4.pcapng.log
+2b632ea272f075b62a1779e6b79c262b pcaps.full/challenge101-5.pcapng.log
+431993ece40a4f8081cc949661be3b98 pcaps.full/challenge101-6.pcapng.log
+fe1fcee267a72f20de6abd423344b247 pcaps.full/challenge101-7.pcapng.log
+63382ae6dddcc347be2f140853ab57a3 pcaps.full/challenge101-8.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/dhcp-serverdiscovery101.pcapng.log
+9267a406fc3cca0cb4d7b755169df6be pcaps.full/dns-nmap101.pcapng.log
+42164f80e7ff0177cf36922c2caf7917 pcaps.full/f5-honeypot-release.pcap.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/ftp-bounce.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/ftp-clientside101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/ftp-crack101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/ftp-download101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/ftp-passwords101.pcapng.log
+ecb77db5b2688d25093117caaa9cf4f6 pcaps.full/gen-startupchatty101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/general101.pcapng.log
+3684f199efa845a7ec8f1053a9944caf pcaps.full/general101b.pcapng.log
+0ed4ba9fdb8cb940d09f92b7075db46f pcaps.full/general101c.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/general101d.pcapng.log
+0d9772fdd4477af9f2b0523e8ff44e63 pcaps.full/http-au101b.pcapng.log
+4df9f89d8991983f811a4cea8aaf2f12 pcaps.full/http-browse101.pcapng.log
+fdea8971f0510eb2d972385d69b05f2b pcaps.full/http-browse101b.pcapng.log
+9433ad4a3133a817f8e90f46a94705fd pcaps.full/http-browse101c.pcapng.log
+9433ad4a3133a817f8e90f46a94705fd pcaps.full/http-browse101d.pcapng.log
+ef0297df04fd11d88a39669ad3de8452 pcaps.full/http-chappellu101.pcapng.log
+ab0e19847b655e25897de3ad675bbf8f pcaps.full/http-chappellu101b.pcapng.log
+8cb81097cbef7c810b7dcd57b8a240cf pcaps.full/http-cheez101.pcapng.log
+542d7c56c0f2cf3c04e72152760392e1 pcaps.full/http-college101.pcapng.log
+ba1d7834a8352ea90b998873a725bd7f pcaps.full/http-disney101.pcapng.log
+8573185b6cac32ae8468471e51c1a0f6 pcaps.full/http-download-a.pcapng.log
+20b8dc7a31500946202d4d191b7edf16 pcaps.full/http-download101.pcapng.log
+01a2be97c1ab675184e97a71b084e25c pcaps.full/http-download101c.pcapng.log
+be0103fda2b6751d6abe396c426a1e52 pcaps.full/http-download101d.pcapng.log
+191508af7467c488cd5e6d61535e6c5b pcaps.full/http-errors101.pcapng.log
+1109ac31b2b0577889340cd98b20eb34 pcaps.full/http-espn101.pcapng.log
+f2543fa2a0bd81ffc279198b7d7a5f62 pcaps.full/http-google101.pcapng.log
+72cab4dcfa839a8275412b36855caeb1 pcaps.full/http-jezebel101.pcapng.log
+77f6419cd8de445503e779b9818a3aa5 pcaps.full/http-misctraffic101.pcapng.log
+28f32e99994ea9cc26df9d2ba3dd12ae pcaps.full/http-nonstandard101.pcapng.log
+f0356a106329dd181bd7fad31d54d979 pcaps.full/http-openoffice101a.pcapng.log
+7dc37f1efab91a9add8c55adb0009bae pcaps.full/http-openoffice101b.pcapng.log
+dfa7cf0ac25800ca7affd3ded10addfd pcaps.full/http-pcaprnet101.pcapng.log
+c9ce2f1e0180a51560f610323f9fa8e7 pcaps.full/http-pictures101.pcapng.log
+6bd2962179641be17333fb5b7f9cb1c6 pcaps.full/http-sfgate101.pcapng.log
+585cc0f172039cc04acb7b5da58f20ed pcaps.full/http-slow101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/http-winpcap101.cap.log
+4236a958c61e54031d40fc6d8c958291 pcaps.full/http-wiresharkdownload101.pcapng.log
+1b1359e2d6822016dc1a168c41667de4 pcaps.full/mybackground101.pcapng.log
+e95ab82bbf83b0d0af0b9f74fdad88f8 pcaps.full/net-lost-route.pcapng.log
+d293fcc821f7d8b19e1217f55bbb179b pcaps.full/sec-concern101.pcapng.log
+f47bc44475d79ccb61b1ff5b8562018d pcaps.full/sec-nessus101.pcapng.log
+85ec363f6ac000d7132b970ec39d78cc pcaps.full/sec-suspicious101.pcapng.log
+b78b407b7f11fd2e9a980c71bce44b29 pcaps.full/smb-join101.pcapng.log
+0807f74c4cd5ddb7fbd6aab6ff4c8270 pcaps.full/split250_00000_20160704110754.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/split250_00001_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/split250_00002_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/split250_00003_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/split250_00004_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/split250_00005_20160704110804.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/tcp-decodeas.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/tr-twohosts.pcapng.log
+5ee264aa76210466a389ba1e9fd5f3aa pcaps.full/tr-winsize.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.full/wlan-ipadstartstop101.pcapng.log

log-analysis/hashes/pcaps.small.md5sum

@@ -0,0 +1,69 @@
+4e489b35942d0c45d8b16d1cb1a06210 pcaps.small/4SICS-GeekLounge-151020.pcap.log
+890b35d7af03d13aab446e306ba60782 pcaps.small/4SICS-GeekLounge-151021.pcap.log
+1ebd451920d098aa4b7002054900a0b8 pcaps.small/4SICS-GeekLounge-151022.pcap.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/README.log
+f5b48dd8c8037efc43951efc3e1e16af pcaps.small/challenge101-0.pcapng.log
+6b2c4e89211bc977ac538ad218845794 pcaps.small/challenge101-1.pcapng.log
+765d385418861c272c081f85abee4448 pcaps.small/challenge101-3.pcapng.log
+6c5777e9a6a4601487f5bb0d637eaef0 pcaps.small/challenge101-4.pcapng.log
+2b632ea272f075b62a1779e6b79c262b pcaps.small/challenge101-5.pcapng.log
+431993ece40a4f8081cc949661be3b98 pcaps.small/challenge101-6.pcapng.log
+fe1fcee267a72f20de6abd423344b247 pcaps.small/challenge101-7.pcapng.log
+63382ae6dddcc347be2f140853ab57a3 pcaps.small/challenge101-8.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/dhcp-serverdiscovery101.pcapng.log
+9267a406fc3cca0cb4d7b755169df6be pcaps.small/dns-nmap101.pcapng.log
+42164f80e7ff0177cf36922c2caf7917 pcaps.small/f5-honeypot-release.pcap.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/ftp-bounce.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/ftp-clientside101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/ftp-crack101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/ftp-download101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/ftp-passwords101.pcapng.log
+ecb77db5b2688d25093117caaa9cf4f6 pcaps.small/gen-startupchatty101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/general101.pcapng.log
+3684f199efa845a7ec8f1053a9944caf pcaps.small/general101b.pcapng.log
+0ed4ba9fdb8cb940d09f92b7075db46f pcaps.small/general101c.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/general101d.pcapng.log
+0d9772fdd4477af9f2b0523e8ff44e63 pcaps.small/http-au101b.pcapng.log
+4df9f89d8991983f811a4cea8aaf2f12 pcaps.small/http-browse101.pcapng.log
+fdea8971f0510eb2d972385d69b05f2b pcaps.small/http-browse101b.pcapng.log
+9433ad4a3133a817f8e90f46a94705fd pcaps.small/http-browse101c.pcapng.log
+9433ad4a3133a817f8e90f46a94705fd pcaps.small/http-browse101d.pcapng.log
+ef0297df04fd11d88a39669ad3de8452 pcaps.small/http-chappellu101.pcapng.log
+ab0e19847b655e25897de3ad675bbf8f pcaps.small/http-chappellu101b.pcapng.log
+8cb81097cbef7c810b7dcd57b8a240cf pcaps.small/http-cheez101.pcapng.log
+542d7c56c0f2cf3c04e72152760392e1 pcaps.small/http-college101.pcapng.log
+ba1d7834a8352ea90b998873a725bd7f pcaps.small/http-disney101.pcapng.log
+8573185b6cac32ae8468471e51c1a0f6 pcaps.small/http-download-a.pcapng.log
+20b8dc7a31500946202d4d191b7edf16 pcaps.small/http-download101.pcapng.log
+01a2be97c1ab675184e97a71b084e25c pcaps.small/http-download101c.pcapng.log
+be0103fda2b6751d6abe396c426a1e52 pcaps.small/http-download101d.pcapng.log
+191508af7467c488cd5e6d61535e6c5b pcaps.small/http-errors101.pcapng.log
+1109ac31b2b0577889340cd98b20eb34 pcaps.small/http-espn101.pcapng.log
+f2543fa2a0bd81ffc279198b7d7a5f62 pcaps.small/http-google101.pcapng.log
+72cab4dcfa839a8275412b36855caeb1 pcaps.small/http-jezebel101.pcapng.log
+77f6419cd8de445503e779b9818a3aa5 pcaps.small/http-misctraffic101.pcapng.log
+28f32e99994ea9cc26df9d2ba3dd12ae pcaps.small/http-nonstandard101.pcapng.log
+f0356a106329dd181bd7fad31d54d979 pcaps.small/http-openoffice101a.pcapng.log
+7dc37f1efab91a9add8c55adb0009bae pcaps.small/http-openoffice101b.pcapng.log
+dfa7cf0ac25800ca7affd3ded10addfd pcaps.small/http-pcaprnet101.pcapng.log
+c9ce2f1e0180a51560f610323f9fa8e7 pcaps.small/http-pictures101.pcapng.log
+6bd2962179641be17333fb5b7f9cb1c6 pcaps.small/http-sfgate101.pcapng.log
+585cc0f172039cc04acb7b5da58f20ed pcaps.small/http-slow101.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/http-winpcap101.cap.log
+4236a958c61e54031d40fc6d8c958291 pcaps.small/http-wiresharkdownload101.pcapng.log
+1b1359e2d6822016dc1a168c41667de4 pcaps.small/mybackground101.pcapng.log
+e95ab82bbf83b0d0af0b9f74fdad88f8 pcaps.small/net-lost-route.pcapng.log
+d293fcc821f7d8b19e1217f55bbb179b pcaps.small/sec-concern101.pcapng.log
+f47bc44475d79ccb61b1ff5b8562018d pcaps.small/sec-nessus101.pcapng.log
+85ec363f6ac000d7132b970ec39d78cc pcaps.small/sec-suspicious101.pcapng.log
+b78b407b7f11fd2e9a980c71bce44b29 pcaps.small/smb-join101.pcapng.log
+0807f74c4cd5ddb7fbd6aab6ff4c8270 pcaps.small/split250_00000_20160704110754.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/split250_00001_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/split250_00002_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/split250_00003_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/split250_00004_20160704110759.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/split250_00005_20160704110804.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/tcp-decodeas.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/tr-twohosts.pcapng.log
+5ee264aa76210466a389ba1e9fd5f3aa pcaps.small/tr-winsize.pcapng.log
+d41d8cd98f00b204e9800998ecf8427e pcaps.small/wlan-ipadstartstop101.pcapng.log

log-analysis/input.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+# creates input/pcaps and input/nginx-logs
+
+REPO_TOP=$(git rev-parse --show-toplevel)
+eval_dir="${REPO_TOP}/log-analysis"
+input_dir="${eval_dir}/input"
+
+url=https://atlas-group.cs.brown.edu/data/pcaps.zip
+zip_dst="$input_dir/pcaps.zip"
+wget --no-check-certificate $url -O "$zip_dst"
+unzip "$zip_dst" -d "$input_dir"
+rm "$zip_dst"
+
+url=https://atlas-group.cs.brown.edu/data/nginx.zip
+zip_dst="$input_dir/nginx.zip"
+wget --no-check-certificate $url -O "$zip_dst"
+unzip "$zip_dst" -d "$input_dir"
+rm $zip_dst

log-analysis/input/.gitignore

@@ -0,0 +1 @@
+