Replace Puppy with standard HTTP client

.github/workflows/integration-test.yml

@@ -22,7 +22,7 @@ jobs:
         run: |
           cd takajo
           nimble update
-          nimble build -d:release --threads:on
+          nimble build -d:release -d:release --threads:on
           cd ../
 
       - name: clone hayabusa

.github/workflows/release.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-latest, ubuntu-latest, macos-latest, macos-12]
+        os: [windows-latest, ubuntu-latest, ubuntu-20.04, macos-latest, macos-12]
 
     steps:
       - name: Checkout Repository
@@ -26,14 +26,19 @@ jobs:
           nimble update
 
       - name: Build Takajo binary
-        if: matrix.os != 'macos-12'
+        if: matrix.os == 'windows-latest' || matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest'
         run: |
-          nimble build -d:release --threads:on
+          nimble build -d:release -d:ssl --threads:on
 
       - name: Build Takajo binary for Intel Mac
         if: matrix.os == 'macos-12'
         run: |
-          nimble build -d:release --threads:on --os:macosx --cpu:amd64 
+          nimble build -d:release -d:ssl --threads:on --os:macosx --cpu:amd64 
+
+      - name: Build Takajo binary for musl
+        if: matrix.os == 'ubuntu-20.04'
+        run: |
+          sudo apt install musl-tools && sudo apt install musl-dev && nimble build -d:release -d:ssl --threads:on --gcc.exe:"musl-gcc" --gcc.linkerexe:"musl-gcc" --passL="-static"
 
       - name: Package and Zip - Windows
         if: matrix.os == 'windows-latest'
@@ -52,6 +57,7 @@ jobs:
           cp mitre-attack.json release-binaries/
           case ${{ matrix.os }} in
             'ubuntu-latest') zip -j release-binaries/takajo-${{ github.event.inputs.release_ver }}-linux.zip release-binaries/* ;;
+            'ubuntu-20.04') zip -j release-binaries/takajo-${{ github.event.inputs.release_ver }}-linux-musl.zip release-binaries/* ;;
             'macos-latest') zip -j release-binaries/takajo-${{ github.event.inputs.release_ver }}-mac-arm.zip release-binaries/* ;;
             'macos-12') zip -j release-binaries/takajo-${{ github.event.inputs.release_ver }}-mac-intel.zip release-binaries/* ;;
           esac

README-Japanese.md

@@ -161,7 +161,7 @@ Nimがインストールされている場合、以下のコマンドでソー
 
 ```
 > nimble update
-> nimble build -d:release --threads:on
+> nimble build -d:release -d:ssl --threads:on
 ```
 
 # コマンド一覧

README.md

@@ -160,7 +160,7 @@ If you have Nim installed, you can compile from source with the following comman
 
 ```
 > nimble update
-> nimble build -d:release --threads:on
+> nimble build -d:release -d:ssl --threads:on
 ```
 
 # Command List

src/takajo.nim

@@ -1,8 +1,9 @@
 import algorithm
+import bearssl
 import cligen
+import httpclient
 import json
 import nancy
-import puppy
 import re
 import sets
 import sequtils

src/takajopkg/vtDomainLookup.nim

@@ -5,15 +5,15 @@
 
 var vtAPIDomainChannel: Channel[VirusTotalResult] # channel for receiving parallel query results
 
-proc queryDomainAPI(domain:string, headers: httpheaders.HttpHeaders) {.thread.} =
-    let response = get("https://www.virustotal.com/api/v3/domains/" & encodeUrl(domain), headers)
+proc queryDomainAPI(client: HttpClient, domain:string) {.thread.} =
+    let response = client.get("https://www.virustotal.com/api/v3/domains/" & encodeUrl(domain))
     var jsonResponse = %* {}
     var singleResultTable = newTable[string, string]()
     var malicious = false
     singleResultTable["Domain"] = domain
     singleResultTable["Link"] = "https://www.virustotal.com/gui/domain/" & domain
-    singleResultTable["Response"] = intToStr(response.code)
-    if response.code == 200:
+    singleResultTable["Response"] = response.status
+    if response.status == "200":
         jsonResponse = parseJson(response.body)
         # Parse values that need epoch time to human readable time
         singleResultTable["CreationDate"] = getJsonDate(jsonResponse, @["data", "attributes", "creation_date"])
@@ -78,17 +78,18 @@ proc vtDomainLookup(apiKey: string, domainList: string, jsonOutput: string = "",
         bar: SuruBar = initSuruBar()
         seqOfResultsTables: seq[TableRef[string, string]]
         jsonResponses: seq[JsonNode]  # Declare sequence to store Json responses
-        headers: httpheaders.HttpHeaders
+        headers = newHttpHeaders({"x-apikey": apiKey})
+        client = newHttpClient()
 
-    headers["x-apikey"] = apiKey
     bar[0].total = len(lines)
     bar.setup()
+    client.headers = headers
     vtAPIDomainChannel.open()
 
     for domain in lines:
         inc bar
         bar.update(1000000000) # refresh every second
-        spawn queryDomainAPI(domain, headers) # run queries in parallel
+        spawn queryDomainAPI(client, domain) # run queries in parallel
 
         # Sleep to respect the rate limit.
         sleep(int(timePerRequest * 1000)) # Convert to milliseconds.

src/takajopkg/vtHashLookup.nim

@@ -4,15 +4,15 @@
 
 var vtAPIHashChannel: Channel[VirusTotalResult] # channel for receiving parallel query results
 
-proc queryHashAPI(hash:string, headers: httpheaders.HttpHeaders) {.thread.} =
-    let response = get("https://www.virustotal.com/api/v3/files/" & hash, headers)
+proc queryHashAPI(client: HttpClient, hash:string) {.thread.} =
+    let response = client.get("https://www.virustotal.com/api/v3/files/" & hash)
     var jsonResponse = %* {}
     var singleResultTable = newTable[string, string]()
     var malicious = false
     singleResultTable["Hash"] = hash
     singleResultTable["Link"] = "https://www.virustotal.com/gui/file/" & hash
-    singleResultTable["Response"] = intToStr(response.code)
-    if response.code == 200:
+    singleResultTable["Response"] = response.status
+    if response.status == "200":
         jsonResponse = parseJson(response.body)
         # Parse values that need epoch time to human readable time
         singleResultTable["CreationDate"] = getJsonDate(jsonResponse, @["data", "attributes", "creation_date"])
@@ -67,17 +67,18 @@ proc vtHashLookup(apiKey: string, hashList: string, jsonOutput: string = "", out
         bar: SuruBar = initSuruBar()
         seqOfResultsTables: seq[TableRef[string, string]]
         jsonResponses: seq[JsonNode]  # Declare sequence to store Json responses
-        headers: httpheaders.HttpHeaders
+        headers = newHttpHeaders({"x-apikey": apiKey})
+        client = newHttpClient()
 
-    headers["x-apikey"] = apiKey
     bar[0].total = len(lines)
     bar.setup()
+    client.headers = headers
     vtAPIHashChannel.open()
 
     for hash in lines:
         inc bar
         bar.update(1000000000) # refresh every second
-        spawn queryHashAPI(hash, headers) # run queries in parallel
+        spawn queryHashAPI(client, hash) # run queries in parallel
 
         # Sleep to respect the rate limit.
         sleep(int(timePerRequest * 1000)) # Convert to milliseconds.

src/takajopkg/vtIpLookup.nim

@@ -2,15 +2,15 @@
 
 var vtIpAddressChannel: Channel[VirusTotalResult] # channel for receiving parallel query results
 
-proc queryIpAPI(ipAddress:string, headers: httpheaders.HttpHeaders) {.thread.} =
-    let response = get("https://www.virustotal.com/api/v3/ip_addresses/" & ipAddress, headers)
+proc queryIpAPI(client: HttpClient, ipAddress:string) {.thread.} =
+    let response = client.get("https://www.virustotal.com/api/v3/ip_addresses/" & ipAddress)
     var jsonResponse = %* {}
     var singleResultTable = newTable[string, string]()
     var malicious = false
     singleResultTable["IP-Address"] = ipAddress
     singleResultTable["Link"] = "https://www.virustotal.com/gui/ip_addresses/" & ipAddress
-    singleResultTable["Response"] = intToStr(response.code)
-    if response.code == 200:
+    singleResultTable["Response"] = response.status
+    if response.status == "200":
         jsonResponse = parseJson(response.body)
 
         # Parse values that need epoch time to human readable time
@@ -80,17 +80,17 @@ proc vtIpLookup(apiKey: string, ipList: string, jsonOutput: string = "", output:
         bar: SuruBar = initSuruBar()
         seqOfResultsTables: seq[TableRef[string, string]]
         jsonResponses: seq[JsonNode]  # Declare sequence to store Json responses
-        headers: httpheaders.HttpHeaders
+        headers = newHttpHeaders({"x-apikey": apiKey})
+        client = newHttpClient()
 
-    headers["x-apikey"] = apiKey
     bar[0].total = len(lines)
     bar.setup()
+    client.headers = headers
     vtIpAddressChannel.open()
-
     for ipAddress in lines:
         inc bar
         bar.update(1000000000) # refresh every second
-        spawn queryIpAPI(ipAddress, headers) # run queries in parallel
+        spawn queryIpAPI(client, ipAddress) # run queries in parallel
 
         # Sleep to respect the rate limit.
         sleep(int(timePerRequest * 1000)) # Convert to milliseconds.

takajo.nimble

@@ -14,7 +14,7 @@ bin           = @["takajo"]
 requires "nim >= 2.0.0"
 requires "cligen >= 1.5"
 requires "suru#f6f1e607c585b2bc2f71309996643f0555ff6349"
-requires "puppy >= 2.1.0"
 requires "termstyle"
 requires "nancy"
-requires "jsony >= 1.1.5"
+requires "jsony >= 1.1.5"
+requires "bearssl"