Merge pull request #31 from juerkkil/support_for_multiple_urls

scan multiple targets at once

README.md

@@ -34,51 +34,51 @@ $ pip install secheaders
 
 ## Usage
 ```
-$ secheaders --help
-usage: secheaders [-h] [--max-redirects N] [--insecure] [--json] [--no-color]
-                  [--verbose]
-                  URL
+usage: secheaders [-h] [--target-list FILE] [--max-redirects N] [--insecure] [--json] [--no-color] [--verbose] [URL]
 
 Scan HTTP security headers
 
 positional arguments:
-  URL                Target URL
+  URL                 Target URL (default: None)
 
 options:
-  -h, --help         show this help message and exit
-  --max-redirects N  Max redirects, set 0 to disable (default: 2)
-  --insecure         Do not verify TLS certificate chain (default: False)
-  --json             JSON output instead of text (default: False)
-  --no-color         Do not output colors in terminal (default: False)
-  --verbose, -v      Verbose output (default: False)
+  -h, --help          show this help message and exit
+  --target-list FILE  Read multiple target URLs from a file and scan them all (default: None)
+  --max-redirects N   Max redirects, set 0 to disable (default: 2)
+  --insecure          Do not verify TLS certificate chain (default: False)
+  --json              JSON output instead of text (default: False)
+  --no-color          Do not output colors in terminal (default: False)
+  --verbose, -v       Verbose output (default: False)
 ```
 
 
 ## Example output
 ```
 $ secheaders example.com
+Scanning target https://example.com ...
 Header 'x-frame-options' is missing                                   [ WARN ]
 Header 'strict-transport-security' is missing                         [ WARN ]
 Header 'content-security-policy' is missing                           [ WARN ]
 Header 'x-content-type-options' is missing                            [ WARN ]
 Header 'x-xss-protection' is missing                                   [ OK ]
 Header 'referrer-policy' is missing                                   [ WARN ]
 Header 'permissions-policy' is missing                                [ WARN ]
-server: ECAcc (nyd/D147)                                              [ WARN ]
+server: ECAcc (nyd/D191)                                              [ WARN ]
 HTTPS supported                                                        [ OK ]
 HTTPS valid certificate                                                [ OK ]
 HTTP -> HTTPS automatic redirect                                      [ WARN ]
+
 ```
 
 ## Design principles
 
 The following design principles have been considered:
 
-* Simplicity of the codebase. 
+* Simplicity of the codebase.
 	* The code should be easy to understand and follow without in-depth Python knowledge.
 * Avoidance of external dependencies.
 	* The Python Standard Libary provides enough tools and libraries for quite many use cases.
-* Unix philosophy in general 
+* Unix philosophy in general
 	* *"Do one thing and do it well"*
 
 These are not rules set in stone, but should be revisited when doing big design choices.

pyproject.toml

@@ -30,4 +30,4 @@ Repository = "https://github.com/juerkkil/secheaders"
 
 
 [project.scripts]
-secheaders = "secheaders.securityheaders:main"
+secheaders = "secheaders.secheaders:main"

secheaders/__main__.py

@@ -1,3 +1,3 @@
 if __name__ == "__main__":
-    from .securityheaders import main
+    from .secheaders import main
     main()

secheaders/cmd_utils.py

@@ -20,12 +20,12 @@ def get_eval_output(warn, no_color):
     return f"[ {color_start}{eval_result}{color_end} ]"
 
 
-def output_text(target_url, headers, https, args) -> str:
+def output_text(target_url, headers, https, no_color=False, verbose=False) -> str:
     terminal_width = shutil.get_terminal_size().columns
-    output_str = f"Scan target: {target_url}\n"
+    output_str = f"Scanning target {target_url} ...\n"
 
     # If the stdout is not going into terminal, disable colors
-    no_color = args.no_color or not sys.stdout.isatty()
+    no_color = no_color or not sys.stdout.isatty()
     for header, value in headers.items():
         truncated = False
         if not value['defined']:
@@ -44,7 +44,7 @@ def output_text(target_url, headers, https, args) -> str:
             # This is a dirty hack required to align ANSI-colored str correctly
             output_str += f"{output:<{terminal_width - COLUMN_WIDTH_R}}{eval_value:^{COLUMN_WIDTH_R + 9}}\n"
 
-        if truncated and args.verbose:
+        if truncated and verbose:
             output_str += f"Full header contents: {value['contents']}\n"
         for note in value['notes']:
             output_str += textwrap.fill(f" * {note}", terminal_width - COLUMN_WIDTH_R, subsequent_indent='   ')
@@ -59,10 +59,10 @@ def output_text(target_url, headers, https, args) -> str:
         output = f"{msg_map[key]}"
         eval_value = get_eval_output(not https[key], no_color)
         if no_color:
-            output = f"{output:<{terminal_width - COLUMN_WIDTH_R}}{eval_value:^{COLUMN_WIDTH_R}}"
+            output = f"{output:<{terminal_width - COLUMN_WIDTH_R}}{eval_value:^{COLUMN_WIDTH_R}}\n"
         else:
             # This is a dirty hack required to align ANSI-colored str correctly
-            output = f"{output:<{terminal_width - COLUMN_WIDTH_R}}{eval_value:^{COLUMN_WIDTH_R + 9}}"
+            output = f"{output:<{terminal_width - COLUMN_WIDTH_R}}{eval_value:^{COLUMN_WIDTH_R + 9}}\n"
 
         output_str += output
 

secheaders/exceptions.py

@@ -8,3 +8,7 @@ class InvalidTargetURL(SecurityHeadersException):
 
 class UnableToConnect(SecurityHeadersException):
     pass
+
+
+class FailedToFetchHeaders(SecurityHeadersException):
+    pass

secheaders/secheaders.py

@@ -0,0 +1,103 @@
+import argparse
+import asyncio
+import json
+import sys
+
+
+from . import cmd_utils
+from .exceptions import SecurityHeadersException, FailedToFetchHeaders
+from .securityheaders import SecurityHeaders
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Scan HTTP security headers',
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('url', metavar='URL', nargs='?', default=None, type=str, help='Target URL')
+    parser.add_argument('--target-list', dest='target_list', metavar='FILE', default=None, type=str,
+                        help='Read multiple target URLs from a file and scan them all')
+    parser.add_argument('--max-redirects', dest='max_redirects', metavar='N', default=2, type=int,
+                        help='Max redirects, set 0 to disable')
+    parser.add_argument('--insecure', dest='insecure', action='store_true',
+                        help='Do not verify TLS certificate chain')
+    parser.add_argument('--json', dest='json', action='store_true', help='JSON output instead of text')
+    parser.add_argument('--no-color', dest='no_color', action='store_true', help='Do not output colors in terminal')
+    parser.add_argument('--verbose', '-v', dest='verbose', action='store_true',
+                        help='Verbose output')
+    args = parser.parse_args()
+
+    if not args.url and not args.target_list:
+        print("No target url provided.", file=sys.stderr)
+        parser.print_usage(sys.stderr)
+        sys.exit(1)
+
+    if args.url:
+        try:
+            res = scan_target(args.url, args)
+        except SecurityHeadersException as e:
+            print(e, file=sys.stderr)
+            sys.exit(1)
+        if args.json:
+            print(json.dumps(res))
+        else:
+            print(cmd_utils.output_text(res['target'], res['headers'], res['https'], args.no_color, args.verbose))
+    elif args.target_list:
+        asyncio.run(scan_multiple_targets(args))
+
+
+def async_scan_done(scan):
+    res, args = scan.result()
+    if 'error' in res:
+        print(f"Scanning target {res['target']}...")
+        print(f"Error: {res['error']}\n")
+    else:
+        print(cmd_utils.output_text(res['target'], res['headers'], res['https'], args.no_color, args.verbose))
+
+
+def scan_target(url, args):
+    header_check = SecurityHeaders(url, args.max_redirects, args.insecure)
+    header_check.fetch_headers()
+    headers = header_check.check_headers()
+
+    if not headers:
+        raise FailedToFetchHeaders("Failed to fetch headers")
+
+    https = header_check.test_https()
+    return {'target': header_check.get_full_url(), 'headers': headers, 'https': https}
+
+
+def scan_target_wrapper(url, args):
+    try:
+        # Return the args also for the callback function
+        return scan_target(url, args), args
+    except SecurityHeadersException as e:
+        return {'target': url, 'error': str(e)}, args
+
+
+async def scan_multiple_targets(args):
+    with open(args.target_list, encoding='utf-8') as file:
+        targets = [line.rstrip() for line in file]
+
+    targets = list(set(targets))  # Remove possible duplicates
+    loop = asyncio.get_event_loop()
+    tasks = []
+    for target in targets:
+        task = loop.run_in_executor(None, scan_target_wrapper, target, args)
+        if not args.json:
+            # Output result of each scan immediately
+            task.add_done_callback(async_scan_done)
+        tasks.append(task)
+
+    res = []
+    for task in tasks:
+        await task
+
+    # When json output, aggregate the results and output the json dump at the end
+    if args.json:
+        for t in tasks:
+            result, _args = t.result()
+            res.append(result)
+
+        print(json.dumps(res, indent=2))
+
+if __name__ == "__main__":
+    main()

secheaders/securityheaders.py

@@ -1,14 +1,11 @@
-import argparse
 import http.client
-import json
 import re
 import socket
 import ssl
-import sys
 from typing import Union
 from urllib.parse import ParseResult, urlparse
 
-from . import utils, cmd_utils
+from . import utils
 from .constants import DEFAULT_TIMEOUT, DEFAULT_URL_SCHEME, EVAL_WARN, REQUEST_HEADERS, HEADER_STRUCTURED_LIST, \
         SERVER_VERSION_HEADERS
 from .exceptions import SecurityHeadersException, InvalidTargetURL, UnableToConnect
@@ -192,39 +189,3 @@ def check_headers(self) -> dict:
 
     def get_full_url(self) -> str:
         return f"{self.protocol_scheme}://{self.hostname}{self.path}"
-
-
-def main():
-    parser = argparse.ArgumentParser(description='Scan HTTP security headers',
-                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument('url', metavar='URL', type=str, help='Target URL')
-    parser.add_argument('--max-redirects', dest='max_redirects', metavar='N', default=2, type=int,
-                        help='Max redirects, set 0 to disable')
-    parser.add_argument('--insecure', dest='insecure', action='store_true',
-                        help='Do not verify TLS certificate chain')
-    parser.add_argument('--json', dest='json', action='store_true', help='JSON output instead of text')
-    parser.add_argument('--no-color', dest='no_color', action='store_true', help='Do not output colors in terminal')
-    parser.add_argument('--verbose', '-v', dest='verbose', action='store_true',
-                        help='Verbose output')
-    args = parser.parse_args()
-    try:
-        header_check = SecurityHeaders(args.url, args.max_redirects, args.insecure)
-        header_check.fetch_headers()
-        headers = header_check.check_headers()
-    except SecurityHeadersException as e:
-        print(e, file=sys.stderr)
-        sys.exit(1)
-
-    if not headers:
-        print("Failed to fetch headers, exiting...", file=sys.stderr)
-        sys.exit(1)
-
-    https = header_check.test_https()
-    if args.json:
-        print(json.dumps({'target': header_check.get_full_url(), 'headers': headers, 'https': https}, indent=2))
-    else:
-        print(cmd_utils.output_text(header_check.get_full_url(), headers, https, args))
-
-
-if __name__ == "__main__":
-    main()