From 39c609cbfc0a7255702b0a8e52491cdc26b97806 Mon Sep 17 00:00:00 2001 From: Jussi-Pekka Erkkila Date: Mon, 2 Dec 2024 01:54:41 +0200 Subject: [PATCH 1/5] scan multiple targets at once --- secheaders/securityheaders.py | 66 +++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/secheaders/securityheaders.py b/secheaders/securityheaders.py index 27a050d..962fcb6 100644 --- a/secheaders/securityheaders.py +++ b/secheaders/securityheaders.py @@ -1,4 +1,5 @@ import argparse +import asyncio import http.client import json import re @@ -194,10 +195,32 @@ def get_full_url(self) -> str: return f"{self.protocol_scheme}://{self.hostname}{self.path}" +def scan_target(url, args): + try: + header_check = SecurityHeaders(url, args.max_redirects, args.insecure) + header_check.fetch_headers() + headers = header_check.check_headers() + except SecurityHeadersException as e: + print(e, file=sys.stderr) + sys.exit(1) + + if not headers: + print("Failed to fetch headers, exiting...", file=sys.stderr) + sys.exit(1) + + https = header_check.test_https() + if args.json: + return json.dumps({'target': header_check.get_full_url(), 'headers': headers, 'https': https}, indent=2) + + return cmd_utils.output_text(header_check.get_full_url(), headers, https, args) + + def main(): parser = argparse.ArgumentParser(description='Scan HTTP security headers', formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('url', metavar='URL', type=str, help='Target URL') + parser.add_argument('url', metavar='URL', nargs='?', default=None, type=str, help='Target URL') + parser.add_argument('--target-list', dest='target_list', metavar='FILE', default=None, type=str, + help='Input from list of target URLs') parser.add_argument('--max-redirects', dest='max_redirects', metavar='N', default=2, type=int, help='Max redirects, set 0 to disable') parser.add_argument('--insecure', dest='insecure', action='store_true', @@ -207,23 +230,36 @@ def main(): parser.add_argument('--verbose', '-v', dest='verbose', action='store_true', help='Verbose output') args = parser.parse_args() - try: - header_check = SecurityHeaders(args.url, args.max_redirects, args.insecure) - header_check.fetch_headers() - headers = header_check.check_headers() - except SecurityHeadersException as e: - print(e, file=sys.stderr) - sys.exit(1) - if not headers: - print("Failed to fetch headers, exiting...", file=sys.stderr) + if not args.url and not args.target_list: + print("No target url provided.", file=sys.stderr) + parser.print_usage(sys.stderr) sys.exit(1) - https = header_check.test_https() - if args.json: - print(json.dumps({'target': header_check.get_full_url(), 'headers': headers, 'https': https}, indent=2)) - else: - print(cmd_utils.output_text(header_check.get_full_url(), headers, https, args)) + if args.url: + print(scan_target(args.url, args)) + elif args.target_list: + asyncio.run(scan_multiple_targets(args)) + + +def done(f): + print(f.result()) + print("========================\n") + + +async def scan_multiple_targets(args): + with open(args.target_list, encoding='utf-8') as file: + targets = [line.rstrip() for line in file] + + loop = asyncio.get_event_loop() + tasks = [] + for t in targets: + task = loop.run_in_executor(None, scan_target, t, args) + task.add_done_callback(done) + tasks.append(task) + + for task in tasks: + await task if __name__ == "__main__": From 83048dd3c51ced9fc36ac5f8c67e6d7686124794 Mon Sep 17 00:00:00 2001 From: Jussi-Pekka Erkkila Date: Sat, 14 Dec 2024 02:12:19 +0200 Subject: [PATCH 2/5] add main to a dedicated file --- secheaders/__main__.py | 2 +- secheaders/cmd_utils.py | 8 +-- secheaders/exceptions.py | 4 ++ secheaders/secheaders.py | 102 ++++++++++++++++++++++++++++++++++ secheaders/securityheaders.py | 77 +------------------------ 5 files changed, 112 insertions(+), 81 deletions(-) create mode 100644 secheaders/secheaders.py diff --git a/secheaders/__main__.py b/secheaders/__main__.py index e06b46b..cdbb298 100644 --- a/secheaders/__main__.py +++ b/secheaders/__main__.py @@ -1,3 +1,3 @@ if __name__ == "__main__": - from .securityheaders import main + from .secheaders import main main() diff --git a/secheaders/cmd_utils.py b/secheaders/cmd_utils.py index a58cfa5..6ae7ec6 100644 --- a/secheaders/cmd_utils.py +++ b/secheaders/cmd_utils.py @@ -20,12 +20,12 @@ def get_eval_output(warn, no_color): return f"[ {color_start}{eval_result}{color_end} ]" -def output_text(target_url, headers, https, args) -> str: +def output_text(target_url, headers, https, no_color=False, verbose=False) -> str: terminal_width = shutil.get_terminal_size().columns - output_str = f"Scan target: {target_url}\n" + output_str = f"Scanning target {target_url} ...\n\n" # If the stdout is not going into terminal, disable colors - no_color = args.no_color or not sys.stdout.isatty() + no_color = no_color or not sys.stdout.isatty() for header, value in headers.items(): truncated = False if not value['defined']: @@ -44,7 +44,7 @@ def output_text(target_url, headers, https, args) -> str: # This is a dirty hack required to align ANSI-colored str correctly output_str += f"{output:<{terminal_width - COLUMN_WIDTH_R}}{eval_value:^{COLUMN_WIDTH_R + 9}}\n" - if truncated and args.verbose: + if truncated and verbose: output_str += f"Full header contents: {value['contents']}\n" for note in value['notes']: output_str += textwrap.fill(f" * {note}", terminal_width - COLUMN_WIDTH_R, subsequent_indent=' ') diff --git a/secheaders/exceptions.py b/secheaders/exceptions.py index 68196dc..36d3d71 100644 --- a/secheaders/exceptions.py +++ b/secheaders/exceptions.py @@ -8,3 +8,7 @@ class InvalidTargetURL(SecurityHeadersException): class UnableToConnect(SecurityHeadersException): pass + + +class FailedToFetchHeaders(SecurityHeadersException): + pass diff --git a/secheaders/secheaders.py b/secheaders/secheaders.py new file mode 100644 index 0000000..d836c20 --- /dev/null +++ b/secheaders/secheaders.py @@ -0,0 +1,102 @@ +import argparse +import asyncio +import json +import sys + + +from . import cmd_utils +from .exceptions import SecurityHeadersException, FailedToFetchHeaders +from .securityheaders import SecurityHeaders + + +def main(): + parser = argparse.ArgumentParser(description='Scan HTTP security headers', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('url', metavar='URL', nargs='?', default=None, type=str, help='Target URL') + parser.add_argument('--target-list', dest='target_list', metavar='FILE', default=None, type=str, + help='Input from list of target URLs') + parser.add_argument('--max-redirects', dest='max_redirects', metavar='N', default=2, type=int, + help='Max redirects, set 0 to disable') + parser.add_argument('--insecure', dest='insecure', action='store_true', + help='Do not verify TLS certificate chain') + parser.add_argument('--json', dest='json', action='store_true', help='JSON output instead of text') + parser.add_argument('--no-color', dest='no_color', action='store_true', help='Do not output colors in terminal') + parser.add_argument('--verbose', '-v', dest='verbose', action='store_true', + help='Verbose output') + args = parser.parse_args() + + if not args.url and not args.target_list: + print("No target url provided.", file=sys.stderr) + parser.print_usage(sys.stderr) + sys.exit(1) + + target_list = [] + if args.target_list: + with open(args.target_list, encoding='utf-8') as file: + target_list = [line.rstrip() for line in file] + if args.url: + target_list.append(args.url) + + target_list = list(set(target_list)) # Remove possible duplicates + + if args.url: + + try: + res = scan_target(args.url, args) + except SecurityHeadersException as e: + print(e, file=sys.stderr) + sys.exit(1) + if args.json: + print(json.dumps(res)) + else: + print(cmd_utils.output_text(res['target'], res['headers'], res['https'], args.no_color, args.verbose)) + elif args.target_list: + asyncio.run(scan_multiple_targets(args)) + + +scan_results = [] + + +def async_scan_done(scan): + try: + res = scan.result() + print(cmd_utils.output_text(res['target'], res['headers'], res['https'])) + print("========================\n") + except SecurityHeadersException as e: + print(e, file=sys.stderr) + + +def scan_target(url, args): + try: + header_check = SecurityHeaders(url, args.max_redirects, args.insecure) + header_check.fetch_headers() + headers = header_check.check_headers() + except SecurityHeadersException as e: + raise e + + if not headers: + raise FailedToFetchHeaders("Failed to fetch headers") + + https = header_check.test_https() + return {'target': header_check.get_full_url(), 'headers': headers, 'https': https} + + +async def scan_multiple_targets(args): + with open(args.target_list, encoding='utf-8') as file: + targets = [line.rstrip() for line in file] + + loop = asyncio.get_event_loop() + tasks = [] + for t in targets: + task = loop.run_in_executor(None, scan_target, t, args) + task.add_done_callback(async_scan_done) + tasks.append(task) + + for task in tasks: + await task + + print("ALL COMPLETED!") + + +if __name__ == "__main__": + main() diff --git a/secheaders/securityheaders.py b/secheaders/securityheaders.py index 962fcb6..34778f6 100644 --- a/secheaders/securityheaders.py +++ b/secheaders/securityheaders.py @@ -1,15 +1,11 @@ -import argparse -import asyncio import http.client -import json import re import socket import ssl -import sys from typing import Union from urllib.parse import ParseResult, urlparse -from . import utils, cmd_utils +from . import utils from .constants import DEFAULT_TIMEOUT, DEFAULT_URL_SCHEME, EVAL_WARN, REQUEST_HEADERS, HEADER_STRUCTURED_LIST, \ SERVER_VERSION_HEADERS from .exceptions import SecurityHeadersException, InvalidTargetURL, UnableToConnect @@ -193,74 +189,3 @@ def check_headers(self) -> dict: def get_full_url(self) -> str: return f"{self.protocol_scheme}://{self.hostname}{self.path}" - - -def scan_target(url, args): - try: - header_check = SecurityHeaders(url, args.max_redirects, args.insecure) - header_check.fetch_headers() - headers = header_check.check_headers() - except SecurityHeadersException as e: - print(e, file=sys.stderr) - sys.exit(1) - - if not headers: - print("Failed to fetch headers, exiting...", file=sys.stderr) - sys.exit(1) - - https = header_check.test_https() - if args.json: - return json.dumps({'target': header_check.get_full_url(), 'headers': headers, 'https': https}, indent=2) - - return cmd_utils.output_text(header_check.get_full_url(), headers, https, args) - - -def main(): - parser = argparse.ArgumentParser(description='Scan HTTP security headers', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('url', metavar='URL', nargs='?', default=None, type=str, help='Target URL') - parser.add_argument('--target-list', dest='target_list', metavar='FILE', default=None, type=str, - help='Input from list of target URLs') - parser.add_argument('--max-redirects', dest='max_redirects', metavar='N', default=2, type=int, - help='Max redirects, set 0 to disable') - parser.add_argument('--insecure', dest='insecure', action='store_true', - help='Do not verify TLS certificate chain') - parser.add_argument('--json', dest='json', action='store_true', help='JSON output instead of text') - parser.add_argument('--no-color', dest='no_color', action='store_true', help='Do not output colors in terminal') - parser.add_argument('--verbose', '-v', dest='verbose', action='store_true', - help='Verbose output') - args = parser.parse_args() - - if not args.url and not args.target_list: - print("No target url provided.", file=sys.stderr) - parser.print_usage(sys.stderr) - sys.exit(1) - - if args.url: - print(scan_target(args.url, args)) - elif args.target_list: - asyncio.run(scan_multiple_targets(args)) - - -def done(f): - print(f.result()) - print("========================\n") - - -async def scan_multiple_targets(args): - with open(args.target_list, encoding='utf-8') as file: - targets = [line.rstrip() for line in file] - - loop = asyncio.get_event_loop() - tasks = [] - for t in targets: - task = loop.run_in_executor(None, scan_target, t, args) - task.add_done_callback(done) - tasks.append(task) - - for task in tasks: - await task - - -if __name__ == "__main__": - main() From b96ea0f00784c1d65b2bab1173967ffae47a5fb4 Mon Sep 17 00:00:00 2001 From: Jussi-Pekka Erkkila Date: Sun, 15 Dec 2024 12:11:16 +0200 Subject: [PATCH 3/5] cmd output tuning for multi target scan --- secheaders/cmd_utils.py | 3 ++- secheaders/secheaders.py | 28 ++++++++++++++++++---------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/secheaders/cmd_utils.py b/secheaders/cmd_utils.py index 6ae7ec6..81276e0 100644 --- a/secheaders/cmd_utils.py +++ b/secheaders/cmd_utils.py @@ -22,7 +22,7 @@ def get_eval_output(warn, no_color): def output_text(target_url, headers, https, no_color=False, verbose=False) -> str: terminal_width = shutil.get_terminal_size().columns - output_str = f"Scanning target {target_url} ...\n\n" + output_str = f"Scanning target {target_url} ...\n" # If the stdout is not going into terminal, disable colors no_color = no_color or not sys.stdout.isatty() @@ -66,4 +66,5 @@ def output_text(target_url, headers, https, no_color=False, verbose=False) -> st output_str += output + output_str += '\n' return output_str diff --git a/secheaders/secheaders.py b/secheaders/secheaders.py index d836c20..e908e31 100644 --- a/secheaders/secheaders.py +++ b/secheaders/secheaders.py @@ -54,14 +54,10 @@ def main(): asyncio.run(scan_multiple_targets(args)) -scan_results = [] - - def async_scan_done(scan): try: - res = scan.result() - print(cmd_utils.output_text(res['target'], res['headers'], res['https'])) - print("========================\n") + res, args = scan.result() + print(cmd_utils.output_text(res['target'], res['headers'], res['https'], args.no_color, args.verbose)) except SecurityHeadersException as e: print(e, file=sys.stderr) @@ -81,22 +77,34 @@ def scan_target(url, args): return {'target': header_check.get_full_url(), 'headers': headers, 'https': https} +def scan_target_wrapper(url, args): + # A bit of a dirty hack to pass args to the done callback + return scan_target(url, args), args + + async def scan_multiple_targets(args): with open(args.target_list, encoding='utf-8') as file: targets = [line.rstrip() for line in file] loop = asyncio.get_event_loop() tasks = [] - for t in targets: - task = loop.run_in_executor(None, scan_target, t, args) - task.add_done_callback(async_scan_done) + for target in targets: + if args.json: + task = loop.run_in_executor(None, scan_target, target, args) + else: + task = loop.run_in_executor(None, scan_target_wrapper, target, args) + task.add_done_callback(async_scan_done) tasks.append(task) + res = [] for task in tasks: await task - print("ALL COMPLETED!") + if args.json: + for t in tasks: + res.append(t.result()) + print(str(res)) if __name__ == "__main__": main() From 36c74ad12d6aafbf74875f9e675b0b407dbf58d8 Mon Sep 17 00:00:00 2001 From: Jussi-Pekka Erkkila Date: Thu, 26 Dec 2024 01:50:03 +0200 Subject: [PATCH 4/5] error handling for async scanning --- secheaders/secheaders.py | 49 +++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/secheaders/secheaders.py b/secheaders/secheaders.py index e908e31..de948d2 100644 --- a/secheaders/secheaders.py +++ b/secheaders/secheaders.py @@ -30,17 +30,7 @@ def main(): parser.print_usage(sys.stderr) sys.exit(1) - target_list = [] - if args.target_list: - with open(args.target_list, encoding='utf-8') as file: - target_list = [line.rstrip() for line in file] if args.url: - target_list.append(args.url) - - target_list = list(set(target_list)) # Remove possible duplicates - - if args.url: - try: res = scan_target(args.url, args) except SecurityHeadersException as e: @@ -55,20 +45,18 @@ def main(): def async_scan_done(scan): - try: - res, args = scan.result() + res, args = scan.result() + if 'error' in res: + print(f"Scanning target {res['target']}...") + print(f"Error: {res['error']}\n") + else: print(cmd_utils.output_text(res['target'], res['headers'], res['https'], args.no_color, args.verbose)) - except SecurityHeadersException as e: - print(e, file=sys.stderr) def scan_target(url, args): - try: - header_check = SecurityHeaders(url, args.max_redirects, args.insecure) - header_check.fetch_headers() - headers = header_check.check_headers() - except SecurityHeadersException as e: - raise e + header_check = SecurityHeaders(url, args.max_redirects, args.insecure) + header_check.fetch_headers() + headers = header_check.check_headers() if not headers: raise FailedToFetchHeaders("Failed to fetch headers") @@ -78,21 +66,24 @@ def scan_target(url, args): def scan_target_wrapper(url, args): - # A bit of a dirty hack to pass args to the done callback - return scan_target(url, args), args + try: + # Return the args also for the callback function + return scan_target(url, args), args + except SecurityHeadersException as e: + return {'target': url, 'error': str(e)}, args async def scan_multiple_targets(args): with open(args.target_list, encoding='utf-8') as file: targets = [line.rstrip() for line in file] + targets = list(set(targets)) # Remove possible duplicates loop = asyncio.get_event_loop() tasks = [] for target in targets: - if args.json: - task = loop.run_in_executor(None, scan_target, target, args) - else: - task = loop.run_in_executor(None, scan_target_wrapper, target, args) + task = loop.run_in_executor(None, scan_target_wrapper, target, args) + if not args.json: + # Output result of each scan immediately task.add_done_callback(async_scan_done) tasks.append(task) @@ -100,11 +91,13 @@ async def scan_multiple_targets(args): for task in tasks: await task + # When json output, aggregate the results and output the json dump at the end if args.json: for t in tasks: - res.append(t.result()) + result, _args = t.result() + res.append(result) - print(str(res)) + print(json.dumps(res, indent=2)) if __name__ == "__main__": main() From 7c31257f9b9520f9b351d375ae76c3109c73e2b4 Mon Sep 17 00:00:00 2001 From: Jussi-Pekka Erkkila Date: Thu, 26 Dec 2024 02:28:31 +0200 Subject: [PATCH 5/5] update README, fix newline issue in cmd output --- README.md | 28 ++++++++++++++-------------- pyproject.toml | 2 +- secheaders/cmd_utils.py | 5 ++--- secheaders/secheaders.py | 2 +- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 55881cc..8f1c669 100644 --- a/README.md +++ b/README.md @@ -34,29 +34,28 @@ $ pip install secheaders ## Usage ``` -$ secheaders --help -usage: secheaders [-h] [--max-redirects N] [--insecure] [--json] [--no-color] - [--verbose] - URL +usage: secheaders [-h] [--target-list FILE] [--max-redirects N] [--insecure] [--json] [--no-color] [--verbose] [URL] Scan HTTP security headers positional arguments: - URL Target URL + URL Target URL (default: None) options: - -h, --help show this help message and exit - --max-redirects N Max redirects, set 0 to disable (default: 2) - --insecure Do not verify TLS certificate chain (default: False) - --json JSON output instead of text (default: False) - --no-color Do not output colors in terminal (default: False) - --verbose, -v Verbose output (default: False) + -h, --help show this help message and exit + --target-list FILE Read multiple target URLs from a file and scan them all (default: None) + --max-redirects N Max redirects, set 0 to disable (default: 2) + --insecure Do not verify TLS certificate chain (default: False) + --json JSON output instead of text (default: False) + --no-color Do not output colors in terminal (default: False) + --verbose, -v Verbose output (default: False) ``` ## Example output ``` $ secheaders example.com +Scanning target https://example.com ... Header 'x-frame-options' is missing [ WARN ] Header 'strict-transport-security' is missing [ WARN ] Header 'content-security-policy' is missing [ WARN ] @@ -64,21 +63,22 @@ Header 'x-content-type-options' is missing [ WARN ] Header 'x-xss-protection' is missing [ OK ] Header 'referrer-policy' is missing [ WARN ] Header 'permissions-policy' is missing [ WARN ] -server: ECAcc (nyd/D147) [ WARN ] +server: ECAcc (nyd/D191) [ WARN ] HTTPS supported [ OK ] HTTPS valid certificate [ OK ] HTTP -> HTTPS automatic redirect [ WARN ] + ``` ## Design principles The following design principles have been considered: -* Simplicity of the codebase. +* Simplicity of the codebase. * The code should be easy to understand and follow without in-depth Python knowledge. * Avoidance of external dependencies. * The Python Standard Libary provides enough tools and libraries for quite many use cases. -* Unix philosophy in general +* Unix philosophy in general * *"Do one thing and do it well"* These are not rules set in stone, but should be revisited when doing big design choices. diff --git a/pyproject.toml b/pyproject.toml index bb6b3d2..7dccb9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,4 +30,4 @@ Repository = "https://github.com/juerkkil/secheaders" [project.scripts] -secheaders = "secheaders.securityheaders:main" +secheaders = "secheaders.secheaders:main" diff --git a/secheaders/cmd_utils.py b/secheaders/cmd_utils.py index 81276e0..fe41c0d 100644 --- a/secheaders/cmd_utils.py +++ b/secheaders/cmd_utils.py @@ -59,12 +59,11 @@ def output_text(target_url, headers, https, no_color=False, verbose=False) -> st output = f"{msg_map[key]}" eval_value = get_eval_output(not https[key], no_color) if no_color: - output = f"{output:<{terminal_width - COLUMN_WIDTH_R}}{eval_value:^{COLUMN_WIDTH_R}}" + output = f"{output:<{terminal_width - COLUMN_WIDTH_R}}{eval_value:^{COLUMN_WIDTH_R}}\n" else: # This is a dirty hack required to align ANSI-colored str correctly - output = f"{output:<{terminal_width - COLUMN_WIDTH_R}}{eval_value:^{COLUMN_WIDTH_R + 9}}" + output = f"{output:<{terminal_width - COLUMN_WIDTH_R}}{eval_value:^{COLUMN_WIDTH_R + 9}}\n" output_str += output - output_str += '\n' return output_str diff --git a/secheaders/secheaders.py b/secheaders/secheaders.py index de948d2..d30889a 100644 --- a/secheaders/secheaders.py +++ b/secheaders/secheaders.py @@ -14,7 +14,7 @@ def main(): formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('url', metavar='URL', nargs='?', default=None, type=str, help='Target URL') parser.add_argument('--target-list', dest='target_list', metavar='FILE', default=None, type=str, - help='Input from list of target URLs') + help='Read multiple target URLs from a file and scan them all') parser.add_argument('--max-redirects', dest='max_redirects', metavar='N', default=2, type=int, help='Max redirects, set 0 to disable') parser.add_argument('--insecure', dest='insecure', action='store_true',