From faae826f6f2d7dec5e0d96356211b3574f139114 Mon Sep 17 00:00:00 2001 From: Dmitriy Bannikov Date: Thu, 26 May 2016 12:54:47 -0700 Subject: [PATCH 1/5] Added support of field list, output format (text or json), custom delay between Graylog API calls. --- .gitignore | 1 + gtail/gtail.py | 162 ++++++++++++++++++++++++++++++------------------- 2 files changed, 101 insertions(+), 62 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..485dee6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/gtail/gtail.py b/gtail/gtail.py index b6951ce..063430a 100755 --- a/gtail/gtail.py +++ b/gtail/gtail.py @@ -12,6 +12,7 @@ import sys import time import urllib +from json import dumps MAX_DELAY = 10 DEFAULT_CONFIG_PATHS = [".gtail", os.path.expanduser("~/.gtail")] @@ -60,10 +61,13 @@ def list_streams(streams): def fetch_messages(server_config, query = None, stream_ids = None, - last_message_id = None): + last_message_id = None, + fields = None, + delay = MAX_DELAY): url = [] url.append(server_config.uri) - url.append("/search/universal/relative?range=7200&limit=100") + range = max(delay*5, 300) + url.append("/search/universal/relative?range={range}&limit=1000".format(range=range)) # query terms if query: @@ -71,6 +75,12 @@ def fetch_messages(server_config, else: url.append("&query=*") + # fields list + if fields: + if "_id" not in fields: + fields.append("_id") + url.append("&fields=" + "%2C".join(fields)) + # stream ID if stream_ids: quoted = map(urllib.quote_plus, stream_ids) @@ -109,37 +119,46 @@ def fetch_messages(server_config, # pretty prints a message # streams, if provided, is the full list of streams; it is used for pretty # printing of the stream name -def print_message(message, streams=None): - s = [] - if "timestamp" in message: - timestamp = message["timestamp"] - s.append(timestamp) - if streams and "streams" in message: - stream_ids = message["streams"] - stream_names = [] - for sid in stream_ids: - stream_names.append(streams[sid]["title"]) - s.append("[" + ", ".join(stream_names) + "]") - if "facility" in message: - facility = message["facility"] - s.append(facility) - if "level" in message: - level = message["level"] - s.append(level) - if "source" in message: - source = message["source"] - s.append(source) - if "loggerName" in message: - logger_name = message["loggerName"] - s.append(logger_name) - - if "full_message" in message: - text = message["full_message"] +def print_message(message, streams=None, fields=None, format="json"): + s = dict() + text = None + if fields: + count = 0 + for field in fields: + if field != "_id" and field in message: + count += 1 + s[field] = message[field] + else: + if "timestamp" in message: + s["timestamp"] = message["timestamp"] + if streams and "streams" in message: + stream_ids = message["streams"] + stream_names = [] + for sid in stream_ids: + stream_names.append(streams[sid]["title"]) + s["streams"] = "[" + ", ".join(stream_names) + "]" + if "facility" in message: + s["facility"] = message["facility"] + if "level" in message: + s["level"] = message["level"] + if "source" in message: + s["source"] = message["source"] + if "loggerName" in message: + s["loggerName"] = message["loggerName"] + + if "full_message" in message: + text = message["full_message"] + elif "message" in message: + text = message["message"] + + if format == "text": + out = map(str, s.values()) else: - text = message["message"] + out = dumps(s) + print bold(out) - print bold(" ".join(map(str, s))) - print text + if text: + print text # config object and config parsing Config = namedtuple("Config", "server_config") @@ -234,6 +253,15 @@ def main(): parser.add_argument("--query", dest="query", nargs="+", help="Query terms to search on") + parser.add_argument("--fields", dest="fields", + nargs="+", + help="Fields to display") + parser.add_argument("--format", dest="format", + choices=["text", "json"], default="json", + help="Display format") + parser.add_argument("--delay", dest="delay", + type=int, default=MAX_DELAY, + help="Delay between Rest API calls (seconds)") parser.add_argument("--config", dest="config_paths", nargs="+", help="Config files. Default: " + ", ".join(DEFAULT_CONFIG_PATHS)) @@ -280,37 +308,47 @@ def main(): # print log messages # - last_message_id = None - while True: - # time-forward messages - query = None - if args.query: - query = ' '.join(args.query) - try: - messages = fetch_messages( - server_config = server_config, - query = query, - stream_ids = stream_ids, - last_message_id = last_message_id) - except Exception as e: - print e - time.sleep(MAX_DELAY) - continue - - # print new messages - last_timestamp = None - for m in messages: - print_message(m, streams) - last_message_id = m["_id"] - last_timestamp = m["timestamp"] - - if last_timestamp: - seconds_since_last_message = max(0, (datetime.datetime.utcnow() - last_timestamp).total_seconds()) - delay = min(seconds_since_last_message, MAX_DELAY) - if delay > 2: - time.sleep(delay) - else: - time.sleep(MAX_DELAY) + try: + last_message_id = None + while True: + # time-forward messages + query = None + fields = None + if args.query: + query = ' '.join(args.query) + if args.fields: + fields = [] + for field in args.fields: + fields.extend(field.split(",")) + try: + messages = fetch_messages( + server_config = server_config, + query = query, + stream_ids = stream_ids, + last_message_id = last_message_id, + fields=fields, + delay=args.delay) + except Exception as e: + print e + time.sleep(args.delay) + continue + + # print new messages + last_timestamp = None + for m in messages: + print_message(m, streams, fields=fields, format=args.format) + last_message_id = m["_id"] + last_timestamp = m["timestamp"] + + if last_timestamp: + seconds_since_last_message = max(0, (datetime.datetime.utcnow() - last_timestamp).total_seconds()) + delay = min(seconds_since_last_message, args.delay) + if delay > 2: + time.sleep(delay) + else: + time.sleep(args.delay) + except KeyboardInterrupt: + os._exit(0) if __name__ == "__main__": rc = main() From 88ec3a861acc25f4274f876488871e1b0f02030f Mon Sep 17 00:00:00 2001 From: Dmitriy Bannikov Date: Thu, 26 May 2016 12:58:37 -0700 Subject: [PATCH 2/5] Updated documentation --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 6845e1a..a54c166 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,18 @@ This uses Graylog's search mechanism, so you can use `field: value` syntax. `gtail --stream nile --query crocodile` +### Select fields + +`gtail --fields comma-separated-fields-here` + +This will display only selected fields. + +### Display format + +`gtail --format [text or json]` + +Choose between comma separated values map and JSON output. + ### Full usage instructions `gtail --help` From 8997bd34901ed949b33606c0309627e0e4fb1931 Mon Sep 17 00:00:00 2001 From: Dmitriy Bannikov Date: Thu, 26 May 2016 14:06:49 -0700 Subject: [PATCH 3/5] Added support of time range --- README.md | 6 ++++++ gtail/gtail.py | 41 +++++++++++++++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a54c166..1962cfd 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,12 @@ This will display only selected fields. Choose between comma separated values map and JSON output. +### Initial range + +`gtail --range [interval]` + +Time range for initial fetch. Use "h" for hours, "m" for minutes, "s" or no identifier for seconds. + ### Full usage instructions `gtail --help` diff --git a/gtail/gtail.py b/gtail/gtail.py index 063430a..9c28bb5 100755 --- a/gtail/gtail.py +++ b/gtail/gtail.py @@ -15,8 +15,27 @@ from json import dumps MAX_DELAY = 10 +DEFAULT_RANGE="5m" DEFAULT_CONFIG_PATHS = [".gtail", os.path.expanduser("~/.gtail")] +# converts human readable time interval into seconds +def convert_time_interval(value): + value = value.lower() + value_int = 0 + if "h" in value: + value_parts = value.split("h", 1) + value_int+=int(value_parts[0])*3600 + value=value_parts[1] + if "m" in value: + value_parts = value.split("m", 1) + value_int += int(value_parts[0]) * 60 + value = value_parts[1] + value_parts = value.split("s", 1) + if value_parts[0] == "": + value_parts[0] = "0" + value_int += int(value_parts[0]) + return value_int + # returns a bold version of text using ansi characters def bold(text): make_bold = "\033[1m" @@ -63,11 +82,17 @@ def fetch_messages(server_config, stream_ids = None, last_message_id = None, fields = None, - delay = MAX_DELAY): + delay = MAX_DELAY, + initial_range = None): url = [] url.append(server_config.uri) - range = max(delay*5, 300) - url.append("/search/universal/relative?range={range}&limit=1000".format(range=range)) + if last_message_id: + limit = "&limit={0}".format(1000) + range = max(delay * 5, 300) + else: + range=initial_range + limit="" + url.append("/search/universal/relative?range={range}{limit}".format(range=range, limit=limit)) # query terms if query: @@ -127,10 +152,10 @@ def print_message(message, streams=None, fields=None, format="json"): for field in fields: if field != "_id" and field in message: count += 1 - s[field] = message[field] + s[field] = str(message[field]) else: if "timestamp" in message: - s["timestamp"] = message["timestamp"] + s["timestamp"] = str(message["timestamp"]) if streams and "streams" in message: stream_ids = message["streams"] stream_names = [] @@ -262,6 +287,9 @@ def main(): parser.add_argument("--delay", dest="delay", type=int, default=MAX_DELAY, help="Delay between Rest API calls (seconds)") + parser.add_argument("--range", dest="range", + type=str, default=DEFAULT_RANGE, + help="Time range for initial fetch") parser.add_argument("--config", dest="config_paths", nargs="+", help="Config files. Default: " + ", ".join(DEFAULT_CONFIG_PATHS)) @@ -327,7 +355,8 @@ def main(): stream_ids = stream_ids, last_message_id = last_message_id, fields=fields, - delay=args.delay) + delay=args.delay, + initial_range=convert_time_interval(args.range)) except Exception as e: print e time.sleep(args.delay) From e9417572a2284c64015d5d51d45a8bcb7e12f2b8 Mon Sep 17 00:00:00 2001 From: Dmitriy Bannikov Date: Thu, 26 May 2016 14:19:21 -0700 Subject: [PATCH 4/5] Fixed source url --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e30d615..2cca413 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ description='A tool to tail Graylog logs.', long_description=readme, author='Brandon Vargo', - url='https://github.com/bvargo/gtail', + url='https://github.com/demonus/gtail.git', packages=find_packages(), include_package_data=True, install_requires=install_requires, From 9fbd1a54c1cc323f8edf3f214dae6dc74d6148cf Mon Sep 17 00:00:00 2001 From: Dmitriy Bannikov Date: Thu, 26 May 2016 14:29:13 -0700 Subject: [PATCH 5/5] restored source url --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2cca413..e30d615 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ description='A tool to tail Graylog logs.', long_description=readme, author='Brandon Vargo', - url='https://github.com/demonus/gtail.git', + url='https://github.com/bvargo/gtail', packages=find_packages(), include_package_data=True, install_requires=install_requires,