From c1be0b9cd2431dc1687c4e28156d00b949589fb6 Mon Sep 17 00:00:00 2001
From: zhangliyong <zhangliyong@umeng.com>
Date: Wed, 16 Apr 2014 17:19:34 +0800
Subject: [PATCH] Add an option to process fields after parse pharse

Can specify an expression to process a field, and store the processed result
---
 ngxtop/ngxtop.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py
index afb5bc3..bb24ddd 100755
--- a/ngxtop/ngxtop.py
+++ b/ngxtop/ngxtop.py
@@ -1,10 +1,10 @@
 """ngxtop - ad-hoc query for nginx access log.
 
 Usage:
-    ngxtop [options]
-    ngxtop [options] (print|top|avg|sum) <var> ...
+    ngxtop [options] [--field-expression <field:expression> ...]
+    ngxtop [options] [--field-expression <field:expression> ...] (print|top|avg|sum) <var> ...
     ngxtop info
-    ngxtop [options] query <query> ...
+    ngxtop [options] [--field-expression <field:expression> ...] query <query> ...
 
 Options:
     -l <file>, --access-log <file>  access log file to parse.
@@ -29,6 +29,7 @@
     -c <file>, --config <file>  allow ngxtop to parse nginx config file for log format and location.
     -i <filter-expression>, --filter <filter-expression>  filter in, records satisfied given expression are processed.
     -p <filter-expression>, --pre-filter <filter-expression> in-filter expression to check in pre-parsing phase.
+    --field-expression <field:expression>... process field in post-parsing phase
 
 Examples:
     All examples read nginx config file for access log location and format.
@@ -52,6 +53,9 @@
     Average body bytes sent of 200 responses of requested path begin with 'foo':
     $ ngxtop avg bytes_sent --filter 'status == 200 and request_path.startswith("foo")'
 
+    Process request_path, retrieve path from start to the second '/'
+    $ ngxtop --field-expression "request_path: "/".join(request_path.split('/',1)[:2])"
+
     Analyze apache access log from remote machine using 'common' log format
     $ ssh remote tail -f /var/log/apache2/access.log | ngxtop -f common
 """
@@ -264,6 +268,18 @@ def process_log(lines, pattern, processor, arguments):
     if filter_exp:
         records = (r for r in records if eval(filter_exp, {}, r))
 
+    field_exps = dict(field_exp.split(":", 1)
+                      for field_exp in arguments['--field-expression'])
+
+    def proc(record):
+        for field, exp in field_exps.iteritems():
+            try:
+                record[field] = eval(exp, {}, record)
+            except Exception:
+                logging.exception("Process %s failed" % field)
+        return record
+    records = (proc(record) for record in records)
+
     processor.process(records)
     print(processor.report())  # this will only run when start in --no-follow mode