diff --git a/tailon/commands.py b/tailon/commands.py index 460e0ee..e25ac65 100644 --- a/tailon/commands.py +++ b/tailon/commands.py @@ -80,6 +80,12 @@ def tail(self, n, fn, stdout, stderr, **kw): log.debug('running tail %s, pid: %s', cmd, proc.proc.pid) return proc + def grep_tail(self, n, stdout, stderr, **kw): + cmd = [self.toolpaths.cmd_tail, '-n', str(n)] + proc = process.Subprocess(cmd, stdout=stdout, stderr=stderr, bufsize=1, **kw) + log.debug('running tail %s, pid: %s', cmd, proc.proc.pid) + return proc + def zcat(self, fn, stdout, stderr, **kw): cmd = [self.toolpaths.cmd_zcat, '-f', '-r'] cmd.extend(fn) @@ -103,19 +109,22 @@ def tail_sed(self, n, fn, script, stdout, stderr): sed = self.sed(script, None, stdout=STREAM, stderr=STREAM, stdin=tail.stdout) tail.stdout.close() return tail, sed + def all_awk(self, fn, script, stdout, stderr): zcat = self.zcat(fn, stdout=subprocess.PIPE, stderr=STREAM) awk = self.awk(script, None, stdout=STREAM, stderr=STREAM, stdin=zcat.stdout) return zcat, awk - def all_grep(self, fn, regex, stdout, stderr): + def all_grep(self, grep_lines, fn, regex, stdout, stderr): if self.toolpaths.cmd_sift: - grep = self.sift(regex, fn, stdout=STREAM, stderr=STREAM) - return grep + grep = self.sift(regex, fn, stdout=subprocess.PIPE, stderr=STREAM) + tail = self.grep_tail(grep_lines, stdout=STREAM, stderr=STREAM, stdin=grep.stdout) + return grep, tail else: zcat = self.zcat(fn, stdout=subprocess.PIPE, stderr=STREAM) - grep = self.grep(regex, None, stdout=STREAM, stderr=STREAM, stdin=zcat.stdout) - return zcat, grep + grep = self.grep(regex, None, stdout=subprocess.PIPE, stderr=STREAM, stdin=zcat.stdout) + tail = self.grep_tail(grep_lines, stdout=STREAM, stderr=STREAM, stdin=grep.stdout) + return zcat, grep, tail def all_sed(self, fn, script, stdout, stderr): zcat = self.zcat(fn, stdout=subprocess.PIPE, stderr=STREAM) diff --git a/tailon/main.py b/tailon/main.py index 175a3ed..a8c0363 100644 --- a/tailon/main.py +++ b/tailon/main.py @@ -72,6 +72,7 @@ def parseconfig(cfg): 'users': raw_config.get('users', {}), 'wrap-lines': raw_config.get('wrap-lines', True), 'tail-lines': raw_config.get('tail-lines', 10), + 'grep-lines': raw_config.get('grep-lines', 3000), 'live-view': raw_config.get('live-view', False), 'download-url': raw_config.get('download-url', None), } @@ -117,6 +118,7 @@ def parseopts(args=None): relative-root: /tailon # web app root path (default: '') commands: [tail, grep] # allowed commands tail-lines: 10 # number of lines to tail initially + grep-lines: 3000 # number max of lines to grep wrap-lines: true # initial line-wrapping state live-view: False # view files live (tail) or just search in files @@ -184,6 +186,8 @@ def parseopts(args=None): arg('-t', '--tail-lines', default=10, type=int, metavar='num', help='number of lines to tail initially') + arg('-g', '--grep-lines', default=3000, type=int, metavar='num', + help='number max of lines to grep') arg('-m', '--commands', nargs='*', metavar='cmd', choices=commands.ToolPaths.command_names, default=['tail', 'grep', 'awk'], @@ -216,6 +220,7 @@ def setup(opts): 'relative-root': opts.__dict__.get('relative_root', ''), 'debug': opts.__dict__.get('debug', False), 'tail-lines': opts.__dict__.get('tail_lines', 10), + 'grep-lines': opts.__dict__.get('grep_lines', 3000), 'wrap-lines': opts.__dict__.get('wrap-lines', True), 'live-view': opts.__dict__.get('live-view', False), 'download-url': opts.__dict__.get('download-url', False), diff --git a/tailon/server.py b/tailon/server.py index ebc2e69..2c0ba83 100644 --- a/tailon/server.py +++ b/tailon/server.py @@ -111,6 +111,7 @@ def __init__(self, *args, **kw): self.cmd_control = self.application.cmd_control self.toolpaths = self.application.toolpaths self.initial_tail_lines = self.config['tail-lines'] + self.initial_grep_lines = self.config['grep-lines'] self.last_stdout_line = [] self.last_stderr_line = [] @@ -182,7 +183,7 @@ def on_message(self, message): log.debug('received message: %r', command) log.debug(command.keys()) - if not set(command.keys()) <= {'command', 'live-view', 'path', 'tail-lines', 'script'}: + if not set(command.keys()) <= {'command', 'live-view', 'path', 'tail-lines', 'grep-lines', 'script'}: return if command['command'] not in allowed_commands: @@ -213,13 +214,14 @@ def on_message(self, message): elif 'grep' == command['command']: n = command.get('tail-lines', self.initial_tail_lines) + grep_lines = command.get('grep-lines', self.initial_grep_lines) regex = command.get('script', '.*') log.debug('n = %s, path = %s, regex = %s' %(n, path, regex)) if not command['live-view'] and self.toolpaths.cmd_sift: - proc_grep = self.cmd_control.all_grep(path, regex, STREAM, STREAM) + proc_pregrep, proc_grep = self.cmd_control.all_grep(grep_lines, path, regex, STREAM, STREAM) elif not command['live-view'] and not self.toolpaths.cmd_sift: - proc_zcat, proc_grep = self.cmd_control.all_grep(path, regex, STREAM, STREAM) + proc_zcat, proc_pregrep, proc_grep = self.cmd_control.all_grep(grep_lines, path, regex, STREAM, STREAM) elif command['live-view']: proc_tail, proc_grep = self.cmd_control.tail_grep(n, live_path, regex, STREAM, STREAM) self.processes['grep'] = proc_grep