Summer Updates

bin/tinyfeedback

@@ -14,5 +14,5 @@ LOG_LEVEL = 'DEBUG'
 
 
 if __name__ == '__main__':
-    tinyfeedback.webserver.set_up_server(port=tinyfeedback.helper.PORT,
+    tinyfeedback.webserver.set_up_server(host=tinyfeedback.helper.HOST, port=tinyfeedback.helper.PORT,
             log_path=LOG_PATH, log_level=LOG_LEVEL)

requirements.txt

@@ -3,3 +3,5 @@ simplejson
 twisted
 txredisapi
 txroutes
+numpy
+pandas

simple_tail.py

@@ -0,0 +1,82 @@
+import os, time
+import traceback
+
+class SimpleTail(object):
+    def __init__(self, filename):
+        self.filename = filename
+        self.file = None
+        self.inode = None
+
+    def readline(self, sleep_on_empty=False):
+        line = None
+        try:
+            # 1) open the file if it's not open
+            if not self.file:
+                self.open_file()
+
+            # 2) try to read a line of the file
+            line = self.file.readline()
+
+            # 3a) if a line is read, update the timestamp of the last line read
+            if line:
+                self.last_read = time.time()
+            elif time.time() - self.last_read > 10:
+                # 3b) If no changes to the file in 10 seconds reopen it.
+                try:
+                    self.file.close()
+                except:
+                    traceback.print_exc()
+
+                self.open_file()
+        except ValueError:
+            # 4) if we got a ValueError than it means that readline() was probably closed
+            try:
+                # 5) try to close it again
+                self.file.close()
+            except:
+                traceback.print_exc()
+
+            # 6) reopen
+            self.open_file()
+        except:
+            traceback.print_exc()
+            time.sleep(5)
+
+        # 7) if line is empty, and we sleep on empty lines.. sleep
+        if line == '' and sleep_on_empty:
+            time.sleep(1)
+
+        # 8) if we threw an exception line will be None, set it to empty string
+        if line == None:
+            line = ''
+
+        # 9) return value
+        return line
+
+    def open_file(self):
+        self.file = open(self.filename,'r')
+
+        st_results = os.stat(self.filename)
+        st_size = st_results[6]
+        inode = st_results[1]
+
+        inode_has_not_changed = inode == self.inode or self.inode == None
+
+        if inode_has_not_changed:
+            # seek to end of file if we just started tailing, or are
+            # 
+            self.file.seek(st_size)
+        else:
+            # do not seek to the end of the file, since it's a new file which
+            # in theory contains new items to tail
+            pass
+
+        # store last inode
+        self.inode = st_results[1]
+        self.last_read = time.time()
+
+if __name__ == '__main__':
+    a = SimpleTail("/var/log/test.log")
+
+    while True:
+        print a.readline(sleep_on_empty=True)

tinyfeedback/helper.py

@@ -1,70 +1,76 @@
 import platform
-import subprocess
 import time
 import urllib
+import urllib2
+
 from twisted.web.client import getPage
 
+import simple_tail
 
 PORT = 8000
 HOST = '127.0.0.1'
 
-
 def send_once(component, data_dict):
     url = 'http://%s:%s/data/%s' % (HOST, PORT, component)
 
     try:
-        urllib.urlopen(url, data=urllib.urlencode(data_dict))
+        urllib2.urlopen(url, data=urllib.urlencode(data_dict), timeout=5)
 
     except IOError:
         # Failed to send, just keep going
         pass
 
-
-def send_once_using_twisted(component, data_dict):
+def send_once_using_twisted(component, data_dict, timeout=35, ignore_errors=True):
     url = 'http://%s:%s/data/%s' % (HOST, PORT, component)
 
     d = getPage(
-            str(url),
-            method='POST',
-            postdata=urllib.urlencode(data_dict),
-            headers={'Content-Type':'application/x-www-form-urlencoded'},
-            timeout=10,
-            )
+        str(url),
+        method='POST',
+        postdata=urllib.urlencode(data_dict),
+        headers={'Content-Type':'application/x-www-form-urlencoded'},
+        timeout=timeout)
 
-    # swallow errors
-    d.addErrback(lambda x: None)
+    if ignore_errors:
+        d.addErrback(lambda x: None)
 
+    return d
 
 def tail_monitor(component, log_filename, line_callback_func, data_arg={},
-        format_data_callback_func=None, interval=60):
+        format_data_callback_func=None, interval=60, line_callback_args=None,
+        log_subcomponent='monitor'):
 
     url = 'http://%s:%s/data/%s' % (HOST, PORT, component)
 
-    initial_data = data_arg
-    current_data = data_arg.copy()
-
     if is_osx():
-        arguments = '-F'
+    	arguments = '-F'
+
     else:
-        arguments = '--follow=name'
+    	arguments = '--follow=name'
+
+    # initialize a copy of that initial data
+    initial_data = data_arg
+    current_data = initial_data.copy()
 
-    tail_process = subprocess.Popen(['tail', arguments, log_filename],
-            stdout=subprocess.PIPE)
+    tail = simple_tail.SimpleTail(log_filename)
 
     last_update = time.time()
 
     while True:
-        line = tail_process.stdout.readline()
+        line = tail.readline()
 
-        if line.strip() == '':
+        # sleep one second if EOF or ''
+        if '\0' in line or line.strip() == '':
             time.sleep(1)
         else:
-            line_callback_func(current_data, line)
+            if line_callback_args:
+                line_callback_func(current_data, line, **line_callback_args)
+            else:
+                line_callback_func(current_data, line)
 
         current_time = time.time()
 
         if current_time - last_update >= interval:
-            last_update = current_time
+	    last_update = current_time
 
             if format_data_callback_func:
                 current_data = format_data_callback_func(current_data)