Adding code for python-based pinot dashboard

.gitignore

@@ -35,4 +35,10 @@ pinot-transport/test-output/
 pinot-broker/test-output/
 pinot-core/test-output/
 
-
+pinot-dashboard/.env
+pinot-dashboard/activate
+pinot-dashboard/config.yml
+pinot-dashboard/**/*.swp
+pinot-dashboard/**/*.pyc
+pinot-dashboard/**/*.swo
+pinot-dashboard/logs/*

pinot-dashboard/README

@@ -0,0 +1,15 @@
+This is a simple flask app that lets you browse Pinot 2.0 clusters. It gathers
+info by hitting pinot-controller endpoints and querying ZK.
+
+Set up steps:
+
+1) Run ./bootstrap.sh to install initial python dependencies and create virtual
+   env.
+
+2) Copy config.sample.yml to config.yml and hand edit as needed
+
+3) Do ./start.sh or ./stop.sh to start in background
+
+Do ". activate" and python run.py to start in foreground
+
+4) Logs will be written to logs/webui.log

pinot-dashboard/app/__init__.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python2.6
+
+import os
+from flask import Flask, jsonify, request
+
+from pinot_resource import PinotResource
+from pinot_fabric import PinotFabric
+from zk import PinotZk
+
+from config import ConfigManager
+
+app = Flask(__name__)
+app.config['DEBUG'] = True
+
+config = ConfigManager(app.logger)
+config.load()
+
+
+@app.route('/runpql/<string:fabric>')
+def send_pql(fabric):
+  pql = request.args.get('pql')
+  pinot_fabric = PinotFabric(config, fabric)
+  return jsonify(dict(pinot_fabric.run_pql(pql)))
+
+
+@app.route('/clusters/<string:fabric>')
+def list_resources(fabric):
+  pinot_fabric = PinotFabric(config, fabric)
+  return jsonify(dict(clusters=pinot_fabric.get_resources()))
+
+
+@app.route('/cluster/<string:fabric>/<string:cluster>')
+def cluster_info(fabric, cluster):
+  resource = PinotResource(config, app.logger, fabric, cluster)
+  zk = PinotZk(config, fabric)
+  return jsonify(dict(info=resource.get_info(), nodes=resource.get_nodes(zk)))
+
+
+@app.route('/segments/<string:fabric>/<string:cluster>/<string:table>')
+def get_segments(fabric, cluster, table):
+  resource = PinotResource(config, app.logger, fabric, cluster)
+  return jsonify(dict(segments=resource.get_table_segments(table)))
+
+
+@app.route('/fabrics')
+def list_fabrics():
+  return jsonify(dict(fabrics=config.get_fabrics()))
+
+
+@app.route('/')
+def index():
+
+  # not using flask.render_template() as the angular-js {{ }} notation throws it off
+  return open(os.path.join(os.path.dirname(__file__), 'templates/home.html')).read()

pinot-dashboard/app/config.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python2.6
+
+import yaml
+
+
+class ConfigManager(object):
+  def __init__(self, logger):
+    self.path = 'config.yml'
+    self.logger = logger
+    self.config = {}
+
+  def load(self):
+    try:
+      with open(self.path, 'r') as h:
+        contents = h.read()
+    except IOError:
+      self.logger.exception('Failed reading config file')
+      return
+
+    try:
+      self.config = yaml.load(contents)
+    except yaml.YAMLError:
+      self.logger.exception('Failed parsing config yaml')
+
+  def get_controller_url(self, fabric):
+    try:
+      return self.config['fabrics'][fabric]['controller_url']
+    except KeyError:
+      self.logger.exception('Failed getting controller url from config')
+
+  def get_zk_host(self, fabric):
+    try:
+      return self.config['fabrics'][fabric]['zk_host']
+    except KeyError:
+      self.logger.exception('Failed getting zookeeper host from config')
+
+  def get_zk_root(self, fabric):
+    try:
+      return self.config['fabrics'][fabric]['zk_root']
+    except KeyError:
+      self.logger.exception('Failed getting zookeeper root from config')
+
+  def get_fabrics(self):
+    try:
+      return self.config['fabrics'].keys()
+    except KeyError:
+      self.logger.exception('Failed getting list of fabrics from config')
+
+  def get_flask_port(self):
+    try:
+      return int(self.config['listen_port'])
+    except (KeyError, ValueError):
+      self.logger.exception('Failed getting flask port from config')

pinot-dashboard/app/pinot_fabric.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python2.6
+
+import requests
+
+
+class PinotFabric(object):
+
+  def __init__(self, config, fabric):
+    self.config = config
+    self.fabric = fabric
+
+  def get_resources(self):
+    host = self.config.get_controller_url(self.fabric)
+    r = requests.get('{0}/dataresources/'.format(host))
+    data = r.json()
+    return filter(lambda x: x != 'brokerResource', data['resources'])
+
+  def run_pql(self, pql):
+    host = self.config.get_controller_url(self.fabric)
+    r = requests.get('{0}/pql'.format(host), params={'pql': pql})
+    success = True
+    try:
+      result = r.json()
+    except ValueError:
+      result = r.text
+      success = False
+    return {
+      'success': success,
+      'result': result
+    }

pinot-dashboard/app/pinot_resource.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python2.6
+
+import re
+import json
+import os
+import kazoo
+import requests
+import string
+from collections import defaultdict
+
+
+class PinotResource(object):
+
+  def __init__(self, config, logger, fabric, resource):
+    self.config = config
+    self.logger = logger
+    self.fabric = fabric
+    self.resource = resource
+
+  def get_info(self):
+    host = self.config.get_controller_url(self.fabric)
+    r = requests.get('{0}/dataresources/{1}'.format(host, self.resource))
+
+    # hacky stopgap
+    c = r.json()['config'].values().pop()
+    results = re.findall('\{(\w+=[^\}]+)\}', c)
+    data = {}
+    for result in results:
+      result = result.split('{')[-1]
+      m = re.search('([^=]+)=\[([^]]+)\]', result)
+      if m:
+        k, v = m.groups()
+        data.update({k: map(string.strip, v.split(','))})
+      else:
+        m = re.findall('(\w+)=([^,]+)', result)
+        if m:
+          data.update(dict(m))
+
+    return data
+
+  def get_table_segments(self, table):
+    host = self.config.get_controller_url(self.fabric)
+    r = requests.get('{0}/dataresources/{1}/{2}/segments'.format(host, self.resource, table))
+    segments = r.json()['segments']
+    return segments
+
+  def get_nodes(self, pinot_zoo):
+    if not re.match('^[a-zA-Z0-9_]+$', self.resource):
+      self.logger.error('potentially unsafe resource name: {0}'.format(self.resource))
+      return False
+
+    zk = pinot_zoo.get_handle()
+
+    if not zk:
+      return False
+
+    root = self.config.get_zk_root(self.fabric)
+    state_path = os.path.join(root, 'IDEALSTATES', self.resource)
+
+    try:
+      ideal_state = zk.get(state_path)[0]
+    except kazoo.exceptions.NoNodeError:
+      self.logger.exception('Failed getting statefile')
+      pinot_zoo.close()
+      return False
+
+    host_maps = json.loads(ideal_state)['mapFields']
+
+    nodes_list = set()
+
+    for servers in host_maps.itervalues():
+      for hostname in servers.iterkeys():
+        nodes_list.add(hostname)
+
+    nodes_status = defaultdict(dict)
+
+    for node in nodes_list:
+      instance_path = os.path.join(root, 'LIVEINSTANCES', node)
+      parts = node.split('_')
+      node = parts[1]
+      nodes_status[node]['helix_port'] = parts[2]
+      nodes_status[node]['type'] = parts[0]
+      if zk.exists(instance_path):
+        nodes_status[node]['online'] = True
+      else:
+        nodes_status[node]['online'] = False
+
+    pinot_zoo.close()
+
+    return nodes_status

pinot-dashboard/app/static/css/codemirror-elegant.css

@@ -0,0 +1,13 @@
+.cm-s-elegant span.cm-number, .cm-s-elegant span.cm-string, .cm-s-elegant span.cm-atom {color: #762;}
+.cm-s-elegant span.cm-comment {color: #262; font-style: italic; line-height: 1em;}
+.cm-s-elegant span.cm-meta {color: #555; font-style: italic; line-height: 1em;}
+.cm-s-elegant span.cm-variable {color: black;}
+.cm-s-elegant span.cm-variable-2 {color: #b11;}
+.cm-s-elegant span.cm-qualifier {color: #555;}
+.cm-s-elegant span.cm-keyword {color: #730;}
+.cm-s-elegant span.cm-builtin {color: #30a;}
+.cm-s-elegant span.cm-link {color: #762;}
+.cm-s-elegant span.cm-error {background-color: #fdd;}
+
+.cm-s-elegant .CodeMirror-activeline-background {background: #e8f2ff !important;}
+.cm-s-elegant .CodeMirror-matchingbracket {outline:1px solid grey; color:black !important;}