ScyllaTop: top-like tool to see live scylla metrics

requires a local collectd configured with the unix-sock plugin,
use the --help option for more. Run it with:

        $ scyllatop.py --help

Signed-off-by: Yoav Kleinberger <yoav@scylladb.com>
Message-Id: <bd3f8c7e120996fc464f41f60130c82e3fb55ac6.1456164703.git.yoav@scylladb.com>

.gitignore

@@ -8,3 +8,4 @@ cscope.*
 dist/ami/files/*.rpm
 dist/ami/variables.json
 dist/ami/scylla_deploy.sh
+*.pyc

tools/scyllatop/collectd.py

@@ -0,0 +1,51 @@
+import socket
+import re
+import atexit
+import os
+import logging
+
+COLLECTD_EXAMPLE_CONFIGURATION = ['LoadPlugin unixsock',
+                                  '',
+                                  '<Plugin unixsock>',
+                                  '   SocketFile "{socket}"',
+                                  '   SocketGroup "collectd"',
+                                  '   SocketPerms "0660"',
+                                  '   DeleteSocket false',
+                                  '</Plugin>']
+
+
+class Collectd(object):
+    _FIRST_LINE_PATTERN = re.compile('^(?P<lines>\d+)')
+
+    def __init__(self, socketName):
+        try:
+            self._connect(socketName)
+            atexit.register(self._cleanup)
+        except Exception as e:
+            logging.error('could not connect to {0}. {1}'.format(socketName, e))
+            quit()
+
+    def _connect(self, socketName):
+        logging.info('connecting to unix socket: {0}'.format(socketName))
+        self._socket = socket.socket(socket.SOCK_STREAM, socket.AF_UNIX)
+        self._socket.connect(socketName)
+        self._lineReader = os.fdopen(self._socket.fileno())
+
+    def query(self, command):
+        self._send(command)
+        return self._readLines()
+
+    def _send(self, command):
+        withNewline = '{command}\n'.format(command=command)
+        octets = withNewline.encode('ascii')
+        self._socket.send(octets)
+
+    def _readLines(self):
+        line = self._lineReader.readline()
+        match = self._FIRST_LINE_PATTERN.search(line)
+        howManyLines = int(match.groupdict()['lines'])
+        return [self._lineReader.readline() for _ in range(howManyLines)]
+
+    def _cleanup(self):
+        self._lineReader.close()
+        self._socket.close()

tools/scyllatop/livedata.py

@@ -0,0 +1,42 @@
+import logging
+import time
+import metric
+
+
+class LiveData(object):
+    def __init__(self, metrics, interval, collectd):
+        logging.info('will query collectd every {0} seconds'.format(interval))
+        self._startedAt = time.time()
+        self._measurements = []
+        self._interval = interval
+        self._collectd = collectd
+        self._initializeMetrics(metrics)
+        self._views = []
+        self._stop = False
+
+    def addView(self, view):
+        self._views.append(view)
+
+    @property
+    def measurements(self):
+        return self._measurements
+
+    def _initializeMetrics(self, metrics):
+        if len(metrics) > 0:
+            for metricName in metrics:
+                self._measurements.append(metric.Metric(metricName, self._collectd))
+            return
+
+        self._measurements = metric.Metric.discover(self._collectd)
+
+    def go(self):
+        while not self._stop:
+            for metric in self._measurements:
+                metric.update()
+
+            for view in self._views:
+                view.update(self)
+            time.sleep(self._interval)
+
+    def stop(self):
+        self._stop = True

tools/scyllatop/metric.py

@@ -0,0 +1,48 @@
+import logging
+import re
+
+
+class Metric(object):
+    _METRIC_SYMBOL_HOST_PATTERN = re.compile('^[^/]+/')
+    _METRIC_INFO_PATTERN = re.compile('^(?P<key>[^=]+)=(?P<value>.*)$')
+    _METRIC_DISCOVER_PATTERN = re.compile('[^ ]+ (?P<metric>.+)$')
+
+    def __init__(self, symbol, collectd):
+        self._symbol = symbol
+        self._collectd = collectd
+        self._status = {}
+
+    @property
+    def name(self):
+        hostStripped = self._METRIC_SYMBOL_HOST_PATTERN.sub('', self._symbol)
+        return hostStripped
+
+    @property
+    def symbol(self):
+        return self._symbol
+
+    @property
+    def status(self):
+        return self._status
+
+    def update(self):
+        response = self._collectd.query('GETVAL "{metric}"'.format(metric=self._symbol))
+        for line in response:
+            match = self._METRIC_INFO_PATTERN.search(line)
+            key = match.groupdict()['key']
+            value = match.groupdict()['value']
+            self._status[key] = value
+
+    @classmethod
+    def discover(cls, collectd):
+        results = []
+        logging.info('discovering metrics...')
+        response = collectd.query('LISTVAL')
+        for line in response:
+            logging.debug('LISTVAL result: {0}'.format(line))
+            match = cls._METRIC_DISCOVER_PATTERN.search(line)
+            metric = match.groupdict()['metric']
+            results.append(Metric(metric, collectd))
+
+        logging.info('found {0} metrics'.format(len(results)))
+        return results

tools/scyllatop/scyllatop.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+import argparse
+import curses
+import pprint
+import logging
+import collectd
+import metric
+import livedata
+import views.simple
+import views.means
+import userinput
+import signal
+
+
+def halt(* args):
+    quit()
+
+signal.signal(signal.SIGINT, halt)
+
+
+def main(screen, metrics, interval, collectd):
+    curses.curs_set(0)
+    liveData = livedata.LiveData(metrics, interval, collectd)
+    simpleView = views.simple.Simple(screen)
+    meansView = views.means.Means(screen)
+    liveData.addView(simpleView)
+    liveData.addView(meansView)
+    meansView.onTop()
+    userinput.UserInput(liveData, screen, simpleView, meansView)
+    liveData.go()
+
+if __name__ == '__main__':
+    description = '\n'.join(['A top-like tool for scylladb collectd metrics.',
+                            'Keyborad shortcuts: S - simple view, M - avergages over multiple cores, Q -quits',
+                            '',
+                            'You need to configure the unix-sock plugin for collectd'
+                            'before you can use this, use the -c option to give you a configuration example',
+                            'enjoy!'])
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument(
+        '-v',
+        '--verbosity',
+        help='python log level, e.g. DEBUG, INFO or ERROR',
+        choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
+        default='ERROR')
+    parser.add_argument('metrics', nargs='*', default=[], help='metrics to query, separated by spaces')
+    parser.add_argument('-i', '--interval', help="time resolution in seconds, default: 1", type=float, default=1)
+    parser.add_argument('-s', '--socket', default='/var/run/collectd-unixsock', help="unixsock plugin to connect to, default: /var/run/collectd-unixsock")
+    parser.add_argument('--print-config', action='store_true',
+                        help="print out a configuration to put in your collectd.conf (you can use -s here to define the socket path)")
+    parser.add_argument('-l', '--list', action='store_true',
+                        help="print out a list of all metrics exposed by collectd and exit")
+    arguments = parser.parse_args()
+    logging.basicConfig(filename='scyllatop.log',
+                        level=getattr(logging, arguments.verbosity),
+                        format='%(asctime)s %(levelname)s: %(message)s')
+    if arguments.print_config:
+        print(collectd.COLLECTD_EXAMPLE_CONFIGURATION.format(socket=arguments.socket))
+        quit()
+    collectd = collectd.Collectd(arguments.socket)
+    if arguments.list:
+        pprint.pprint([m.symbol for m in metric.Metric.discover(collectd)])
+        quit()
+
+    curses.wrapper(main, arguments.metrics, arguments.interval, collectd)

tools/scyllatop/userinput.py

@@ -0,0 +1,26 @@
+import threading
+import logging
+
+
+class UserInput(threading.Thread):
+    def __init__(self, liveData, screen, simpleView, meansView):
+        self._liveData = liveData
+        self._screen = screen
+        self._simpleView = simpleView
+        self._meansView = meansView
+        threading.Thread.__init__(self)
+        self.daemon = True
+        self.start()
+
+    def run(self):
+        while True:
+            keypress = self._screen.getch()
+            logging.debug('key pressed {0}'.format(keypress))
+            if keypress == ord('m'):
+                self._meansView.onTop()
+            if keypress == ord('s'):
+                self._simpleView.onTop()
+            if keypress == ord('q'):
+                logging.info('quitting on user request')
+                self._liveData.stop()
+                return

tools/scyllatop/views/base.py

@@ -0,0 +1,41 @@
+import time
+import curses
+import curses.panel
+import logging
+
+
+class Base(object):
+    def __init__(self, window):
+        lines, columns = window.getmaxyx()
+        self._window = curses.newwin(lines, columns)
+        self._panel = curses.panel.new_panel(self._window)
+
+    def writeStatusLine(self, measurements):
+        line = 'time: {0}| {1} measurements, at most {2} visible'.format(time.asctime(), len(measurements), self.availableLines())
+        columns = self.dimensions()['columns']
+        self._window.addstr(0, 0, line.ljust(columns), curses.A_REVERSE)
+
+    def availableLines(self):
+        STATUS_LINE = 1
+        return self.dimensions()['lines'] - STATUS_LINE
+
+    def refresh(self):
+        curses.panel.update_panels()
+        curses.doupdate()
+
+    def onTop(self):
+        logging.info('put {0} view on top'.format(self.__class__.__name__))
+        self._panel.top()
+        curses.panel.update_panels()
+        curses.doupdate()
+
+    def clearScreen(self):
+        self._window.clear()
+        self._window.move(0, 0)
+
+    def writeLine(self, thing, line):
+        self._window.addstr(line, 0, str(thing))
+
+    def dimensions(self):
+        lines, columns = self._window.getmaxyx()
+        return {'lines': lines, 'columns': columns}

tools/scyllatop/views/groups.py

@@ -0,0 +1,43 @@
+import re
+
+
+class Group(object):
+    _HEAD_PATTERN = re.compile('^([^/]+)-\d+/')
+
+    def __init__(self, label):
+        self._label = label
+        self._metrics = []
+
+    def add(self, metric):
+        self._metrics.append(metric)
+
+    @property
+    def metrics(self):
+        return self._metrics
+
+    @property
+    def label(self):
+        return self._label
+
+    @classmethod
+    def extractLabel(cls, metric):
+        return cls._HEAD_PATTERN.sub(r'\1-*/', metric.name)
+
+    @property
+    def size(self):
+        return len(self._metrics)
+
+
+class Groups(object):
+    def __init__(self, measurements):
+        self._groups = {}
+        self._load(measurements)
+
+    def _load(self, measurements):
+        for metric in measurements:
+            label = Group.extractLabel(metric)
+            self._groups.setdefault(label, Group(label))
+            self._groups[label].add(metric)
+
+    def all(self):
+        return self._groups.values()

tools/scyllatop/views/helpers.py

@@ -0,0 +1,9 @@
+def formatValues(status):
+    values = []
+    if len(status) == 1:
+        value = status.values()[0]
+        return '{value:.1f}'.format(value=float(value))
+    for key, value in status.iteritems():
+        values.append('{key}: {value:.1f}'.format(key=key, value=float(value)))
+
+    return ' '.join(values)

tools/scyllatop/views/means.py

@@ -0,0 +1,41 @@
+import mergeable
+import groups
+import table
+import base
+import helpers
+
+
+class Means(base.Base):
+    def update(self, liveData):
+        self.clearScreen()
+        self.writeStatusLine(liveData.measurements)
+        metricGroups = groups.Groups(liveData.measurements)
+        visible = metricGroups.all()[:self.availableLines()]
+        tableForm = self._prepareTable(visible)
+        for index, row in enumerate(tableForm.rows()):
+            line = row[: self.dimensions()['columns']]
+            self.writeLine(line, index + 1)
+
+        self.refresh()
+
+    def _prepareTable(self, groups):
+        result = table.Table('lr')
+        for group in groups:
+            result.add(self._label(group), self._values(group))
+        return result
+
+    def _label(self, group):
+        label = '{label}({size})'.format(label=group.label, size=group.size)
+        return label
+
+    def _values(self, group):
+        means = self._meanMerge(group)
+        return helpers.formatValues(means)
+
+    def _meanMerge(self, group):
+        mean = lambda vector: sum(float(x) for x in vector) / len(vector)
+        merger = mergeable.Mergeable(mean)
+        for metric in group.metrics:
+            merger.add(metric.status)
+
+        return merger.merged()

tools/scyllatop/views/mergeable.py

@@ -0,0 +1,15 @@
+class Mergeable(dict):
+    def __init__(self, mergeMethod):
+        self._mergeMethod = mergeMethod
+        dict.__init__(self)
+
+    def add(self, dictionary):
+        for key, value in dictionary.iteritems():
+            self.setdefault(key, [])
+            self[key].append(value)
+
+    def merged(self):
+        result = {}
+        for key, values in self.iteritems():
+            result[key] = self._mergeMethod(values)
+        return result