Merge pull request #7 from objectrocket/elasticsearch_2_x_updates

Elasticsearch 2.x updates

README.md

@@ -82,7 +82,7 @@ optional arguments:
     - RTR: a client node, node.master = false, node.data = false
     - UNK: node with an unkown or undetermined role
 - os
-  - load: the 1 minute / 5 minute / 15 minute [load average](http://blog.scoutapp.com/articles/2009/07/31/understanding-load-averages) of the node
+  - load: the 1 minute / 5 minute / 15 minute [load average](http://blog.scoutapp.com/articles/2009/07/31/understanding-load-averages) of the node (only 1 minute load average for Elasticsearch 2.x+)
   - mem: percentage of total memory used on the node (including memory used by the kernel and other processes besides Elasticsearch)
 - [jvm](https://www.elastic.co/guide/en/elasticsearch/guide/current/_monitoring_individual_nodes.html#_jvm_section)
   - heap: percentage of Java heap memory in use.  Java garbage collections occur when this reaches or exceeds 75%.
@@ -94,7 +94,6 @@ optional arguments:
   - search: all search and query requests
   - bulk: bulk requests
   - get: all get-by-ID operations
-  - merge: threadpool for managing Lucene merges
 - [fielddata](https://www.elastic.co/guide/en/elasticsearch/guide/current/_limiting_memory_usage.html#fielddata-size)
   - fde: count of field data evictions that have occurred since last update
   - fdt: number of times the field data circuit breaker has tripped since the last update
@@ -104,6 +103,7 @@ optional arguments:
 - data_nodes: metrics useful only for data-bearing nodes
   - merges: total time spent in Lucene segment merges since the last time the node was restarted
   - idx st: [index store throttle](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules-store.html#store-throttling), the total time indexing has been throttled to a single thread since the last time the node was restarted (see [Segments and Merging](https://www.elastic.co/guide/en/elasticsearch/guide/current/indexing-performance.html#segments-and-merging))
+  - disk usage: the total space used and percentage of space used for storing Elasticsearch data files
   - docs: the total number of documents in all index shards allocated to this node.  If there is a second number, this is the total number of deleted documents not yet merged
 
 ## License

elasticstat/__init__.py

@@ -1,2 +1,2 @@
 __author__ = 'Jeff Tharp'
-__version__ = '1.0.0'
+__version__ = '1.1.0'

elasticstat/elasticstat.py

@@ -44,7 +44,7 @@
 NODES_TEMPLATE['threads'] = """{threads:<8}"""
 NODES_TEMPLATE['fielddata'] = """{fielddata:^7}"""
 NODES_TEMPLATE['connections'] = """{http_conn:>6} {transport_conn:>6}"""
-NODES_TEMPLATE['data_nodes'] = """{merge_time:>8} {store_throttle:>8}  {docs}"""
+NODES_TEMPLATE['data_nodes'] = """{merge_time:>8} {store_throttle:>8} {fs:>16}  {docs}"""
 NODES_FAILED_TEMPLATE = """{name:24} {role:<6}       (No data received, node may have left cluster)"""
 NODE_HEADINGS = {}
 NODE_HEADINGS["name"] = "nodes"
@@ -61,7 +61,8 @@
 NODE_HEADINGS["merge_time"] = "merges"
 NODE_HEADINGS["store_throttle"] = "idx st"
 NODE_HEADINGS["docs"] = "docs"
-DEFAULT_THREAD_POOLS = ["index", "search", "bulk", "get", "merge"]
+NODE_HEADINGS["fs"] = "disk usage"
+DEFAULT_THREAD_POOLS = ["index", "search", "bulk", "get"]
 CATEGORIES = ['general', 'os', 'jvm', 'threads', 'fielddata', 'connections', 'data_nodes']
 
 class ESArgParser(argparse.ArgumentParser):
@@ -161,7 +162,28 @@ def colorize(self, msg, color):
 
     def thetime(self):
         return datetime.datetime.now().strftime("%H:%M:%S")
-
+
+    def size_human(self, size):
+        for unit in ['B','KB','MB','GB','TB','PB','EB','ZB']:
+            if abs(size) < 1024.0:
+                return "{:6.2f} {}".format(size, unit)
+            size /= 1024.0
+        return "{:6.2f} {}".format(size, 'YB')
+
+    def get_disk_usage(self, node_fs_stats):
+        # Calculate used disk space
+        if node_fs_stats["total"] == {}:
+            # Not a data node
+            return "-"
+
+        total_in_bytes = node_fs_stats["total"]["total_in_bytes"]
+        used_in_bytes = total_in_bytes - node_fs_stats["total"]["available_in_bytes"]
+
+        used_percent = int((float(used_in_bytes) / float(total_in_bytes)) * 100)
+        used_human = self.size_human(used_in_bytes)
+
+        return "{}|{}%".format(used_human, used_percent)
+
     def get_role(self, attributes):
         # This is dumb, but if data/master is true, ES doesn't include the key in 
         # the attributes subdoc.  Why?? :-P
@@ -224,7 +246,7 @@ def get_fd_stats(self, node_id, current_evictions, current_tripped):
             fdt_delta = current_tripped - self.node_counters['fd'][node_id]['fdt']
             self.node_counters['fd'][node_id]['fdt'] = current_tripped
             return("{0}|{1}".format(fde_delta, fdt_delta))
-        
+
     def get_http_conns(self, node_id, http_conns):
         # check if this is a new node
         if node_id not in self.node_counters['hconn']:
@@ -249,9 +271,15 @@ def process_node_general(self, role, node_id, node):
         return(NODES_TEMPLATE['general'].format(name=node_name, role=node_role))
 
     def process_node_os(self, role, node_id, node):
-        return(NODES_TEMPLATE['os'].format(load_avg="/".join(str(x) for x in node['os']['load_average']),
+        node_load_avg = node['os']['load_average']
+        if isinstance(node_load_avg, list):
+            node_load_avg="/".join(str(x) for x in node_load_avg)
+        else:
+            # Elasticsearch 2.x+ only return 1 load average, not the standard 5/10/15 min avgs
+            node_load_avg = str(node_load_avg)
+        return(NODES_TEMPLATE['os'].format(load_avg=node_load_avg,
                                            used_mem="{0}%".format(node['os']['mem']['used_percent'])))
-    
+
     def process_node_jvm(self, role, node_id, node):
         processed_node_jvm = {}
         processed_node_jvm['used_heap'] = "{0}%".format(node['jvm']['mem']['heap_used_percent'])
@@ -297,10 +325,12 @@ def process_node_data_nodes(self, role, node_id, node):
                 processed_node_dn['docs'] = "{0}|{1}".format(doc_count, deleted_count)
             else:
                 processed_node_dn['docs'] = str(doc_count)
+            processed_node_dn['fs'] = self.get_disk_usage(node['fs'])
         else:
             processed_node_dn['merge_time'] = "-"
             processed_node_dn['store_throttle'] = "-"
             processed_node_dn['docs'] = "-"
+            processed_node_dn['fs'] = "-"
         return(NODES_TEMPLATE['data_nodes'].format(**processed_node_dn))        
 
     def process_node(self, role, node_id, node):

extra/man/elasticstat.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "ELASTICSTAT" "1" "September 2015" "" ""
+.TH "ELASTICSTAT" "1" "December 2015" "" ""
 .
 .SH "NAME"
 \fBelasticstat\fR \- Real\-time performance monitoring of an Elasticsearch cluster
@@ -121,7 +121,7 @@ The role this node serves in the cluster\.
 .
 .TP
 load
-The 1min/5min/15min load average of the node\.
+The 1min/5min/15min load average of the node\. (Only 1 minute load average for Elasticsearch 2\.x+)
 .
 .TP
 mem
@@ -167,10 +167,6 @@ Bulk requests\.
 get
 All get\-by\-ID operations\.
 .
-.TP
-merge
-Threadpool for managing Lucene merges\.
-.
 .SS "FIELD DATA"
 .
 .TP
@@ -202,6 +198,10 @@ idx st
 This is the "index store throttle": the total time indexing has been throttled to a single thread since the last time the node was restarted\.
 .
 .TP
+disk usage
+The total space used and percentage of space used for storing Elasticsearch data files\.
+.
+.TP
 docs
 The total number of documents in all index shards allocated to this node\. If there is a second number, this is the total number of deleted documents not yet merged\.
 .

extra/man/elasticstat.ronn

@@ -100,7 +100,7 @@ These metrics are displayed for each node in the cluster.
 ### OS METRICS
 
   * load:
-    The 1min/5min/15min load average of the node.
+    The 1min/5min/15min load average of the node.  (Only 1 minute load average for Elasticsearch 2.x+)
 
   * mem:
     Percentage of total memory used on the node, including memory used by the kernel and other processes besides Elasticsearch.
@@ -130,8 +130,6 @@ Default threadpools listed are as follows:
     Bulk requests.
   * get:
     All get-by-ID operations.
-  * merge:
-    Threadpool for managing Lucene merges.
 
 ### FIELD DATA
 
@@ -153,6 +151,8 @@ Default threadpools listed are as follows:
     Total time spent in Lucene segment merges since the last time the node was restarted.
   * idx st:
     This is the "index store throttle": the total time indexing has been throttled to a single thread since the last time the node was restarted.
+  * disk usage:
+    The total space used and percentage of space used for storing Elasticsearch data files.
   * docs:
     The total number of documents in all index shards allocated to this node.  If there is a second number, this is the total number of deleted documents not yet merged.