Merge pull request #8 from macbre/rename-to-elasticsearch-query

Rename this package to elasticsearch-query

Author: macbre

MANIFEST.in

@@ -1,2 +1 @@
-exclude README.rst
 prune test

Makefile

@@ -1,25 +1,26 @@
-coverage_options = --include='wikia_common_kibana.py' --omit='test/*'
+coverage_options = --include='elasticsearch_query.py' --omit='test/*'
 
 install:
 	pip install -e .[dev]
 
 test:
-	pytest -v
+	pytest -vv
 
 coverage:
 	rm -f .coverage*
 	rm -rf htmlcov/*
-	coverage run -p -m pytest -v
+	coverage run -p -m pytest -vv
 	coverage combine
 	coverage html -d htmlcov $(coverage_options)
 	coverage xml -i
 	coverage report $(coverage_options)
 
 lint:
-	pylint wikia_common_kibana.py
+	pylint elasticsearch_query.py
 
 publish:
 	# run git tag -a v0.0.0 before running make publish
-	python setup.py sdist upload -r pypi
+	python setup.py sdist
+	twine upload dist/*
 
 .PHONY: test

README.md

@@ -0,0 +1,57 @@
+elasticsearch-query
+===================
+
+[![PyPI](https://img.shields.io/pypi/v/elasticsearch-query.svg)](https://pypi.python.org/pypi/elasticsearch-query)
+[![Build Status](https://travis-ci.org/macbre/elasticsearch-query.svg?branch=master)](https://travis-ci.org/macbre/elasticsearch-query)
+
+Run queries against Kibana's Elasticsearch that gets logs from Logstash. Forked from [Wikia's `kibana.py`](https://github.com/Wikia/python-commons/blob/master/wikia/common/kibana/kibana.py).
+
+```
+pip install elasticsearch-query
+```
+
+## Basic Usage
+
+```python
+from elasticsearch_query import ElasticsearchQuery
+es_query = ElasticsearchQuery(es_host='es.prod', since=12345, period=900)
+```
+
+`es_host` needs to be specified with a host of Elasticsearch instance to connect.
+
+Provide either `since` (absolute timestamp) or `period` (last N seconds):
+
+* `since`: UNIX timestamp data should be fetched since (if None, then period specifies the last n seconds).
+* `period`: period (in seconds) before now() to be used when since is empty (defaults to last 15 minutes).
+
+### `get_rows`
+
+> Returns data matching the given query (provided as a `dict`).
+
+```python
+es_query.get_rows(match={"tags": 'edge-cache-requestmessage'}, limit=2000)
+```
+
+* `match`: query to be run against log messages (ex. {"@message": "Foo Bar DB queries"}).
+* `limit`: the number of results (defaults to 10).
+
+### `query_by_string`
+
+> Returns data matching the given query string (provided as a [Lucene query](https://lucene.apache.org/core/2_9_4/queryparsersyntax.html)).
+
+```python
+es_query.query_by_string(query='@message:"^PHP Fatal"', limit=2000)
+es_query.query_by_string(query='@message:"^PHP Fatal"', fields=['@message', '@es_query_host'], limit=2000)
+```
+
+* `query`: query string to be run against log messages (ex. `@message:"^PHP Fatal"`).
+* `fields`: optional list of fields to fetch
+* `limit`: the number of results (defaults to 10).
+
+### `get_to_timestamp`
+
+> Returns the upper time boundary for the requested data.
+
+```python
+es_query.get_to_timestamp()
+```

README.rst

@@ -1,5 +1,5 @@
 """
-Run queries against Kibana's elasticsearch
+Run queries against Kibana's Elasticsearch that gets logs from Logstash.
 @see http://elasticsearch-py.readthedocs.org/en/master/
 """
 import json
@@ -15,14 +15,14 @@
 from elasticsearch.helpers import scan
 
 
-class KibanaError(Exception):
+class ElasticsearchQueryError(Exception):
     """
-    Error that can be raised by Kibana class
+    Error that can be raised by ElasticsearchQuery class
     """
     pass
 
 
-class Kibana(object):
+class ElasticsearchQuery(object):
     """
     Elasticsearch client
     """
@@ -32,32 +32,30 @@ class Kibana(object):
     # seconds in 24h used to get the es index for yesterday
     DAY = 86400
 
-    ELASTICSEARCH_HOST = 'logs-prod.es.service.sjc.consul'  # ES5
-
-    """ Interface for querying Kibana's storage """
+    """ Interface for querying Elasticsearch storage """
     def __init__(
-            self, since=None, period=900, es_host=None,
+            self, es_host, since=None, period=900,
             read_timeout=10, index_prefix='logstash-other', index_sep='-', batch_size=1000):
         """
+        :type es_host str
         :type since int
         :type period int
-        :type es_host str
         :type read_timeout int
         :type index_prefix str
         :type index_sep str
         :type batch_size int
 
+        :arg es_host: Elasticsearch host(s) that should be used for querying
         :arg since: UNIX timestamp data should be fetched since
         :arg period: period (in seconds) before now() to be used when since is empty(defaults to last 15 minutes)
-        :arg es_host: customize Elasticsearch host(s) that should be used for querying
         :arg read_timeout: customize Elasticsearch read timeout (defaults to 10 s)
         :arg index_prefix name of the Elasticsearch index (defaults to 'logstash-other')
         :arg batch_size size of the batch sent in every requests of the ELK scroll API (defaults to 1000)
         """
-        self._es = Elasticsearch(hosts=es_host if es_host else self.ELASTICSEARCH_HOST, timeout=read_timeout)
+        self._es = Elasticsearch(hosts=es_host, timeout=read_timeout)
         self._batch_size = batch_size
 
-        self._logger = logging.getLogger('kibana')
+        self._logger = logging.getLogger(self.__class__.__name__)
 
         # if no timestamp provided, fallback to now() in UTC
         now = int(time.time())

wikia_common_kibana.py renamed to elasticsearch_query.py

@@ -2,17 +2,23 @@
 
 VERSION = '2.2.7'
 
+# @see https://packaging.python.org/tutorials/packaging-projects/#creating-setup-py
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
 # @see https://github.com/pypa/sampleproject/blob/master/setup.py
 setup(
-    name='wikia_common_kibana',
+    name='elasticsearch-query',
     version=VERSION,
-    author='Wikia Engineering',
-    author_email='techteam-l@wikia-inc.com',
+    author='Maciej Brencz',
+    author_email='macbre@wikia-inc.com',
     license='MIT',
-    description='Run queries against Kibana\'s Elasticsearch 6',
+    description='Run queries against Kibana\'s Elasticsearch that gets logs from Logstash.',
     keywords='logstash kibana elasticsearch logging',
-    url='https://github.com/macbre/wikia-common-kibana',
-    py_modules=["wikia_common_kibana"],
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url='https://github.com/macbre/elasticsearch-query',
+    py_modules=["elasticsearch_query"],
     extras_require={
         'dev': [
             'coverage==4.5.1',

setup.py

@@ -0,0 +1,71 @@
+"""
+Set of unit tests for elastic_search.py
+"""
+import time
+
+from elasticsearch_query import ElasticsearchQuery
+
+
+def test_indexes():
+    es_query = ElasticsearchQuery(es_host='foo')
+    assert es_query._index.startswith('logstash-')
+
+
+def test_indexes_prefix():
+    es_query = ElasticsearchQuery(es_host='foo', index_prefix='syslog-ng')
+    assert es_query._index.startswith('syslog-ng-')
+
+
+def test_indexes_prefix_with_separator():
+    es_query = ElasticsearchQuery(es_host='foo', index_prefix='syslog-ng', index_sep="_")
+    assert es_query._index.startswith('syslog-ng_')
+    assert ',syslog-ng_' in es_query._index
+
+
+def test_format_index():
+    assert ElasticsearchQuery.format_index(prefix='logstash', timestamp=1) == 'logstash-1970.01.01'
+    assert ElasticsearchQuery.format_index(prefix='logstash', timestamp=1408450795) == 'logstash-2014.08.19'
+    assert ElasticsearchQuery.format_index(prefix='logstash-foo', timestamp=1408450795) == 'logstash-foo-2014.08.19'
+    assert ElasticsearchQuery.format_index(prefix='syslog-ng', timestamp=1408450795, sep="_") == 'syslog-ng_2014.08.19'
+
+
+def test_time():
+    now = int(time.time())
+
+    cases = [
+        # till now
+        {
+            "since": None,
+            "expected_since": now - 60,
+            "expected_to": now - 5,
+            "period": 60
+        },
+        # strictly defined time period
+        {
+            "since": 12345,
+            "expected_since": 12346,
+            "expected_to": now - 5,
+            "period": 600
+        }
+    ]
+
+    for case in cases:
+        check_time(**case)
+
+
+def check_time(since, expected_since, expected_to, period):
+    es_query = ElasticsearchQuery('foo.host.net', since, period)
+
+    assert es_query._since == expected_since
+    assert es_query.get_to_timestamp() == expected_to
+
+
+def test_get_timestamp_filer():
+    es_query = ElasticsearchQuery(es_host='foo', since=123456, period=60)
+    res = es_query._get_timestamp_filer()
+
+    print(res)
+
+    assert res['range']['@timestamp'] is not None
+    assert res['range']['@timestamp']['gte'] == '1970-01-02T10:17:37.000Z'
+    assert res['range']['@timestamp']['lte'] is not None