From fa35b42b8aa4f1802dc8e42e92b806eecb18e600 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Mon, 24 Oct 2022 11:32:56 +0100 Subject: [PATCH 01/26] Collect events before ingestion --- axiom/logging.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/axiom/logging.py b/axiom/logging.py index e46c34f..73f3f28 100644 --- a/axiom/logging.py +++ b/axiom/logging.py @@ -1,15 +1,19 @@ """Logging contains the AxiomHandler and related methods to do with logging.""" +import time + from logging import Handler, NOTSET, getLogger, WARNING from .client import Client - class AxiomHandler(Handler): """A logging handler that sends logs to Axiom.""" client: Client dataset: str + logcache: list + interval: int + last_run: float - def __init__(self, client: Client, dataset: str, level=NOTSET): + def __init__(self, client: Client, dataset: str, level=NOTSET, interval=3): Handler.__init__(self, level) # set urllib3 logging level to warning, check: # https://github.com/axiomhq/axiom-py/issues/23 @@ -18,7 +22,16 @@ def __init__(self, client: Client, dataset: str, level=NOTSET): getLogger("urllib3").setLevel(WARNING) self.client = client self.dataset = dataset + self.logcache = [] + self.last_run = time.now() + self.interval = interval def emit(self, record): - # FIXME: Don't do an ingest call for every event - self.client.ingest_events(self.dataset, [record.__dict__]) + """emit sends a log to Axiom.""" + self.logcache.append(record) + if time.now() - self.last_run > self.interval: + self.client.ingest_events(self.dataset, self.logcache) + self.last_run = time.now() + self.logcache = [] + else: + return From f96d6df2c745b070bff73d31a19ddf159473b5ca Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Mon, 24 Oct 2022 11:43:58 +0100 Subject: [PATCH 02/26] Flush logs on exit --- axiom/logging.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/axiom/logging.py b/axiom/logging.py index 73f3f28..2bfe8cb 100644 --- a/axiom/logging.py +++ b/axiom/logging.py @@ -1,9 +1,11 @@ """Logging contains the AxiomHandler and related methods to do with logging.""" import time +import atexit from logging import Handler, NOTSET, getLogger, WARNING from .client import Client + class AxiomHandler(Handler): """A logging handler that sends logs to Axiom.""" @@ -26,6 +28,9 @@ def __init__(self, client: Client, dataset: str, level=NOTSET, interval=3): self.last_run = time.now() self.interval = interval + # register flush on exit, + atexit.register(self.flush) + def emit(self, record): """emit sends a log to Axiom.""" self.logcache.append(record) @@ -35,3 +40,8 @@ def emit(self, record): self.logcache = [] else: return + + def flush(self): + """flush sends all logs in the logcache to Axiom.""" + self.client.datasets.ingest_events(self.dataset, self.logcache) + self.logcache = [] From 440746d698b5754591a5f1dcf113c187872d4f7b Mon Sep 17 00:00:00 2001 From: Arne Bahlo Date: Thu, 24 Nov 2022 10:52:21 +0100 Subject: [PATCH 03/26] Replace time.now() w/ time.time() in logging.py --- axiom/logging.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/axiom/logging.py b/axiom/logging.py index 2bfe8cb..65bce27 100644 --- a/axiom/logging.py +++ b/axiom/logging.py @@ -25,7 +25,7 @@ def __init__(self, client: Client, dataset: str, level=NOTSET, interval=3): self.client = client self.dataset = dataset self.logcache = [] - self.last_run = time.now() + self.last_run = time.time() self.interval = interval # register flush on exit, @@ -34,14 +34,14 @@ def __init__(self, client: Client, dataset: str, level=NOTSET, interval=3): def emit(self, record): """emit sends a log to Axiom.""" self.logcache.append(record) - if time.now() - self.last_run > self.interval: + if time.time() - self.last_run > self.interval: self.client.ingest_events(self.dataset, self.logcache) - self.last_run = time.now() + self.last_run = time.time() self.logcache = [] else: return def flush(self): """flush sends all logs in the logcache to Axiom.""" - self.client.datasets.ingest_events(self.dataset, self.logcache) + self.client.ingest_events(self.dataset, self.logcache) self.logcache = [] From 6a9229e3c7713c9a0d5888fc39b4fa0f68134c31 Mon Sep 17 00:00:00 2001 From: Arne Bahlo Date: Thu, 24 Nov 2022 11:43:25 +0100 Subject: [PATCH 04/26] Write very basic test for debounced logger --- axiom/logging.py | 19 ++++++++----------- tests/test_logger.py | 18 +++++++++++++++--- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/axiom/logging.py b/axiom/logging.py index 65bce27..7bb7c57 100644 --- a/axiom/logging.py +++ b/axiom/logging.py @@ -15,7 +15,7 @@ class AxiomHandler(Handler): interval: int last_run: float - def __init__(self, client: Client, dataset: str, level=NOTSET, interval=3): + def __init__(self, client: Client, dataset: str, level=NOTSET, interval=1): Handler.__init__(self, level) # set urllib3 logging level to warning, check: # https://github.com/axiomhq/axiom-py/issues/23 @@ -24,7 +24,7 @@ def __init__(self, client: Client, dataset: str, level=NOTSET, interval=3): getLogger("urllib3").setLevel(WARNING) self.client = client self.dataset = dataset - self.logcache = [] + self.buffer = [] self.last_run = time.time() self.interval = interval @@ -33,15 +33,12 @@ def __init__(self, client: Client, dataset: str, level=NOTSET, interval=3): def emit(self, record): """emit sends a log to Axiom.""" - self.logcache.append(record) - if time.time() - self.last_run > self.interval: - self.client.ingest_events(self.dataset, self.logcache) - self.last_run = time.time() - self.logcache = [] - else: - return + self.buffer.append(record) + if len(self.buffer) >= 1000 or time.time() - self.last_run > self.interval: + self.flush() def flush(self): """flush sends all logs in the logcache to Axiom.""" - self.client.ingest_events(self.dataset, self.logcache) - self.logcache = [] + self.client.ingest_events(self.dataset, self.buffer) + self.buffer = [] + self.last_run = time.time() diff --git a/tests/test_logger.py b/tests/test_logger.py index 1eb2a55..e466a36 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -25,9 +25,21 @@ def test_log(self): axiom_handler = AxiomHandler(client, dataset_name) - root = logging.getLogger() - root.addHandler(axiom_handler) + logger = logging.getLogger() + logger.addHandler(axiom_handler) + + logger.warning("foo") + + # this log shouldn't be ingested yet + res = client.datasets.apl_query(dataset_name) + self.assertEqual(0, res.status.rowsExamined) + + # flush events + axiom_handler.flush() + + # this log shouldn't be ingested yet + res = client.datasets.apl_query(dataset_name) + self.assertEqual(1, res.status.rowsExamined) - root.warning("foo") # cleanup created dataset client.datasets.delete(dataset_name) From a480e6999dd7d52028c67f0eab564290fada277d Mon Sep 17 00:00:00 2001 From: Arne Bahlo Date: Thu, 24 Nov 2022 12:07:00 +0100 Subject: [PATCH 05/26] Add opts object --- tests/test_logger.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_logger.py b/tests/test_logger.py index e466a36..3713ca7 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -31,14 +31,14 @@ def test_log(self): logger.warning("foo") # this log shouldn't be ingested yet - res = client.datasets.apl_query(dataset_name) + res = client.datasets.apl_query(dataset_name, {}) self.assertEqual(0, res.status.rowsExamined) # flush events axiom_handler.flush() # this log shouldn't be ingested yet - res = client.datasets.apl_query(dataset_name) + res = client.datasets.apl_query(dataset_name, {}) self.assertEqual(1, res.status.rowsExamined) # cleanup created dataset From 4df1b4cb89107b4451b06df16ba4a12bd0e3d455 Mon Sep 17 00:00:00 2001 From: Arne Bahlo Date: Mon, 28 Nov 2022 11:55:18 +0100 Subject: [PATCH 06/26] Fix test_logger query calls --- tests/test_logger.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_logger.py b/tests/test_logger.py index 3713ca7..7b536eb 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -31,14 +31,14 @@ def test_log(self): logger.warning("foo") # this log shouldn't be ingested yet - res = client.datasets.apl_query(dataset_name, {}) + res = client.apl_query(dataset_name) self.assertEqual(0, res.status.rowsExamined) # flush events axiom_handler.flush() # this log shouldn't be ingested yet - res = client.datasets.apl_query(dataset_name, {}) + res = client.apl_query(dataset_name) self.assertEqual(1, res.status.rowsExamined) # cleanup created dataset From 9798c82711f0913c0bb7546d80aa87fa0f06de81 Mon Sep 17 00:00:00 2001 From: Arne Bahlo Date: Mon, 28 Nov 2022 11:58:01 +0100 Subject: [PATCH 07/26] Make query options optional --- axiom/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/axiom/client.py b/axiom/client.py index 08c436b..c65141d 100644 --- a/axiom/client.py +++ b/axiom/client.py @@ -241,11 +241,11 @@ def query_legacy( result.savedQueryID = query_id return result - def apl_query(self, apl: str, opts: Optional[AplOptions]) -> QueryResult: + def apl_query(self, apl: str, opts: Optional[AplOptions] = None) -> QueryResult: """Executes the given apl query on the dataset identified by its id.""" return self.query(apl, opts) - def query(self, apl: str, opts: Optional[AplOptions]) -> QueryResult: + def query(self, apl: str, opts: Optional[AplOptions] = None) -> QueryResult: """Executes the given apl query on the dataset identified by its id.""" path = "datasets/_apl" payload = ujson.dumps( From d9b7ab1a5e1b110142407bc9773114db86997e9c Mon Sep 17 00:00:00 2001 From: Arne Bahlo Date: Mon, 28 Nov 2022 12:12:46 +0100 Subject: [PATCH 08/26] Fix _prepare_apl_payload --- axiom/client.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/axiom/client.py b/axiom/client.py index c65141d..f08aaca 100644 --- a/axiom/client.py +++ b/axiom/client.py @@ -316,15 +316,13 @@ def _prepare_apl_payload( self, apl: str, opts: Optional[AplOptions] ) -> Dict[str, Any]: """Prepare the apl query options for the request.""" - if opts is None: - return {} - params = {} params["apl"] = apl - if opts.start_time: - params["startTime"] = opts.start_time - if opts.end_time: - params["endTime"] = opts.end_time + if opts is not None: + if opts.start_time: + params["startTime"] = opts.start_time + if opts.end_time: + params["endTime"] = opts.end_time return params From dcba29ef80bda11e032ab8c5caea3434ca59dce9 Mon Sep 17 00:00:00 2001 From: Arne Bahlo Date: Mon, 28 Nov 2022 13:05:11 +0100 Subject: [PATCH 09/26] Add required format parameter on apl ... even if options are None --- axiom/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/axiom/client.py b/axiom/client.py index f08aaca..fa7e72f 100644 --- a/axiom/client.py +++ b/axiom/client.py @@ -298,11 +298,12 @@ def _prepare_ingest_options(self, opts: Optional[IngestOptions]) -> Dict[str, An def _prepare_apl_options(self, opts: Optional[AplOptions]) -> Dict[str, Any]: """Prepare the apl query options for the request.""" + params = {} if opts is None: - return {} + params["format"] = AplResultFormat.Legacy + return params - params = {} if opts.no_cache: params["nocache"] = opts.no_cache.__str__() if opts.save: From ad3e78c50b84b3cef271ddb773d4a323cb6f9fdf Mon Sep 17 00:00:00 2001 From: Arne Bahlo Date: Mon, 28 Nov 2022 15:03:18 +0100 Subject: [PATCH 10/26] Fix default query result format --- axiom/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/axiom/client.py b/axiom/client.py index fa7e72f..4f1c072 100644 --- a/axiom/client.py +++ b/axiom/client.py @@ -301,7 +301,7 @@ def _prepare_apl_options(self, opts: Optional[AplOptions]) -> Dict[str, Any]: params = {} if opts is None: - params["format"] = AplResultFormat.Legacy + params["format"] = AplResultFormat.Legacy.value return params if opts.no_cache: From 44927d74dab89ca216e35decd30e0cdfcedfb211 Mon Sep 17 00:00:00 2001 From: Arne Bahlo Date: Mon, 28 Nov 2022 16:05:37 +0100 Subject: [PATCH 11/26] Store record.__dict__ in buffer --- axiom/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/axiom/logging.py b/axiom/logging.py index 7bb7c57..423d4e4 100644 --- a/axiom/logging.py +++ b/axiom/logging.py @@ -33,7 +33,7 @@ def __init__(self, client: Client, dataset: str, level=NOTSET, interval=1): def emit(self, record): """emit sends a log to Axiom.""" - self.buffer.append(record) + self.buffer.append(record.__dict__) if len(self.buffer) >= 1000 or time.time() - self.last_run > self.interval: self.flush() From 4b68c1bebb3f034b0dd769a3721926f31a3164ab Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Tue, 29 Nov 2022 14:20:55 +0200 Subject: [PATCH 12/26] create examples directory --- .github/workflows/ci.yml | 18 ++++++++++++++++-- axiom/client.py | 4 ++-- examples/README.md | 16 ++++++++++++++++ examples/create_dataset.py | 12 ++++++++++++ examples/ingest.py | 10 ++++++++++ examples/list_datasets.py | 11 +++++++++++ examples/main.py | 17 +++++++++++++++++ examples/pyproject.toml | 19 +++++++++++++++++++ examples/query.py | 13 +++++++++++++ examples/query_legacy.py | 22 ++++++++++++++++++++++ 10 files changed, 138 insertions(+), 4 deletions(-) create mode 100644 examples/README.md create mode 100644 examples/create_dataset.py create mode 100644 examples/ingest.py create mode 100644 examples/list_datasets.py create mode 100644 examples/main.py create mode 100644 examples/pyproject.toml create mode 100644 examples/query.py create mode 100644 examples/query_legacy.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f647dc2..048820d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,8 +25,22 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - run: pip install poetry==${{ env.POETRY_VERSION }} - run: poetry install - - run: poetry run black --check axiom tests - - run: poetry run pylint -E axiom tests + - run: poetry run black --check axiom tests examples + - run: poetry run pylint -E axiom tests examples + examples: + name: Run Examples + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ env.PYTHON_VERSION }} + - run: pip install poetry==${{ env.POETRY_VERSION }} + working-directory: ./examples + - run: poetry install + working-directory: ./examples + - run: poetry run python ./main.py + working-directory: ./examples test-dev: name: Test (Cloud Dev) runs-on: ubuntu-latest diff --git a/axiom/client.py b/axiom/client.py index 08c436b..0f67746 100644 --- a/axiom/client.py +++ b/axiom/client.py @@ -132,7 +132,7 @@ class Client: # pylint: disable=R0903 def __init__( self, - token: Optional[str], + token: Optional[str] = None, org_id: Optional[str] = None, url_base: Optional[str] = None, ): @@ -245,7 +245,7 @@ def apl_query(self, apl: str, opts: Optional[AplOptions]) -> QueryResult: """Executes the given apl query on the dataset identified by its id.""" return self.query(apl, opts) - def query(self, apl: str, opts: Optional[AplOptions]) -> QueryResult: + def query(self, apl: str, opts: Optional[AplOptions] = None) -> QueryResult: """Executes the given apl query on the dataset identified by its id.""" path = "datasets/_apl" payload = ujson.dumps( diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..f2a8928 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,16 @@ +# Examples + +## Usage + +```shell +export AXIOM_TOKEN="..." +python +``` + + +## Examples + +- [ingest.py](ingest.py) - Ingest events into Axiom +- [list_datasets.py](list_datasets.py) - Retrieve a list of all datasets +- [query.py](query.py) - Query a dataset as part of the request +- [query_legacy.py](query_legacy.py) - Query a dataset using the legacy query method diff --git a/examples/create_dataset.py b/examples/create_dataset.py new file mode 100644 index 0000000..6b5f95f --- /dev/null +++ b/examples/create_dataset.py @@ -0,0 +1,12 @@ +from axiom import Client, DatasetCreateRequest + + +def create_dataset(): + client = Client() + res = client.datasets.create( + DatasetCreateRequest(name="my-dataset", description="") + ) + print(f"created dataset: {res.id}") + + +create_dataset() diff --git a/examples/ingest.py b/examples/ingest.py new file mode 100644 index 0000000..cd688c2 --- /dev/null +++ b/examples/ingest.py @@ -0,0 +1,10 @@ +from axiom import Client + + +def ingest(): + client = Client() + res = client.ingest_events("my-dataset", [{"foo": "bar"}]) + print("Ingested %d events with %d failures".format(res.ingested, res.failed)) + + +ingest() diff --git a/examples/list_datasets.py b/examples/list_datasets.py new file mode 100644 index 0000000..1954acb --- /dev/null +++ b/examples/list_datasets.py @@ -0,0 +1,11 @@ +from axiom import Client + + +def list_datasets(): + client = Client() + res = client.datasets.get_list() + for dataset in res: + print(f"found dataset: {dataset.name}") + + +list_datasets() diff --git a/examples/main.py b/examples/main.py new file mode 100644 index 0000000..239b61b --- /dev/null +++ b/examples/main.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +from create_dataset import create_dataset +from ingest import ingest +from query import query +from query_legacy import queryLegacy + + +def main(): + create_dataset() + ingest() + query() + queryLegacy() + + +if __name__ == "__init__": + main() diff --git a/examples/pyproject.toml b/examples/pyproject.toml new file mode 100644 index 0000000..989e5cf --- /dev/null +++ b/examples/pyproject.toml @@ -0,0 +1,19 @@ +[tool.poetry] +name = "axiom-py-examples" +version = "0.1" +description = "Axiom API Python bindings examples." +authors = ["Axiom, Inc."] +license = "MIT" + +[tool.poetry.dependencies] +python = "^3.8" +axiom-py = { path = "../" } + + +[tool.poetry.dev-dependencies] +pylint = "^2.7.2" + + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/examples/query.py b/examples/query.py new file mode 100644 index 0000000..9787d2a --- /dev/null +++ b/examples/query.py @@ -0,0 +1,13 @@ +from axiom import Client + + +def query(): + aplQuery = "['my-dataset'] | where status == 500" + + client = Client() + res = client.query(aplQuery) + for match in res.matches: + print(match.data) + + +query() diff --git a/examples/query_legacy.py b/examples/query_legacy.py new file mode 100644 index 0000000..38e9db4 --- /dev/null +++ b/examples/query_legacy.py @@ -0,0 +1,22 @@ +from axiom import Client, QueryLegacy, QueryOptions, QueryKind +from datetime import datetime, timedelta + + +def queryLegacy(): + endTime = datetime.now() + startTime = endTime - timedelta(days=1) + query = QueryLegacy(startTime=startTime, endTime=endTime) + + client = Client() + res = client.query_legacy( + "my-dataset", query, QueryOptions(saveAsKind=QueryKind.ANALYTICS) + ) + if res.matches is None or len(res.matches) == 0: + print("No matches found") + return + + for match in res.matches: + print(match.data) + + +queryLegacy() From 7f1927cdb452e34d3209bf2d6897c25cca8abc8f Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Tue, 29 Nov 2022 15:38:35 +0200 Subject: [PATCH 13/26] add delete_dataset example --- examples/delete_dataset.py | 10 ++++++++++ examples/main.py | 2 ++ 2 files changed, 12 insertions(+) create mode 100644 examples/delete_dataset.py diff --git a/examples/delete_dataset.py b/examples/delete_dataset.py new file mode 100644 index 0000000..2ca2478 --- /dev/null +++ b/examples/delete_dataset.py @@ -0,0 +1,10 @@ +from axiom import Client, DatasetCreateRequest + + +def delete_dataset(): + client = Client() + client.datasets.delete("my-dataset") + print(f"deleted dataset: my-dateset") + + +delete_dataset() diff --git a/examples/main.py b/examples/main.py index 239b61b..e95c1ae 100644 --- a/examples/main.py +++ b/examples/main.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from create_dataset import create_dataset +from delete_dataset import delete_dataset from ingest import ingest from query import query from query_legacy import queryLegacy @@ -11,6 +12,7 @@ def main(): ingest() query() queryLegacy() + delete_dataset() if __name__ == "__init__": From 7b3bf7f5975afe3107845d11708c69ac91de91bf Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Tue, 29 Nov 2022 15:44:10 +0200 Subject: [PATCH 14/26] add some comments to main examples --- examples/create_dataset.py | 7 ++----- examples/delete_dataset.py | 7 ++----- examples/ingest.py | 7 ++----- examples/list_datasets.py | 3 --- examples/main.py | 17 ++++++++++++----- examples/query.py | 7 ++----- examples/query_legacy.py | 7 ++----- 7 files changed, 22 insertions(+), 33 deletions(-) diff --git a/examples/create_dataset.py b/examples/create_dataset.py index 6b5f95f..d5de57b 100644 --- a/examples/create_dataset.py +++ b/examples/create_dataset.py @@ -1,12 +1,9 @@ from axiom import Client, DatasetCreateRequest -def create_dataset(): +def create_dataset(dataset_name): client = Client() res = client.datasets.create( - DatasetCreateRequest(name="my-dataset", description="") + DatasetCreateRequest(name=dataset_name, description="") ) print(f"created dataset: {res.id}") - - -create_dataset() diff --git a/examples/delete_dataset.py b/examples/delete_dataset.py index 2ca2478..ab5d6e9 100644 --- a/examples/delete_dataset.py +++ b/examples/delete_dataset.py @@ -1,10 +1,7 @@ from axiom import Client, DatasetCreateRequest -def delete_dataset(): +def delete_dataset(dataset_name): client = Client() - client.datasets.delete("my-dataset") + client.datasets.delete(dataset_name) print(f"deleted dataset: my-dateset") - - -delete_dataset() diff --git a/examples/ingest.py b/examples/ingest.py index cd688c2..8d133ab 100644 --- a/examples/ingest.py +++ b/examples/ingest.py @@ -1,10 +1,7 @@ from axiom import Client -def ingest(): +def ingest(dataset_name): client = Client() - res = client.ingest_events("my-dataset", [{"foo": "bar"}]) + res = client.ingest_events(dataset_name, [{"foo": "bar"}]) print("Ingested %d events with %d failures".format(res.ingested, res.failed)) - - -ingest() diff --git a/examples/list_datasets.py b/examples/list_datasets.py index 1954acb..47fdc8d 100644 --- a/examples/list_datasets.py +++ b/examples/list_datasets.py @@ -6,6 +6,3 @@ def list_datasets(): res = client.datasets.get_list() for dataset in res: print(f"found dataset: {dataset.name}") - - -list_datasets() diff --git a/examples/main.py b/examples/main.py index e95c1ae..6651f57 100644 --- a/examples/main.py +++ b/examples/main.py @@ -2,17 +2,24 @@ from create_dataset import create_dataset from delete_dataset import delete_dataset +from list_datasets import list_datasets from ingest import ingest from query import query from query_legacy import queryLegacy def main(): - create_dataset() - ingest() - query() - queryLegacy() - delete_dataset() + dataset_name = "my-dataset" + # create a new dataset + create_dataset(dataset_name) + list_datasets() + # ingest some data + ingest(dataset_name) + # query the ingested data + query(dataset_name) + queryLegacy(dataset_name) + # finally, delete the dataset + delete_dataset(dataset_name) if __name__ == "__init__": diff --git a/examples/query.py b/examples/query.py index 9787d2a..a266a8d 100644 --- a/examples/query.py +++ b/examples/query.py @@ -1,13 +1,10 @@ from axiom import Client -def query(): - aplQuery = "['my-dataset'] | where status == 500" +def query(dataset_name): + aplQuery = f"['{dataset_name}'] | where status == 500" client = Client() res = client.query(aplQuery) for match in res.matches: print(match.data) - - -query() diff --git a/examples/query_legacy.py b/examples/query_legacy.py index 38e9db4..f89c721 100644 --- a/examples/query_legacy.py +++ b/examples/query_legacy.py @@ -2,14 +2,14 @@ from datetime import datetime, timedelta -def queryLegacy(): +def queryLegacy(dataset_name): endTime = datetime.now() startTime = endTime - timedelta(days=1) query = QueryLegacy(startTime=startTime, endTime=endTime) client = Client() res = client.query_legacy( - "my-dataset", query, QueryOptions(saveAsKind=QueryKind.ANALYTICS) + dataset_name, query, QueryOptions(saveAsKind=QueryKind.ANALYTICS) ) if res.matches is None or len(res.matches) == 0: print("No matches found") @@ -17,6 +17,3 @@ def queryLegacy(): for match in res.matches: print(match.data) - - -queryLegacy() From 2d60818819c6f96a2d0dc488ed06e636b29f65d9 Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Tue, 29 Nov 2022 15:57:36 +0200 Subject: [PATCH 15/26] fix __name__ in main.py --- examples/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/main.py b/examples/main.py index 6651f57..6e551a4 100644 --- a/examples/main.py +++ b/examples/main.py @@ -22,5 +22,5 @@ def main(): delete_dataset(dataset_name) -if __name__ == "__init__": +if __name__ == "__main__": main() From 923441798b4796d5e14ad1ac95250a5471542f4c Mon Sep 17 00:00:00 2001 From: Arne Bahlo Date: Tue, 29 Nov 2022 15:55:42 +0100 Subject: [PATCH 16/26] Fix test comment --- tests/test_logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_logger.py b/tests/test_logger.py index 7b536eb..68eed92 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -37,7 +37,7 @@ def test_log(self): # flush events axiom_handler.flush() - # this log shouldn't be ingested yet + # now we should have a log res = client.apl_query(dataset_name) self.assertEqual(1, res.status.rowsExamined) From 217ebe61bd4e1550b9da9dbe9c784557edf51484 Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Wed, 30 Nov 2022 12:57:43 +0200 Subject: [PATCH 17/26] remove examples from the CI actions --- .github/workflows/ci.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 048820d..2648f4b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,20 +27,6 @@ jobs: - run: poetry install - run: poetry run black --check axiom tests examples - run: poetry run pylint -E axiom tests examples - examples: - name: Run Examples - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: ${{ env.PYTHON_VERSION }} - - run: pip install poetry==${{ env.POETRY_VERSION }} - working-directory: ./examples - - run: poetry install - working-directory: ./examples - - run: poetry run python ./main.py - working-directory: ./examples test-dev: name: Test (Cloud Dev) runs-on: ubuntu-latest From ad20188facf9c56214d4f0d6d86d4b8f1b8431fa Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Wed, 30 Nov 2022 13:12:38 +0200 Subject: [PATCH 18/26] add missing examples to the readme --- examples/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index f2a8928..ec14f59 100644 --- a/examples/README.md +++ b/examples/README.md @@ -11,6 +11,9 @@ python ## Examples - [ingest.py](ingest.py) - Ingest events into Axiom -- [list_datasets.py](list_datasets.py) - Retrieve a list of all datasets - [query.py](query.py) - Query a dataset as part of the request - [query_legacy.py](query_legacy.py) - Query a dataset using the legacy query method +- [create_dateset.py](create_dataset.py) - Create a new dataset +- [list_datasets.py](list_datasets.py) - Retrieve a list of all datasets +- [delete_dataset.py](delete_dataset.py) - Delete a dataset +- From 97697eef0f1fc8c79d542243c2a1c6ed2eeb6895 Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Wed, 30 Nov 2022 13:13:14 +0200 Subject: [PATCH 19/26] mention the examples dir in the main README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 94a15fd..5336935 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,8 @@ client.datasets.ingest_events( client.datasets.query(r"['my-dataset'] | where foo == 'bar' | limit 100") ``` +for more examples, check out the [examples](examples) directory. + ## Contributing This project uses [Poetry](https://python-poetry.org) for dependecy management From 6e8652e6ace93badcd0706d9b73de294b37f88fb Mon Sep 17 00:00:00 2001 From: Lukas Malkmus Date: Tue, 6 Dec 2022 10:26:49 +0100 Subject: [PATCH 20/26] Add community Slack to README --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5336935..d4ed4d8 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,9 @@ - **Query everything, all the time:** Whether DevOps, SecOps, or EverythingOps, query all your data no matter its age. No provisioning, no moving data from cold/archive to “hot”, and no worrying about slow queries. All your data, all. the. time. - **Powerful dashboards, for continuous observability:** Build dashboards to collect related queries and present information that’s quick and easy to digest for you and your team. Dashboards can be kept private or shared with others, and are the perfect way to bring together data from different sources -For more information check out the [official documentation](https://axiom.co/docs). +For more information check out the [official documentation](https://axiom.co/docs) +and our +[community Slack](https://axiomfm.slack.com/join/shared_invite/zt-w7d1vepe-L0upiOL6n6MXfjr33sCBUQ). ## Quickstart @@ -87,4 +89,4 @@ Distributed under MIT License (`The MIT License`). [ci_badge]: https://github.com/axiomhq/axiom-py/actions/workflows/ci.yml/badge.svg [pypi]: https://pypi.org/project/axiom-py/ [pypi_badge]: https://img.shields.io/pypi/v/axiom-py.svg -[version_badge]: https://img.shields.io/pypi/pyversions/axiom-py.svg \ No newline at end of file +[version_badge]: https://img.shields.io/pypi/pyversions/axiom-py.svg From 20510f910d45f621b490cc9b8ff15d4d470d76d2 Mon Sep 17 00:00:00 2001 From: Tyler Thompson Date: Tue, 3 Jan 2023 15:21:34 +0000 Subject: [PATCH 21/26] Use shields.io badges instead of GitHub badges --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d4ed4d8..b1bdf25 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ Distributed under MIT License (`The MIT License`). [ci]: https://github.com/axiomhq/axiom-py/actions/workflows/ci.yml -[ci_badge]: https://github.com/axiomhq/axiom-py/actions/workflows/ci.yml/badge.svg +[ci_badge]: https://img.shields.io/github/actions/workflow/status/axiomhq/axiom-py/ci.yml?branch=main&ghcache=unused [pypi]: https://pypi.org/project/axiom-py/ [pypi_badge]: https://img.shields.io/pypi/v/axiom-py.svg [version_badge]: https://img.shields.io/pypi/pyversions/axiom-py.svg From 515f41f5d9eeb4e32afd12aaff86043a3ae576cb Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Thu, 5 Jan 2023 13:22:49 +0300 Subject: [PATCH 22/26] fix profile url in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b1bdf25..e30b18a 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ pip3 install axiom-py If you use the [Axiom CLI](https://github.com/axiomhq/cli), run `eval $(axiom config export -f)` to configure your environment variables. -Otherwise create a personal token in [the Axiom settings](https://cloud.axiom.co/settings/profile) and export it as `AXIOM_TOKEN`. Set `AXIOM_ORG_ID` to the organization ID from the settings page of the organization you want to access. +Otherwise create a personal token in [the Axiom settings](https://app.axiom.co/profile) and export it as `AXIOM_TOKEN`. Set `AXIOM_ORG_ID` to the organization ID from the settings page of the organization you want to access. You can also configure the client using options passed to the client constructor: From f0f1b3c562e99ebdc66bbda1749ea46dc7d48547 Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Mon, 9 Jan 2023 13:45:34 +0300 Subject: [PATCH 23/26] revet axiom app changes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e30b18a..4b85e09 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ pip3 install axiom-py If you use the [Axiom CLI](https://github.com/axiomhq/cli), run `eval $(axiom config export -f)` to configure your environment variables. -Otherwise create a personal token in [the Axiom settings](https://app.axiom.co/profile) and export it as `AXIOM_TOKEN`. Set `AXIOM_ORG_ID` to the organization ID from the settings page of the organization you want to access. +Otherwise create a personal token in [the Axiom settings](https://cloud.axiom.co/profile) and export it as `AXIOM_TOKEN`. Set `AXIOM_ORG_ID` to the organization ID from the settings page of the organization you want to access. You can also configure the client using options passed to the client constructor: From 93dc3ff9d625b2f9f993a0c342d04dbb336bd2c7 Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Wed, 11 Jan 2023 14:07:21 +0300 Subject: [PATCH 24/26] use new organisation secrets --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2648f4b..0a6dbbe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,9 +41,9 @@ jobs: - run: poetry install - run: poetry run python -m pytest env: - AXIOM_URL: ${{ secrets.TESTING_CLOUD_DEV_DEPLOYMENT_URL }} - AXIOM_TOKEN: ${{ secrets.TESTING_CLOUD_DEV_ACCESS_TOKEN }} - AXIOM_ORG_ID: ${{ secrets.TESTING_CLOUD_DEV_ORG_ID }} + AXIOM_URL: ${{ secrets.TESTING_DEV_APP_URL }} + AXIOM_TOKEN: ${{ secrets.TESTING_DEV_TOKEN }} + AXIOM_ORG_ID: ${{ secrets.TESTING_DEV_ORG_ID }} publish: name: Publish on PyPi runs-on: ubuntu-latest From 1f405f58cd1b5f153f3adb2b93182d46c1479141 Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Wed, 11 Jan 2023 15:47:13 +0300 Subject: [PATCH 25/26] switch to api.axiom.co --- .github/workflows/ci.yml | 2 +- axiom/client.py | 6 +++--- tests/test_client.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a6dbbe..4634c66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: - run: poetry install - run: poetry run python -m pytest env: - AXIOM_URL: ${{ secrets.TESTING_DEV_APP_URL }} + AXIOM_URL: ${{ secrets.TESTING_DEV_API_URL }} AXIOM_TOKEN: ${{ secrets.TESTING_DEV_TOKEN }} AXIOM_ORG_ID: ${{ secrets.TESTING_DEV_ORG_ID }} publish: diff --git a/axiom/client.py b/axiom/client.py index c6e690d..8bc1553 100644 --- a/axiom/client.py +++ b/axiom/client.py @@ -20,7 +20,7 @@ from .__init__ import __version__ -AXIOM_URL = "https://cloud.axiom.co" +AXIOM_URL = "https://api.axiom.co" @dataclass @@ -143,8 +143,8 @@ def __init__( org_id = os.getenv("AXIOM_ORG_ID") if url_base is None: url_base = AXIOM_URL - # Append /api/v1 to the url_base - url_base = url_base.rstrip("/") + "/api/v1/" + # Append /v1 to the url_base + url_base = url_base.rstrip("/") + "/v1/" self.logger = getLogger() self.session = BaseUrlSession(url_base) diff --git a/tests/test_client.py b/tests/test_client.py index d2adaaf..0ca07db 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -83,7 +83,7 @@ def setUpClass(cls): @responses.activate def test_retries(self): - url = os.getenv("AXIOM_URL") + "/api/v1/datasets/test" + url = os.getenv("AXIOM_URL") + "/v1/datasets/test" responses.add(responses.GET, url, status=500) responses.add(responses.GET, url, status=502) responses.add( From 558feaca9fa19640dc3afc5a9b57bc65a936ad86 Mon Sep 17 00:00:00 2001 From: Islam Shehata Date: Thu, 26 Jan 2023 15:50:15 +0200 Subject: [PATCH 26/26] prepare release for v0.1.0 --- README.md | 4 ++-- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4b85e09..ff648f9 100644 --- a/README.md +++ b/README.md @@ -60,13 +60,13 @@ client = axiom.Client() time = datetime.utcnow() - timedelta(hours=1) time_formatted = rfc3339.format(time) -client.datasets.ingest_events( +client.ingest_events( dataset="my-dataset", events=[ {"foo": "bar", "_time": time_formatted}, {"bar": "baz", "_time": time_formatted}, ]) -client.datasets.query(r"['my-dataset'] | where foo == 'bar' | limit 100") +client.query(r"['my-dataset'] | where foo == 'bar' | limit 100") ``` for more examples, check out the [examples](examples) directory. diff --git a/pyproject.toml b/pyproject.toml index 20d0236..3da889c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "axiom-py" -version = "0.1.0-beta.5" +version = "0.1.0" description = "Axiom API Python bindings." authors = ["Axiom, Inc."] license = "MIT"