Moved benchmarks folder.

Signed-off-by: dblock <dblock@amazon.com>
opensearch-project · Oct 13, 2023 · e2ccdd0 · e2ccdd0
1 parent fcfd6a4
commit e2ccdd0
Show file tree

Hide file tree

Showing 12 changed files with 71 additions and 110 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added point-in-time APIs (create_pit, delete_pit, delete_all_pits, get_all_pits) and Security Client APIs (health and update_audit_configuration) ([#502](https://github.com/opensearch-project/opensearch-py/pull/502))
 - Added new guide for using index templates with the client ([#531](https://github.com/opensearch-project/opensearch-py/pull/531))
 - Added `pool_maxsize` for `Urllib3HttpConnection` ([#535](https://github.com/opensearch-project/opensearch-py/pull/535))
+- Added benchmarks ([#537](https://github.com/opensearch-project/opensearch-py/pull/537))
 ### Changed
 - Generate `tasks` client from API specs ([#508](https://github.com/opensearch-project/opensearch-py/pull/508))
 - Generate `ingest` client from API specs ([#513](https://github.com/opensearch-project/opensearch-py/pull/513))

diff --git a/README.md b/README.md
@@ -26,7 +26,7 @@ For more information, see [opensearch.org](https://opensearch.org/) and the [API
 
 ## User Guide
 
-To get started with the OpenSearch Python Client, see [User Guide](https://github.com/opensearch-project/opensearch-py/blob/main/USER_GUIDE.md).
+To get started with the OpenSearch Python Client, see [User Guide](https://github.com/opensearch-project/opensearch-py/blob/main/USER_GUIDE.md). This repository also contains [working samples](samples) and [benchmarks](benchmarks).
 
 ## Compatibility with OpenSearch
 

diff --git a/benchmarks/README.md b/benchmarks/README.md
@@ -0,0 +1,55 @@
+- [Benchmarks](#benchmarks)
+  - [Start OpenSearch](#start-opensearch)
+  - [Install Prerequisites](#install-prerequisites)
+  - [Run Benchmarks](#run-benchmarks)
+
+## Benchmarks
+
+Python client benchmarks using [richbench](https://github.com/tonybaloney/rich-bench).
+
+### Start OpenSearch
+
+```
+docker run -p 9200:9200 -e "discovery.type=single-node" opensearchproject/opensearch:latest
+```
+
+### Install Prerequisites
+
+Install [poetry](https://python-poetry.org/docs/), then install package dependencies.
+
+```
+poetry install
+```
+
+Benchmarks use the code in this repository by specifying the dependency as `opensearch-py = { path = "..", develop=true, extras=["async"] }` in [pyproject.toml](pyproject.toml).
+
+### Run Benchmarks
+
+Run all benchmarks available as follows.
+
+```
+$ poetry run richbench . --repeat 1 --times 1
+
+                                             Benchmarks, repeat=1, number=1                                              
+┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
+┃                         Benchmark ┃ Min     ┃ Max     ┃ Mean    ┃ Min (+)         ┃ Max (+)         ┃ Mean (+)        ┃
+┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
+│ 1 client vs. more clients (async) │ 1.640   │ 1.640   │ 1.640   │ 1.102 (1.5x)    │ 1.102 (1.5x)    │ 1.102 (1.5x)    │
+│    1 thread vs. 32 threads (sync) │ 5.526   │ 5.526   │ 5.526   │ 1.626 (3.4x)    │ 1.626 (3.4x)    │ 1.626 (3.4x)    │
+│    1 thread vs. 32 threads (sync) │ 4.639   │ 4.639   │ 4.639   │ 3.363 (1.4x)    │ 3.363 (1.4x)    │ 3.363 (1.4x)    │
+│                sync vs. async (8) │ 3.198   │ 3.198   │ 3.198   │ 0.966 (3.3x)    │ 0.966 (3.3x)    │ 0.966 (3.3x)    │
+└───────────────────────────────────┴─────────┴─────────┴─────────┴─────────────────┴─────────────────┴─────────────────┘
+```
+
+Run a specific benchmark, e.g. [bench_sync.py](bench_sync.py) by specifying `--benchmark [name]`.
+
+```
+$ poetry run richbench . --repeat 1 --times 1 --benchmark sync
+
+                                            Benchmarks, repeat=1, number=1                                            
+┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
+┃                      Benchmark ┃ Min     ┃ Max     ┃ Mean    ┃ Min (+)         ┃ Max (+)         ┃ Mean (+)        ┃
+┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
+│ 1 thread vs. 32 threads (sync) │ 6.804   │ 6.804   │ 6.804   │ 3.409 (2.0x)    │ 3.409 (2.0x)    │ 3.409 (2.0x)    │
+└────────────────────────────────┴─────────┴─────────┴─────────┴─────────────────┴─────────────────┴─────────────────┘
+```
diff --git a/samples/benchmarks/bench_async.py → benchmarks/bench_async.py b/samples/benchmarks/bench_async.py → benchmarks/bench_async.py
@@ -7,11 +7,9 @@
 # compatible open source license.
 
 import asyncio
-import math
 import uuid
-from threading import Thread
 
-from opensearchpy import AsyncHttpConnection, AsyncOpenSearch, OpenSearch
+from opensearchpy import AsyncHttpConnection, AsyncOpenSearch
 
 host = "localhost"
 port = 9200

diff --git a/samples/benchmarks/bench_info_sync.py → benchmarks/bench_info_sync.py b/samples/benchmarks/bench_info_sync.py → benchmarks/bench_info_sync.py
@@ -6,6 +6,8 @@
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
 
+import logging
+import sys
 import time
 
 from thread_with_return_value import ThreadWithReturnValue
@@ -17,18 +19,16 @@
 auth = ("admin", "admin")
 request_count = 250
 
-# import logging
-# import sys
 
-# root = logging.getLogger()
-# # root.setLevel(logging.DEBUG)
-# # logging.getLogger("urllib3.connectionpool").setLevel(logging.DEBUG)
+root = logging.getLogger()
+# root.setLevel(logging.DEBUG)
+# logging.getLogger("urllib3.connectionpool").setLevel(logging.DEBUG)
 
-# handler = logging.StreamHandler(sys.stdout)
-# handler.setLevel(logging.DEBUG)
-# formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-# handler.setFormatter(formatter)
-# root.addHandler(handler)
+handler = logging.StreamHandler(sys.stdout)
+handler.setLevel(logging.DEBUG)
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+handler.setFormatter(formatter)
+root.addHandler(handler)
 
 
 def get_info(client, request_count):

diff --git a/samples/benchmarks/bench_sync.py → benchmarks/bench_sync.py b/samples/benchmarks/bench_sync.py → benchmarks/bench_sync.py
diff --git a/samples/benchmarks/bench_sync_async.py → benchmarks/bench_sync_async.py b/samples/benchmarks/bench_sync_async.py → benchmarks/bench_sync_async.py
diff --git a/samples/benchmarks/poetry.lock → benchmarks/poetry.lock b/samples/benchmarks/poetry.lock → benchmarks/poetry.lock
diff --git a/samples/benchmarks/poetry.toml → benchmarks/poetry.toml b/samples/benchmarks/poetry.toml → benchmarks/poetry.toml
diff --git a/samples/benchmarks/pyproject.toml → benchmarks/pyproject.toml b/samples/benchmarks/pyproject.toml → benchmarks/pyproject.toml
@@ -8,9 +8,8 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.7"
-opensearch-py = { path = "../..", develop=true, extras=["async"] }
+opensearch-py = { path = "..", develop=true, extras=["async"] }
 richbench = "*"
-orjson = "*"
 
 [build-system]
 requires = ["poetry-core"]

diff --git a/...es/benchmarks/thread_with_return_value.py → benchmarks/thread_with_return_value.py b/...es/benchmarks/thread_with_return_value.py → benchmarks/thread_with_return_value.py
diff --git a/samples/benchmarks/README.md b/samples/benchmarks/README.md