From de5dd412ad561426ffb563023ae8c09e159d00a1 Mon Sep 17 00:00:00 2001 From: Niket Singh Date: Thu, 24 Nov 2022 22:45:41 +0530 Subject: [PATCH] feat: Added pool_maxsize for RequestsHttpConnection Signed-off-by: Niket Singh --- CHANGELOG.md | 1 + USER_GUIDE.md | 11 +++++++---- opensearchpy/connection/http_requests.py | 11 +++++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8bb731c..99272b46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased] ### Added - Added Point in time API rest API([#191](https://github.com/opensearch-project/opensearch-py/pull/191)) +- Added pool_maxsize for RequestsHttpConnection ([#216](https://github.com/opensearch-project/opensearch-py/pull/216)) - Github workflow for changelog verification ([#218](https://github.com/opensearch-project/opensearch-py/pull/218)) - Added overload decorators to helpers-actions.pyi-"bulk" ([#239](https://github.com/opensearch-project/opensearch-py/pull/239)) ### Changed diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 3d3a777f..35c5c21b 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -1,6 +1,6 @@ -- [Getting Started with the OpenSearch Python Client](#getting-started-with-the-opensearch-python-client) +- [User guide of OpenSearch Python Client](#user-guide-of-opensearch-python-client) - [Setup](#setup) - - [Sample code](#sample-code) + - [Example](#example) - [Creating a client](#creating-a-client) - [Creating an index](#creating-an-index) - [Adding a document to an index](#adding-a-document-to-an-index) @@ -10,7 +10,7 @@ - [Deleting a document](#deleting-a-document) - [Deleting an index](#deleting-an-index) - [Making API Calls](#making-api-calls) - - [Point in Time API](#point-in-time-api-calls) + - [Point in Time API](#point-in-time-api) - [Using plugins](#using-plugins) - [Alerting plugin](#alerting-plugin) - [**Searching for monitors**](#searching-for-monitors) @@ -384,6 +384,8 @@ Refer the AWS documentation regarding usage of IAM credentials to sign requests Opensearch-py client library also provides an in-house IAM based authentication feature, `AWSV4SignerAuth` that will help users to connect to their opensearch clusters by making use of IAM roles. +`AWSV4SignerAuth` uses RequestHttpConnection as transport class for communication with opensearch clusters. Opensearch-py client library provides `pool_maxsize` option to modify default connection-pool size. + #### Pre-requisites to use `AWSV4SignerAuth` - Python version 3.6 or above, - Install [botocore](https://pypi.org/project/botocore/) using pip @@ -407,7 +409,8 @@ client = OpenSearch( http_auth = auth, use_ssl = True, verify_certs = True, - connection_class = RequestsHttpConnection + connection_class = RequestsHttpConnection, + pool_maxsize = 20 ) q = 'miller' diff --git a/opensearchpy/connection/http_requests.py b/opensearchpy/connection/http_requests.py index a918a635..0ab835a7 100644 --- a/opensearchpy/connection/http_requests.py +++ b/opensearchpy/connection/http_requests.py @@ -66,6 +66,8 @@ class RequestsHttpConnection(Connection): :arg http_compress: Use gzip compression :arg opaque_id: Send this value in the 'X-Opaque-Id' HTTP header For tracing all requests made by this transport. + :arg pool_maxsize: Maximum connection pool size used by pool-manager + For custom connection-pooling on current session """ def __init__( @@ -82,6 +84,7 @@ def __init__( headers=None, http_compress=None, opaque_id=None, + pool_maxsize=None, **kwargs ): if not REQUESTS_AVAILABLE: @@ -94,6 +97,14 @@ def __init__( for key in list(self.session.headers): self.session.headers.pop(key) + # Mount http-adapter with custom connection-pool size. Default=10 + if pool_maxsize and isinstance(pool_maxsize, int): + pool_adapter = requests.adapters.HTTPAdapter( + pool_maxsize=pool_maxsize + ) + self.session.mount("http://", pool_adapter) + self.session.mount("https://", pool_adapter) + super(RequestsHttpConnection, self).__init__( host=host, port=port,