From 4152b4d8f511dd926d0e31e9dc03d5be5e7fa083 Mon Sep 17 00:00:00 2001 From: Sai Medhini Reddy Maryada <117196660+saimedhi@users.noreply.github.com> Date: Fri, 22 Mar 2024 04:53:28 -0700 Subject: [PATCH] Modified generator to generate plugins (#700) Signed-off-by: saimedhi Signed-off-by: AbitraryYu --- CHANGELOG.md | 1 + opensearchpy/_async/client/__init__.py | 5 ++ opensearchpy/_async/client/indices.py | 3 + opensearchpy/client/__init__.py | 5 ++ opensearchpy/client/indices.py | 3 + opensearchpy/plugins/__init__.py | 1 - utils/generate_api.py | 86 +++++++++++++++++++++++--- utils/license_headers.py | 2 +- 8 files changed, 94 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 148914284..287d65cf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added GHA release ([#614](https://github.com/opensearch-project/opensearch-py/pull/614)) - Incorporated API generation into CI workflow and fixed 'generate' nox session ([#660](https://github.com/opensearch-project/opensearch-py/pull/660)) - Added an automated api update bot for opensearch-py ([#664](https://github.com/opensearch-project/opensearch-py/pull/664)) +- Enhance generator to generate plugins ([#700](https://github.com/opensearch-project/opensearch-py/pull/700)) - Enhance generator to update changelog only if generated code differs from existing ([#684](https://github.com/opensearch-project/opensearch-py/pull/684)) - Added guide for configuring ssl_assert_hostname ([#694](https://github.com/opensearch-project/opensearch-py/pull/694)) ### Changed diff --git a/opensearchpy/_async/client/__init__.py b/opensearchpy/_async/client/__init__.py index 8a0af3f0b..bf549977c 100644 --- a/opensearchpy/_async/client/__init__.py +++ b/opensearchpy/_async/client/__init__.py @@ -1603,6 +1603,7 @@ async def scroll( "from_", "ignore_throttled", "ignore_unavailable", + "include_named_queries_score", "lenient", "max_concurrent_shard_requests", "pre_filter_shard_size", @@ -1683,6 +1684,10 @@ async def search( aliased indices should be ignored when throttled. :arg ignore_unavailable: Whether specified concrete indices should be ignored when unavailable (missing or closed). + :arg include_named_queries_score: Indicates whether + hit.matched_queries should be rendered as a map that includes the name + of the matched query associated with its score (true) or as an array + containing the name of the matched queries (false) Default is false. :arg lenient: Specify whether format-based query failures (such as providing text to a numeric field) should be ignored. :arg max_concurrent_shard_requests: The number of concurrent diff --git a/opensearchpy/_async/client/indices.py b/opensearchpy/_async/client/indices.py index cf86b46fa..fc1111d10 100644 --- a/opensearchpy/_async/client/indices.py +++ b/opensearchpy/_async/client/indices.py @@ -1260,6 +1260,7 @@ async def shard_stores( "ignore_unavailable", "max_num_segments", "only_expunge_deletes", + "primary_only", "wait_for_completion", ) async def forcemerge( @@ -1288,6 +1289,8 @@ async def forcemerge( be merged into (default: dynamic). :arg only_expunge_deletes: Specify whether the operation should only expunge deleted documents. + :arg primary_only: Specify whether the operation should only + perform on primary shards. Defaults to false. Default is false. :arg wait_for_completion: Should this request wait until the operation has completed before returning. Default is True. """ diff --git a/opensearchpy/client/__init__.py b/opensearchpy/client/__init__.py index f9c82c45a..8a0f42ed7 100644 --- a/opensearchpy/client/__init__.py +++ b/opensearchpy/client/__init__.py @@ -1603,6 +1603,7 @@ def scroll( "from_", "ignore_throttled", "ignore_unavailable", + "include_named_queries_score", "lenient", "max_concurrent_shard_requests", "pre_filter_shard_size", @@ -1683,6 +1684,10 @@ def search( aliased indices should be ignored when throttled. :arg ignore_unavailable: Whether specified concrete indices should be ignored when unavailable (missing or closed). + :arg include_named_queries_score: Indicates whether + hit.matched_queries should be rendered as a map that includes the name + of the matched query associated with its score (true) or as an array + containing the name of the matched queries (false) Default is false. :arg lenient: Specify whether format-based query failures (such as providing text to a numeric field) should be ignored. :arg max_concurrent_shard_requests: The number of concurrent diff --git a/opensearchpy/client/indices.py b/opensearchpy/client/indices.py index ef2eba819..179e2f5fc 100644 --- a/opensearchpy/client/indices.py +++ b/opensearchpy/client/indices.py @@ -1260,6 +1260,7 @@ def shard_stores( "ignore_unavailable", "max_num_segments", "only_expunge_deletes", + "primary_only", "wait_for_completion", ) def forcemerge( @@ -1288,6 +1289,8 @@ def forcemerge( be merged into (default: dynamic). :arg only_expunge_deletes: Specify whether the operation should only expunge deleted documents. + :arg primary_only: Specify whether the operation should only + perform on primary shards. Defaults to false. Default is false. :arg wait_for_completion: Should this request wait until the operation has completed before returning. Default is True. """ diff --git a/opensearchpy/plugins/__init__.py b/opensearchpy/plugins/__init__.py index 2f42da793..6c0097cd1 100644 --- a/opensearchpy/plugins/__init__.py +++ b/opensearchpy/plugins/__init__.py @@ -6,4 +6,3 @@ # # Modifications Copyright OpenSearch Contributors. See # GitHub history for details. -# diff --git a/utils/generate_api.py b/utils/generate_api.py index 76f6353a8..26ea90c97 100644 --- a/utils/generate_api.py +++ b/utils/generate_api.py @@ -101,9 +101,10 @@ def is_valid_url(url: str) -> bool: class Module: - def __init__(self, namespace: str) -> None: + def __init__(self, namespace: str, is_plugin: bool) -> None: self.namespace: Any = namespace self._apis: Any = [] + self.is_plugin: bool = is_plugin self.parse_orig() def add(self, api: Any) -> None: @@ -118,10 +119,17 @@ def parse_orig(self) -> None: reads the written module and updates with important code specific to this client """ self.orders = [] - self.header = "from typing import Any, Collection, Optional, Tuple, Union\n\n" + if self.is_plugin: + self.header = "from typing import Any\n\n" + else: + self.header = ( + "from typing import Any, Collection, Optional, Tuple, Union\n\n" + ) - namespace_new = "".join(word.capitalize() for word in self.namespace.split("_")) - self.header += "class " + namespace_new + "Client(NamespacedClient):" + self.namespace_new = "".join( + word.capitalize() for word in self.namespace.split("_") + ) + self.header += "class " + self.namespace_new + "Client(NamespacedClient):" if os.path.exists(self.filepath): with open(self.filepath, encoding="utf-8") as file: content = file.read() @@ -164,7 +172,45 @@ def dump(self) -> None: writes the module out to disk """ self.sort() + if not os.path.exists(self.filepath): + # Imports added for new namespaces in appropriate files. + if self.is_plugin: + with open( + "opensearchpy/_async/client/plugins.py", "r+", encoding="utf-8" + ) as file: + content = file.read() + file_content = content.replace( + "super(PluginsClient, self).__init__(client)", + f"super(PluginsClient, self).__init__(client)\n self.{self.namespace} = {self.namespace_new}Client(client)", # pylint: disable=line-too-long + 1, + ) + new_file_content = file_content.replace( + "from .client import Client", + f"from ..plugins.{self.namespace} import {self.namespace_new}Client\nfrom .client import Client", # pylint: disable=line-too-long + 1, + ) + file.seek(0) + file.write(new_file_content) + file.truncate() + else: + with open( + "opensearchpy/_async/client/__init__.py", "r+", encoding="utf-8" + ) as file: + content = file.read() + file_content = content.replace( + "# namespaced clients for compatibility with API names", + f"# namespaced clients for compatibility with API names\n self.{self.namespace} = {self.namespace_new}Client(client)", # pylint: disable=line-too-long + 1, + ) + new_file_content = file_content.replace( + "from .utils import", + f"from .{self.namespace} import {self.namespace_new}Client\nfrom .utils import", # pylint: disable=line-too-long + 1, + ) + file.seek(0) + file.write(new_file_content) + file.truncate() # This code snippet adds headers to each generated module indicating # that the code is generated.The separator is the last line in the # "THIS CODE IS AUTOMATICALLY GENERATED" header. @@ -209,8 +255,14 @@ def dump(self) -> None: # Imports are temporarily removed from the header and are regenerated # later to ensure imports are updated after code generation. + utils = ".utils" + if self.is_plugin: + utils = "..client.utils" + self.header = "\n".join( - line for line in self.header.split("\n") if "from .utils import" not in line + line + for line in self.header.split("\n") + if "from " + utils + " import" not in line ) with open(self.filepath, "w", encoding="utf-8") as file: @@ -252,7 +304,7 @@ def dump(self) -> None: present_keywords = [keyword for keyword in keywords if keyword in content] if present_keywords: - utils_imports = "from .utils import" + utils_imports = "from " + utils + " import" result = f"{utils_imports} {', '.join(present_keywords)}" utils_imports = result file_content = content.replace("#replace_token#", utils_imports) @@ -265,7 +317,10 @@ def filepath(self) -> Any: """ :return: absolute path to the module """ - return CODE_ROOT / f"opensearchpy/_async/client/{self.namespace}.py" + if self.is_plugin: + return CODE_ROOT / f"opensearchpy/_async/plugins/{self.namespace}.py" + else: + return CODE_ROOT / f"opensearchpy/_async/client/{self.namespace}.py" class API: @@ -704,8 +759,12 @@ def read_modules() -> Any: api = apply_patch(namespace, name, api) + is_plugin = False + if "_plugins" in api["url"]["paths"][0]["path"] and namespace != "security": + is_plugin = True + if namespace not in modules: - modules[namespace] = Module(namespace) + modules[namespace] = Module(namespace, is_plugin) modules[namespace].add(API(namespace, name, api)) @@ -752,13 +811,20 @@ def dump_modules(modules: Any) -> None: todir="/opensearchpy/client/", additional_replacements=additional_replacements, ), + unasync.Rule( + fromdir="/opensearchpy/_async/plugins/", + todir="/opensearchpy/plugins/", + additional_replacements=additional_replacements, + ), ] filepaths = [] for root, _, filenames in os.walk(CODE_ROOT / "opensearchpy/_async"): for filename in filenames: - if filename.rpartition(".")[-1] in ("py",) and not filename.startswith( - "utils.py" + if filename.rpartition(".")[-1] in ("py",) and filename not in ( + "utils.py", + "index_management.py", + "alerting.py", ): filepaths.append(os.path.join(root, filename)) diff --git a/utils/license_headers.py b/utils/license_headers.py index 0405476e2..86582c9d4 100644 --- a/utils/license_headers.py +++ b/utils/license_headers.py @@ -79,7 +79,7 @@ def add_header_to_file(filepath: str) -> None: for i, line in enumerate(lines): if len(line) > 0 and line not in LINES_TO_KEEP: break - lines = lines[:i] + [LICENSE_HEADER] + lines[i:] + lines = lines[:i] + [LICENSE_HEADER + "\n\n"] + lines[i:] with open(filepath, mode="w", encoding="utf-8") as file: file.truncate() file.write("".join(lines))