Modified generator to generate plugins (opensearch-project#700)

Signed-off-by: saimedhi <saimedhi@amazon.com>
Signed-off-by: AbitraryYu <nikkoyhc@gmail.com>

CHANGELOG.md

@@ -13,6 +13,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added GHA release ([#614](https://github.com/opensearch-project/opensearch-py/pull/614))
 - Incorporated API generation into CI workflow and fixed 'generate' nox session ([#660](https://github.com/opensearch-project/opensearch-py/pull/660))
 - Added an automated api update bot for opensearch-py ([#664](https://github.com/opensearch-project/opensearch-py/pull/664)) 
+- Enhance generator to generate plugins ([#700](https://github.com/opensearch-project/opensearch-py/pull/700))
 - Enhance generator to update changelog only if generated code differs from existing ([#684](https://github.com/opensearch-project/opensearch-py/pull/684))
 - Added guide for configuring ssl_assert_hostname ([#694](https://github.com/opensearch-project/opensearch-py/pull/694))
 ### Changed

opensearchpy/_async/client/__init__.py

@@ -1603,6 +1603,7 @@ async def scroll(
         "from_",
         "ignore_throttled",
         "ignore_unavailable",
+        "include_named_queries_score",
         "lenient",
         "max_concurrent_shard_requests",
         "pre_filter_shard_size",
@@ -1683,6 +1684,10 @@ async def search(
             aliased indices should be ignored when throttled.
         :arg ignore_unavailable: Whether specified concrete indices
             should be ignored when unavailable (missing or closed).
+        :arg include_named_queries_score: Indicates whether
+            hit.matched_queries should be rendered as a map that includes the name
+            of the matched query associated with its score (true) or as an array
+            containing the name of the matched queries (false) Default is false.
         :arg lenient: Specify whether format-based query failures (such
             as providing text to a numeric field) should be ignored.
         :arg max_concurrent_shard_requests: The number of concurrent

opensearchpy/_async/client/indices.py

@@ -1260,6 +1260,7 @@ async def shard_stores(
         "ignore_unavailable",
         "max_num_segments",
         "only_expunge_deletes",
+        "primary_only",
         "wait_for_completion",
     )
     async def forcemerge(
@@ -1288,6 +1289,8 @@ async def forcemerge(
             be merged into (default: dynamic).
         :arg only_expunge_deletes: Specify whether the operation should
             only expunge deleted documents.
+        :arg primary_only: Specify whether the operation should only
+            perform on primary shards. Defaults to false. Default is false.
         :arg wait_for_completion: Should this request wait until the
             operation has completed before returning. Default is True.
         """

opensearchpy/client/__init__.py

@@ -1603,6 +1603,7 @@ def scroll(
         "from_",
         "ignore_throttled",
         "ignore_unavailable",
+        "include_named_queries_score",
         "lenient",
         "max_concurrent_shard_requests",
         "pre_filter_shard_size",
@@ -1683,6 +1684,10 @@ def search(
             aliased indices should be ignored when throttled.
         :arg ignore_unavailable: Whether specified concrete indices
             should be ignored when unavailable (missing or closed).
+        :arg include_named_queries_score: Indicates whether
+            hit.matched_queries should be rendered as a map that includes the name
+            of the matched query associated with its score (true) or as an array
+            containing the name of the matched queries (false) Default is false.
         :arg lenient: Specify whether format-based query failures (such
             as providing text to a numeric field) should be ignored.
         :arg max_concurrent_shard_requests: The number of concurrent

opensearchpy/client/indices.py

@@ -1260,6 +1260,7 @@ def shard_stores(
         "ignore_unavailable",
         "max_num_segments",
         "only_expunge_deletes",
+        "primary_only",
         "wait_for_completion",
     )
     def forcemerge(
@@ -1288,6 +1289,8 @@ def forcemerge(
             be merged into (default: dynamic).
         :arg only_expunge_deletes: Specify whether the operation should
             only expunge deleted documents.
+        :arg primary_only: Specify whether the operation should only
+            perform on primary shards. Defaults to false. Default is false.
         :arg wait_for_completion: Should this request wait until the
             operation has completed before returning. Default is True.
         """

opensearchpy/plugins/__init__.py

@@ -6,4 +6,3 @@
 #
 # Modifications Copyright OpenSearch Contributors. See
 # GitHub history for details.
-#

utils/generate_api.py

@@ -101,9 +101,10 @@ def is_valid_url(url: str) -> bool:
 
 
 class Module:
-    def __init__(self, namespace: str) -> None:
+    def __init__(self, namespace: str, is_plugin: bool) -> None:
         self.namespace: Any = namespace
         self._apis: Any = []
+        self.is_plugin: bool = is_plugin
         self.parse_orig()
 
     def add(self, api: Any) -> None:
@@ -118,10 +119,17 @@ def parse_orig(self) -> None:
         reads the written module and updates with important code specific to this client
         """
         self.orders = []
-        self.header = "from typing import Any, Collection, Optional, Tuple, Union\n\n"
+        if self.is_plugin:
+            self.header = "from typing import Any\n\n"
+        else:
+            self.header = (
+                "from typing import Any, Collection, Optional, Tuple, Union\n\n"
+            )
 
-        namespace_new = "".join(word.capitalize() for word in self.namespace.split("_"))
-        self.header += "class " + namespace_new + "Client(NamespacedClient):"
+        self.namespace_new = "".join(
+            word.capitalize() for word in self.namespace.split("_")
+        )
+        self.header += "class " + self.namespace_new + "Client(NamespacedClient):"
         if os.path.exists(self.filepath):
             with open(self.filepath, encoding="utf-8") as file:
                 content = file.read()
@@ -164,7 +172,45 @@ def dump(self) -> None:
         writes the module out to disk
         """
         self.sort()
+        if not os.path.exists(self.filepath):
+            # Imports added for new namespaces in appropriate files.
+            if self.is_plugin:
+                with open(
+                    "opensearchpy/_async/client/plugins.py", "r+", encoding="utf-8"
+                ) as file:
+                    content = file.read()
+                    file_content = content.replace(
+                        "super(PluginsClient, self).__init__(client)",
+                        f"super(PluginsClient, self).__init__(client)\n        self.{self.namespace} = {self.namespace_new}Client(client)",  # pylint: disable=line-too-long
+                        1,
+                    )
+                    new_file_content = file_content.replace(
+                        "from .client import Client",
+                        f"from ..plugins.{self.namespace} import {self.namespace_new}Client\nfrom .client import Client",  # pylint: disable=line-too-long
+                        1,
+                    )
+                    file.seek(0)
+                    file.write(new_file_content)
+                    file.truncate()
 
+            else:
+                with open(
+                    "opensearchpy/_async/client/__init__.py", "r+", encoding="utf-8"
+                ) as file:
+                    content = file.read()
+                    file_content = content.replace(
+                        "# namespaced clients for compatibility with API names",
+                        f"# namespaced clients for compatibility with API names\n        self.{self.namespace} = {self.namespace_new}Client(client)",  # pylint: disable=line-too-long
+                        1,
+                    )
+                    new_file_content = file_content.replace(
+                        "from .utils import",
+                        f"from .{self.namespace} import {self.namespace_new}Client\nfrom .utils import",  # pylint: disable=line-too-long
+                        1,
+                    )
+                    file.seek(0)
+                    file.write(new_file_content)
+                    file.truncate()
         # This code snippet adds headers to each generated module indicating
         # that the code is generated.The separator is the last line in the
         # "THIS CODE IS AUTOMATICALLY GENERATED" header.
@@ -209,8 +255,14 @@ def dump(self) -> None:
 
         # Imports are temporarily removed from the header and are regenerated
         # later to ensure imports are updated after code generation.
+        utils = ".utils"
+        if self.is_plugin:
+            utils = "..client.utils"
+
         self.header = "\n".join(
-            line for line in self.header.split("\n") if "from .utils import" not in line
+            line
+            for line in self.header.split("\n")
+            if "from " + utils + " import" not in line
         )
 
         with open(self.filepath, "w", encoding="utf-8") as file:
@@ -252,7 +304,7 @@ def dump(self) -> None:
             present_keywords = [keyword for keyword in keywords if keyword in content]
 
             if present_keywords:
-                utils_imports = "from .utils import"
+                utils_imports = "from " + utils + " import"
                 result = f"{utils_imports} {', '.join(present_keywords)}"
                 utils_imports = result
             file_content = content.replace("#replace_token#", utils_imports)
@@ -265,7 +317,10 @@ def filepath(self) -> Any:
         """
         :return: absolute path to the module
         """
-        return CODE_ROOT / f"opensearchpy/_async/client/{self.namespace}.py"
+        if self.is_plugin:
+            return CODE_ROOT / f"opensearchpy/_async/plugins/{self.namespace}.py"
+        else:
+            return CODE_ROOT / f"opensearchpy/_async/client/{self.namespace}.py"
 
 
 class API:
@@ -704,8 +759,12 @@ def read_modules() -> Any:
 
         api = apply_patch(namespace, name, api)
 
+        is_plugin = False
+        if "_plugins" in api["url"]["paths"][0]["path"] and namespace != "security":
+            is_plugin = True
+
         if namespace not in modules:
-            modules[namespace] = Module(namespace)
+            modules[namespace] = Module(namespace, is_plugin)
 
         modules[namespace].add(API(namespace, name, api))
 
@@ -752,13 +811,20 @@ def dump_modules(modules: Any) -> None:
             todir="/opensearchpy/client/",
             additional_replacements=additional_replacements,
         ),
+        unasync.Rule(
+            fromdir="/opensearchpy/_async/plugins/",
+            todir="/opensearchpy/plugins/",
+            additional_replacements=additional_replacements,
+        ),
     ]
 
     filepaths = []
     for root, _, filenames in os.walk(CODE_ROOT / "opensearchpy/_async"):
         for filename in filenames:
-            if filename.rpartition(".")[-1] in ("py",) and not filename.startswith(
-                "utils.py"
+            if filename.rpartition(".")[-1] in ("py",) and filename not in (
+                "utils.py",
+                "index_management.py",
+                "alerting.py",
             ):
                 filepaths.append(os.path.join(root, filename))
 

utils/license_headers.py

@@ -79,7 +79,7 @@ def add_header_to_file(filepath: str) -> None:
     for i, line in enumerate(lines):
         if len(line) > 0 and line not in LINES_TO_KEEP:
             break
-    lines = lines[:i] + [LICENSE_HEADER] + lines[i:]
+    lines = lines[:i] + [LICENSE_HEADER + "\n\n"] + lines[i:]
     with open(filepath, mode="w", encoding="utf-8") as file:
         file.truncate()
         file.write("".join(lines))