Merge branch 'master' into type-hints

Author: sethmlarson

.ci/jobs/defaults.yml

@@ -30,7 +30,7 @@
         wipe-workspace: 'True'
     triggers:
     - github
-    - timed: '@daily'
+    - timed: 'H */12 * * *'
     axes:
     - axis:
         type: slave

.ci/jobs/elastic+elasticsearch-py+7.7.yml renamed to .ci/jobs/elastic+elasticsearch-py+7.9.yml

@@ -1,12 +1,12 @@
 ---
 - job:
-    name: elastic+elasticsearch-py+7.7
-    display-name: 'elastic / elasticsearch-py # 7.7'
-    description: Testing the elasticsearch-py 7.7 branch.
+    name: elastic+elasticsearch-py+7.9
+    display-name: 'elastic / elasticsearch-py # 7.9'
+    description: Testing the elasticsearch-py 7.9 branch.
     junit_results: "*-junit.xml"
     parameters:
     - string:
         name: branch_specifier
-        default: refs/heads/7.7
+        default: refs/heads/7.9
         description: the Git branch specifier to build (<branchName>, <tagName>,
           <commitId>, etc.)

Changelog.rst

@@ -3,6 +3,18 @@
 Changelog
 =========
 
+7.9.0a1 (2020-07-20)
+--------------------
+
+* Added support for ES 7.9 APIs
+* Fixed retries to not raise an error when ``sniff_on_connection_error=True``
+  and a ``TransportError`` is raised during the sniff step. Instead the
+  retry will continue or the error that triggered the retry will be raised
+  (See `#1279`_ and `#1326`_)
+
+ .. _#1326: https://github.com/elastic/elasticsearch-py/pull/1326
+ .. _#1279: https://github.com/elastic/elasticsearch-py/pull/1279
+
 7.8.0 (2020-06-18)
 ------------------
 

docs/index.rst

@@ -23,7 +23,7 @@ the ``async`` extra:
 
     $ python -m pip install elasticsearch[async]
 
-Read more about `how to use asyncio with this project <async>`_.
+Read more about `how to use asyncio with this project <async.html>`_.
 
 
 Compatibility

elasticsearch/_async/client/indices.py

@@ -1241,7 +1241,8 @@ async def delete_data_stream(self, name, params=None, headers=None):
         Deletes a data stream.
         `<https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html>`_
 
-        :arg name: The name of the data stream
+        :arg name: A comma-separated list of data streams to delete; use
+            `*` to delete all data streams
         """
         if name in SKIP_IN_PATH:
             raise ValueError("Empty value passed for a required argument 'name'.")
@@ -1370,8 +1371,8 @@ async def get_data_stream(self, name=None, params=None, headers=None):
         Returns data streams.
         `<https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html>`_
 
-        :arg name: The name or wildcard expression of the requested data
-            streams
+        :arg name: A comma-separated list of data streams to get; use
+            `*` to get all data streams
         """
         return await self.transport.perform_request(
             "GET", _make_path("_data_stream", name), params=params, headers=headers
@@ -1433,8 +1434,8 @@ async def add_block(self, index, block, params=None, headers=None):
         `<https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-blocks.html>`_
 
         :arg index: A comma separated list of indices to add a block to
-        :arg block: The block to add (one of read, write, read_only,
-            metadata, read_only_allow_delete)
+        :arg block: The block to add (one of read, write, read_only or
+            metadata)
         :arg allow_no_indices: Whether to ignore if a wildcard indices
             expression resolves into no concrete indices. (This includes `_all`
             string or when no indices have been specified)

elasticsearch/_async/client/ml.py

@@ -1529,3 +1529,24 @@ async def update_model_snapshot(
             headers=headers,
             body=body,
         )
+
+    @query_params()
+    async def update_data_frame_analytics(self, id, body, params=None, headers=None):
+        """
+        Updates certain properties of a data frame analytics job.
+        `<https://www.elastic.co/guide/en/elasticsearch/reference/master/update-dfanalytics.html>`_
+
+        :arg id: The ID of the data frame analytics to update
+        :arg body: The data frame analytics settings to update
+        """
+        for param in (id, body):
+            if param in SKIP_IN_PATH:
+                raise ValueError("Empty value passed for a required argument.")
+
+        return await self.transport.perform_request(
+            "POST",
+            _make_path("_ml", "data_frame", "analytics", id, "_update"),
+            params=params,
+            headers=headers,
+            body=body,
+        )

elasticsearch/_async/client/xpack.py

@@ -23,12 +23,14 @@ def __getattr__(self, attr_name):
         return getattr(self.client, attr_name)
 
     # AUTO-GENERATED-API-DEFINITIONS #
-    @query_params("categories")
+    @query_params("accept_enterprise", "categories")
     async def info(self, params=None, headers=None):
         """
         Retrieves information about the installed X-Pack features.
         `<https://www.elastic.co/guide/en/elasticsearch/reference/master/info-api.html>`_
 
+        :arg accept_enterprise: Supported for backwards compatibility
+            with 7.x. If this param is used it must be set to true
         :arg categories: Comma-separated list of info categories. Can be
             any of: build, license, features
         """

elasticsearch/_async/transport.py

@@ -316,20 +316,26 @@ async def perform_request(self, method, url, headers=None, params=None, body=Non
                     retry = True
 
                 if retry:
-                    # only mark as dead if we are retrying
-                    self.mark_dead(connection)
+                    try:
+                        # only mark as dead if we are retrying
+                        self.mark_dead(connection)
+                    except TransportError:
+                        # If sniffing on failure, it could fail too. Catch the
+                        # exception not to interrupt the retries.
+                        pass
                     # raise exception on last retry
                     if attempt == self.max_retries:
-                        raise
+                        raise e
                 else:
-                    raise
+                    raise e
 
             else:
+                # connection didn't fail, confirm it's live status
+                self.connection_pool.mark_live(connection)
+
                 if method == "HEAD":
                     return 200 <= status < 300
 
-                # connection didn't fail, confirm it's live status
-                self.connection_pool.mark_live(connection)
                 if data:
                     data = self.deserializer.loads(data, headers.get("content-type"))
                 return data

elasticsearch/client/indices.py

@@ -1239,7 +1239,8 @@ def delete_data_stream(self, name, params=None, headers=None):
         Deletes a data stream.
         `<https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html>`_
 
-        :arg name: The name of the data stream
+        :arg name: A comma-separated list of data streams to delete; use
+            `*` to delete all data streams
         """
         if name in SKIP_IN_PATH:
             raise ValueError("Empty value passed for a required argument 'name'.")
@@ -1368,8 +1369,8 @@ def get_data_stream(self, name=None, params=None, headers=None):
         Returns data streams.
         `<https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html>`_
 
-        :arg name: The name or wildcard expression of the requested data
-            streams
+        :arg name: A comma-separated list of data streams to get; use
+            `*` to get all data streams
         """
         return self.transport.perform_request(
             "GET", _make_path("_data_stream", name), params=params, headers=headers
@@ -1431,8 +1432,8 @@ def add_block(self, index, block, params=None, headers=None):
         `<https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-blocks.html>`_
 
         :arg index: A comma separated list of indices to add a block to
-        :arg block: The block to add (one of read, write, read_only,
-            metadata, read_only_allow_delete)
+        :arg block: The block to add (one of read, write, read_only or
+            metadata)
         :arg allow_no_indices: Whether to ignore if a wildcard indices
             expression resolves into no concrete indices. (This includes `_all`
             string or when no indices have been specified)

elasticsearch/client/ml.py

@@ -1515,3 +1515,24 @@ def update_model_snapshot(
             headers=headers,
             body=body,
         )
+
+    @query_params()
+    def update_data_frame_analytics(self, id, body, params=None, headers=None):
+        """
+        Updates certain properties of a data frame analytics job.
+        `<https://www.elastic.co/guide/en/elasticsearch/reference/master/update-dfanalytics.html>`_
+
+        :arg id: The ID of the data frame analytics to update
+        :arg body: The data frame analytics settings to update
+        """
+        for param in (id, body):
+            if param in SKIP_IN_PATH:
+                raise ValueError("Empty value passed for a required argument.")
+
+        return self.transport.perform_request(
+            "POST",
+            _make_path("_ml", "data_frame", "analytics", id, "_update"),
+            params=params,
+            headers=headers,
+            body=body,
+        )

elasticsearch/client/xpack.py

@@ -23,12 +23,14 @@ def __getattr__(self, attr_name):
         return getattr(self.client, attr_name)
 
     # AUTO-GENERATED-API-DEFINITIONS #
-    @query_params("categories")
+    @query_params("accept_enterprise", "categories")
     def info(self, params=None, headers=None):
         """
         Retrieves information about the installed X-Pack features.
         `<https://www.elastic.co/guide/en/elasticsearch/reference/master/info-api.html>`_
 
+        :arg accept_enterprise: Supported for backwards compatibility
+            with 7.x. If this param is used it must be set to true
         :arg categories: Comma-separated list of info categories. Can be
             any of: build, license, features
         """

elasticsearch/transport.py

@@ -378,13 +378,18 @@ def perform_request(self, method, url, headers=None, params=None, body=None):
                     retry = True
 
                 if retry:
-                    # only mark as dead if we are retrying
-                    self.mark_dead(connection)
+                    try:
+                        # only mark as dead if we are retrying
+                        self.mark_dead(connection)
+                    except TransportError:
+                        # If sniffing on failure, it could fail too. Catch the
+                        # exception not to interrupt the retries.
+                        pass
                     # raise exception on last retry
                     if attempt == self.max_retries:
-                        raise
+                        raise e
                 else:
-                    raise
+                    raise e
 
             else:
                 # connection didn't fail, confirm it's live status

setup.py

@@ -55,6 +55,11 @@
     author_email="honza.kral@gmail.com, nick@nicklang.com",
     maintainer="Seth Michael Larson",
     maintainer_email="seth.larson@elastic.co",
+    project_urls={
+        "Documentation": "https://elasticsearch-py.readthedocs.io",
+        "Source Code": "https://github.com/elastic/elasticsearch-py",
+        "Issue Tracker": "https://github.com/elastic/elasticsearch-py/issues",
+    },
     packages=find_packages(where=".", exclude=("test_elasticsearch*",)),
     package_data={"elasticsearch": ["py.typed"]},
     include_package_data=True,

test_elasticsearch/test_async/test_transport.py

@@ -18,13 +18,14 @@
 
 from __future__ import unicode_literals
 import asyncio
+import json
 from mock import patch
 import pytest
 
 from elasticsearch import AsyncTransport
 from elasticsearch.connection import Connection
 from elasticsearch.connection_pool import DummyConnectionPool
-from elasticsearch.exceptions import ConnectionError
+from elasticsearch.exceptions import ConnectionError, TransportError
 
 
 pytestmark = pytest.mark.asyncio
@@ -273,16 +274,17 @@ async def test_failed_connection_will_be_marked_as_dead(self):
         assert 0 == len(t.connection_pool.connections)
 
     async def test_resurrected_connection_will_be_marked_as_live_on_success(self):
-        t = AsyncTransport([{}, {}], connection_class=DummyConnection)
-        await t._async_call()
-        con1 = t.connection_pool.get_connection()
-        con2 = t.connection_pool.get_connection()
-        t.connection_pool.mark_dead(con1)
-        t.connection_pool.mark_dead(con2)
-
-        await t.perform_request("GET", "/")
-        assert 1 == len(t.connection_pool.connections)
-        assert 1 == len(t.connection_pool.dead_count)
+        for method in ("GET", "HEAD"):
+            t = AsyncTransport([{}, {}], connection_class=DummyConnection)
+            await t._async_call()
+            con1 = t.connection_pool.get_connection()
+            con2 = t.connection_pool.get_connection()
+            t.connection_pool.mark_dead(con1)
+            t.connection_pool.mark_dead(con2)
+
+            await t.perform_request(method, "/")
+            assert 1 == len(t.connection_pool.connections)
+            assert 1 == len(t.connection_pool.dead_count)
 
     async def test_sniff_will_use_seed_connections(self):
         t = AsyncTransport([{"data": CLUSTER_NODES}], connection_class=DummyConnection)
@@ -368,6 +370,25 @@ async def test_sniff_on_fail_triggers_sniffing_on_fail(self):
         assert 1 == len(t.connection_pool.connections)
         assert "http://1.1.1.1:123" == t.get_connection().host
 
+    @patch("elasticsearch._async.transport.AsyncTransport.sniff_hosts")
+    async def test_sniff_on_fail_failing_does_not_prevent_retires(self, sniff_hosts):
+        sniff_hosts.side_effect = [TransportError("sniff failed")]
+        t = AsyncTransport(
+            [{"exception": ConnectionError("abandon ship")}, {"data": CLUSTER_NODES}],
+            connection_class=DummyConnection,
+            sniff_on_connection_fail=True,
+            max_retries=3,
+            randomize_hosts=False,
+        )
+        await t._async_init()
+
+        conn_err, conn_data = t.connection_pool.connections
+        response = await t.perform_request("GET", "/")
+        assert json.loads(CLUSTER_NODES) == response
+        assert 1 == sniff_hosts.call_count
+        assert 1 == len(conn_err.calls)
+        assert 1 == len(conn_data.calls)
+
     async def test_sniff_after_n_seconds(self, event_loop):
         t = AsyncTransport(
             [{"data": CLUSTER_NODES}],