Merge branch 'master' into client-side-caching-improvements

Author: vladvildanov

.github/workflows/integration.yaml

@@ -28,7 +28,7 @@ env:
   # this speeds up coverage with Python 3.12: https://github.com/nedbat/coveragepy/issues/1665
   COVERAGE_CORE: sysmon
   REDIS_IMAGE: redis:7.4-rc2
-  REDIS_STACK_IMAGE: redis/redis-stack-server:7.4.0-rc2
+  REDIS_STACK_IMAGE: redis/redis-stack-server:latest
 
 jobs:
   dependency-audit:

doctests/dt_set.py

@@ -58,11 +58,11 @@
 r.sadd("bikes:racing:usa", "bike:1", "bike:4")
 # HIDE_END
 res7 = r.sinter("bikes:racing:france", "bikes:racing:usa")
-print(res7)  # >>> {'bike:1'}
+print(res7)  # >>> ['bike:1']
 # STEP_END
 
 # REMOVE_START
-assert res7 == {"bike:1"}
+assert res7 == ["bike:1"]
 # REMOVE_END
 
 # STEP_START scard
@@ -83,12 +83,12 @@
 print(res9)  # >>> 3
 
 res10 = r.smembers("bikes:racing:france")
-print(res10)  # >>> {'bike:1', 'bike:2', 'bike:3'}
+print(res10)  # >>> ['bike:1', 'bike:2', 'bike:3']
 # STEP_END
 
 # REMOVE_START
 assert res9 == 3
-assert res10 == {"bike:1", "bike:2", "bike:3"}
+assert res10 == ['bike:1', 'bike:2', 'bike:3']
 # REMOVE_END
 
 # STEP_START smismember
@@ -109,11 +109,11 @@
 r.sadd("bikes:racing:usa", "bike:1", "bike:4")
 
 res13 = r.sdiff("bikes:racing:france", "bikes:racing:usa")
-print(res13)  # >>> {'bike:2', 'bike:3'}
+print(res13)  # >>> ['bike:2', 'bike:3']
 # STEP_END
 
 # REMOVE_START
-assert res13 == {"bike:2", "bike:3"}
+assert res13 == ['bike:2', 'bike:3']
 r.delete("bikes:racing:france")
 r.delete("bikes:racing:usa")
 # REMOVE_END
@@ -124,27 +124,27 @@
 r.sadd("bikes:racing:italy", "bike:1", "bike:2", "bike:3", "bike:4")
 
 res13 = r.sinter("bikes:racing:france", "bikes:racing:usa", "bikes:racing:italy")
-print(res13)  # >>> {'bike:1'}
+print(res13)  # >>> ['bike:1']
 
 res14 = r.sunion("bikes:racing:france", "bikes:racing:usa", "bikes:racing:italy")
-print(res14)  # >>> {'bike:1', 'bike:2', 'bike:3', 'bike:4'}
+print(res14)  # >>> ['bike:1', 'bike:2', 'bike:3', 'bike:4']
 
 res15 = r.sdiff("bikes:racing:france", "bikes:racing:usa", "bikes:racing:italy")
-print(res15)  # >>> set()
+print(res15)  # >>> []
 
 res16 = r.sdiff("bikes:racing:usa", "bikes:racing:france")
-print(res16)  # >>> {'bike:4'}
+print(res16)  # >>> ['bike:4']
 
 res17 = r.sdiff("bikes:racing:france", "bikes:racing:usa")
-print(res17)  # >>> {'bike:2', 'bike:3'}
+print(res17)  # >>> ['bike:2', 'bike:3']
 # STEP_END
 
 # REMOVE_START
-assert res13 == {"bike:1"}
-assert res14 == {"bike:1", "bike:2", "bike:3", "bike:4"}
-assert res15 == set()
-assert res16 == {"bike:4"}
-assert res17 == {"bike:2", "bike:3"}
+assert res13 == ['bike:1']
+assert res14 == ['bike:1', 'bike:2', 'bike:3', 'bike:4']
+assert res15 == []
+assert res16 == ['bike:4']
+assert res17 == ['bike:2', 'bike:3']
 r.delete("bikes:racing:france")
 r.delete("bikes:racing:usa")
 r.delete("bikes:racing:italy")
@@ -160,7 +160,7 @@
 print(res19)  # >>> bike:3
 
 res20 = r.smembers("bikes:racing:france")
-print(res20)  # >>> {'bike:2', 'bike:4', 'bike:5'}
+print(res20)  # >>> ['bike:2', 'bike:4', 'bike:5']
 
 res21 = r.srandmember("bikes:racing:france")
 print(res21)  # >>> bike:4

redis/_parsers/helpers.py

@@ -783,9 +783,6 @@ def string_keys_to_dict(key_string, callback):
 
 
 _RedisCallbacksRESP2 = {
-    **string_keys_to_dict(
-        "SDIFF SINTER SMEMBERS SUNION", lambda r: r and set(r) or set()
-    ),
     **string_keys_to_dict(
         "ZDIFF ZINTER ZPOPMAX ZPOPMIN ZRANGE ZRANGEBYSCORE ZRANK ZREVRANGE "
         "ZREVRANGEBYSCORE ZREVRANK ZUNION",

redis/_parsers/resp3.py

@@ -88,15 +88,11 @@ def _read_response(self, disable_decoding=False, push_request=False):
         # set response
         elif byte == b"~":
             # redis can return unhashable types (like dict) in a set,
-            # so we need to first convert to a list, and then try to convert it to a set
+            # so we return sets as list, all the time, for predictability
             response = [
                 self._read_response(disable_decoding=disable_decoding)
                 for _ in range(int(response))
             ]
-            try:
-                response = set(response)
-            except TypeError:
-                pass
         # map response
         elif byte == b"%":
             # We cannot use a dict-comprehension to parse stream.
@@ -230,15 +226,11 @@ async def _read_response(
         # set response
         elif byte == b"~":
             # redis can return unhashable types (like dict) in a set,
-            # so we need to first convert to a list, and then try to convert it to a set
+            # so we always convert to a list, to have predictable return types
             response = [
                 (await self._read_response(disable_decoding=disable_decoding))
                 for _ in range(int(response))
             ]
-            try:
-                response = set(response)
-            except TypeError:
-                pass
         # map response
         elif byte == b"%":
             # We cannot use a dict-comprehension to parse stream.

redis/commands/bf/commands.py

@@ -1,6 +1,5 @@
 from redis.client import NEVER_DECODE
-from redis.exceptions import ModuleError
-from redis.utils import HIREDIS_AVAILABLE, deprecated_function
+from redis.utils import deprecated_function
 
 BF_RESERVE = "BF.RESERVE"
 BF_ADD = "BF.ADD"
@@ -139,9 +138,6 @@ def scandump(self, key, iter):
         This command will return successive (iter, data) pairs until (0, NULL) to indicate completion.
         For more information see `BF.SCANDUMP <https://redis.io/commands/bf.scandump>`_.
         """  # noqa
-        if HIREDIS_AVAILABLE:
-            raise ModuleError("This command cannot be used when hiredis is available.")
-
         params = [key, iter]
         options = {}
         options[NEVER_DECODE] = []

redis/commands/search/aggregation.py

@@ -111,6 +111,7 @@ def __init__(self, query: str = "*") -> None:
         self._verbatim = False
         self._cursor = []
         self._dialect = None
+        self._add_scores = False
 
     def load(self, *fields: List[str]) -> "AggregateRequest":
         """
@@ -292,6 +293,13 @@ def with_schema(self) -> "AggregateRequest":
         self._with_schema = True
         return self
 
+    def add_scores(self) -> "AggregateRequest":
+        """
+        If set, includes the score as an ordinary field of the row.
+        """
+        self._add_scores = True
+        return self
+
     def verbatim(self) -> "AggregateRequest":
         self._verbatim = True
         return self
@@ -315,6 +323,9 @@ def build_args(self) -> List[str]:
         if self._verbatim:
             ret.append("VERBATIM")
 
+        if self._add_scores:
+            ret.append("ADDSCORES")
+
         if self._cursor:
             ret += self._cursor
 

redis/utils.py

@@ -6,8 +6,8 @@
 try:
     import hiredis  # noqa
 
-    # Only support Hiredis >= 1.0:
-    HIREDIS_AVAILABLE = not hiredis.__version__.startswith("0.")
+    # Only support Hiredis >= 3.0:
+    HIREDIS_AVAILABLE = int(hiredis.__version__.split(".")[0]) >= 3
     HIREDIS_PACK_AVAILABLE = hasattr(hiredis, "pack_command")
 except ImportError:
     HIREDIS_AVAILABLE = False

setup.py

@@ -56,7 +56,7 @@
         "Programming Language :: Python :: Implementation :: PyPy",
     ],
     extras_require={
-        "hiredis": ["hiredis>=1.0.0"],
+        "hiredis": ["hiredis>=3.0.0"],
         "ocsp": ["cryptography>=36.0.1", "pyopenssl==23.2.1", "requests>=2.31.0"],
     },
 )

tests/test_asyncio/test_bloom.py

@@ -3,8 +3,7 @@
 import pytest
 import pytest_asyncio
 import redis.asyncio as redis
-from redis.exceptions import ModuleError, RedisError
-from redis.utils import HIREDIS_AVAILABLE
+from redis.exceptions import RedisError
 from tests.conftest import (
     assert_resp_response,
     is_resp2_connection,
@@ -105,10 +104,6 @@ async def do_verify():
 
     await do_verify()
     cmds = []
-    if HIREDIS_AVAILABLE:
-        with pytest.raises(ModuleError):
-            cur = await decoded_r.bf().scandump("myBloom", 0)
-        return
 
     cur = await decoded_r.bf().scandump("myBloom", 0)
     first = cur[0]

tests/test_asyncio/test_cluster.py

@@ -1751,49 +1751,49 @@ async def test_cluster_rpoplpush(self, r: RedisCluster) -> None:
 
     async def test_cluster_sdiff(self, r: RedisCluster) -> None:
         await r.sadd("{foo}a", "1", "2", "3")
-        assert await r.sdiff("{foo}a", "{foo}b") == {b"1", b"2", b"3"}
+        assert set(await r.sdiff("{foo}a", "{foo}b")) == {b"1", b"2", b"3"}
         await r.sadd("{foo}b", "2", "3")
-        assert await r.sdiff("{foo}a", "{foo}b") == {b"1"}
+        assert await r.sdiff("{foo}a", "{foo}b") == [b"1"]
 
     async def test_cluster_sdiffstore(self, r: RedisCluster) -> None:
         await r.sadd("{foo}a", "1", "2", "3")
         assert await r.sdiffstore("{foo}c", "{foo}a", "{foo}b") == 3
-        assert await r.smembers("{foo}c") == {b"1", b"2", b"3"}
+        assert set(await r.smembers("{foo}c")) == {b"1", b"2", b"3"}
         await r.sadd("{foo}b", "2", "3")
         assert await r.sdiffstore("{foo}c", "{foo}a", "{foo}b") == 1
-        assert await r.smembers("{foo}c") == {b"1"}
+        assert await r.smembers("{foo}c") == [b"1"]
 
     async def test_cluster_sinter(self, r: RedisCluster) -> None:
         await r.sadd("{foo}a", "1", "2", "3")
-        assert await r.sinter("{foo}a", "{foo}b") == set()
+        assert await r.sinter("{foo}a", "{foo}b") == []
         await r.sadd("{foo}b", "2", "3")
-        assert await r.sinter("{foo}a", "{foo}b") == {b"2", b"3"}
+        assert set(await r.sinter("{foo}a", "{foo}b")) == {b"2", b"3"}
 
     async def test_cluster_sinterstore(self, r: RedisCluster) -> None:
         await r.sadd("{foo}a", "1", "2", "3")
         assert await r.sinterstore("{foo}c", "{foo}a", "{foo}b") == 0
-        assert await r.smembers("{foo}c") == set()
+        assert await r.smembers("{foo}c") == []
         await r.sadd("{foo}b", "2", "3")
         assert await r.sinterstore("{foo}c", "{foo}a", "{foo}b") == 2
-        assert await r.smembers("{foo}c") == {b"2", b"3"}
+        assert set(await r.smembers("{foo}c")) == {b"2", b"3"}
 
     async def test_cluster_smove(self, r: RedisCluster) -> None:
         await r.sadd("{foo}a", "a1", "a2")
         await r.sadd("{foo}b", "b1", "b2")
         assert await r.smove("{foo}a", "{foo}b", "a1")
-        assert await r.smembers("{foo}a") == {b"a2"}
-        assert await r.smembers("{foo}b") == {b"b1", b"b2", b"a1"}
+        assert await r.smembers("{foo}a") == [b"a2"]
+        assert set(await r.smembers("{foo}b")) == {b"b1", b"b2", b"a1"}
 
     async def test_cluster_sunion(self, r: RedisCluster) -> None:
         await r.sadd("{foo}a", "1", "2")
         await r.sadd("{foo}b", "2", "3")
-        assert await r.sunion("{foo}a", "{foo}b") == {b"1", b"2", b"3"}
+        assert set(await r.sunion("{foo}a", "{foo}b")) == {b"1", b"2", b"3"}
 
     async def test_cluster_sunionstore(self, r: RedisCluster) -> None:
         await r.sadd("{foo}a", "1", "2")
         await r.sadd("{foo}b", "2", "3")
         assert await r.sunionstore("{foo}c", "{foo}a", "{foo}b") == 3
-        assert await r.smembers("{foo}c") == {b"1", b"2", b"3"}
+        assert set(await r.smembers("{foo}c")) == {b"1", b"2", b"3"}
 
     @skip_if_server_version_lt("6.2.0")
     async def test_cluster_zdiff(self, r: RedisCluster) -> None: