diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 9f046a0a8e..731e2e112a 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -62,7 +62,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - + - name: Set up Python 3.10 uses: actions/setup-python@v4 with: @@ -107,7 +107,7 @@ jobs: # all installed dependencies and build files source .env/bin/activate pip install mypy types-protobuf - # Install the benchmark requirements + # Install the benchmark requirements pip install -r ../benchmarks/python/requirements.txt python -m mypy .. @@ -169,7 +169,7 @@ jobs: yum -y remove git yum -y remove git-* yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm - yum install -y git + yum install -y git git --version - uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index a1e677c31f..1c934181ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,7 @@ * Python: Added LCS command ([#1716](https://github.com/aws/glide-for-redis/pull/1716)) * Python: Added WAIT command ([#1710](https://github.com/aws/glide-for-redis/pull/1710)) * Python: Added XAUTOCLAIM command ([#1718](https://github.com/aws/glide-for-redis/pull/1718)) +* Python: Add ZSCAN and HSCAN commands ([#1732](https://github.com/aws/glide-for-redis/pull/1732)) ### Breaking Changes * Node: Update XREAD to return a Map of Map ([#1494](https://github.com/aws/glide-for-redis/pull/1494)) diff --git a/python/python/glide/async_commands/core.py b/python/python/glide/async_commands/core.py index 2f4c20b20a..799e0175d8 100644 --- a/python/python/glide/async_commands/core.py +++ b/python/python/glide/async_commands/core.py @@ -5578,6 +5578,128 @@ async def sscan( await self._execute_command(RequestType.SScan, args), ) + async def zscan( + self, + key: str, + cursor: str, + match: Optional[str] = None, + count: Optional[int] = None, + ) -> List[Union[str, List[str]]]: + """ + Iterates incrementally over a sorted set. + + See https://valkey.io/commands/zscan for more details. + + Args: + key (str): The key of the sorted set. + cursor (str): The cursor that points to the next iteration of results. A value of "0" indicates the start of + the search. + match (Optional[str]): The match filter is applied to the result of the command and will only include + strings that match the pattern specified. If the sorted set is large enough for scan commands to return + only a subset of the sorted set then there could be a case where the result is empty although there are + items that match the pattern specified. This is due to the default `COUNT` being `10` which indicates + that it will only fetch and match `10` items from the list. + count (Optional[int]): `COUNT` is a just a hint for the command for how many elements to fetch from the + sorted set. `COUNT` could be ignored until the sorted set is large enough for the `SCAN` commands to + represent the results as compact single-allocation packed encoding. + + Returns: + List[Union[str, List[str]]]: An `Array` of the `cursor` and the subset of the sorted set held by `key`. + The first element is always the `cursor` for the next iteration of results. `0` will be the `cursor` + returned on the last iteration of the sorted set. The second element is always an `Array` of the subset + of the sorted set held in `key`. The `Array` in the second element is always a flattened series of + `String` pairs, where the value is at even indices and the score is at odd indices. + + Examples: + # Assume "key" contains a sorted set with multiple members + >>> result_cursor = "0" + >>> while True: + ... result = await redis_client.zscan("key", "0", match="*", count=5) + ... new_cursor = str(result [0]) + ... print("Cursor: ", new_cursor) + ... print("Members: ", result[1]) + ... if new_cursor == "0": + ... break + ... result_cursor = new_cursor + Cursor: 123 + Members: ['value 163', '163', 'value 114', '114', 'value 25', '25', 'value 82', '82', 'value 64', '64'] + Cursor: 47 + Members: ['value 39', '39', 'value 127', '127', 'value 43', '43', 'value 139', '139', 'value 211', '211'] + Cursor: 0 + Members: ['value 55', '55', 'value 24', '24', 'value 90', '90', 'value 113', '113'] + """ + args = [key, cursor] + if match is not None: + args += ["MATCH", match] + if count is not None: + args += ["COUNT", str(count)] + + return cast( + List[Union[str, List[str]]], + await self._execute_command(RequestType.ZScan, args), + ) + + async def hscan( + self, + key: str, + cursor: str, + match: Optional[str] = None, + count: Optional[int] = None, + ) -> List[Union[str, List[str]]]: + """ + Iterates incrementally over a hash. + + See https://valkey.io/commands/hscan for more details. + + Args: + key (str): The key of the set. + cursor (str): The cursor that points to the next iteration of results. A value of "0" indicates the start of + the search. + match (Optional[str]): The match filter is applied to the result of the command and will only include + strings that match the pattern specified. If the hash is large enough for scan commands to return only a + subset of the hash then there could be a case where the result is empty although there are items that + match the pattern specified. This is due to the default `COUNT` being `10` which indicates that it will + only fetch and match `10` items from the list. + count (Optional[int]): `COUNT` is a just a hint for the command for how many elements to fetch from the hash. + `COUNT` could be ignored until the hash is large enough for the `SCAN` commands to represent the results + as compact single-allocation packed encoding. + + Returns: + List[Union[str, List[str]]]: An `Array` of the `cursor` and the subset of the hash held by `key`. + The first element is always the `cursor` for the next iteration of results. `0` will be the `cursor` + returned on the last iteration of the hash. The second element is always an `Array` of the subset of the + hash held in `key`. The `Array` in the second element is always a flattened series of `String` pairs, + where the value is at even indices and the score is at odd indices. + + Examples: + # Assume "key" contains a hash with multiple members + >>> result_cursor = "0" + >>> while True: + ... result = await redis_client.hscan("key", "0", match="*", count=3) + ... new_cursor = str(result [0]) + ... print("Cursor: ", new_cursor) + ... print("Members: ", result[1]) + ... if new_cursor == "0": + ... break + ... result_cursor = new_cursor + Cursor: 31 + Members: ['field 79', 'value 79', 'field 20', 'value 20', 'field 115', 'value 115'] + Cursor: 39 + Members: ['field 63', 'value 63', 'field 293', 'value 293', 'field 162', 'value 162'] + Cursor: 0 + Members: ['field 420', 'value 420', 'field 221', 'value 221'] + """ + args = [key, cursor] + if match is not None: + args += ["MATCH", match] + if count is not None: + args += ["COUNT", str(count)] + + return cast( + List[Union[str, List[str]]], + await self._execute_command(RequestType.HScan, args), + ) + @dataclass class PubSubMsg: """ diff --git a/python/python/glide/async_commands/transaction.py b/python/python/glide/async_commands/transaction.py index aee0ff58e4..631f2003fa 100644 --- a/python/python/glide/async_commands/transaction.py +++ b/python/python/glide/async_commands/transaction.py @@ -4036,6 +4036,86 @@ def sscan( return self.append_command(RequestType.SScan, args) + def zscan( + self: TTransaction, + key: str, + cursor: str, + match: Optional[str] = None, + count: Optional[int] = None, + ) -> TTransaction: + """ + Iterates incrementally over a sorted set. + + See https://valkey.io/commands/zscan for more details. + + Args: + key (str): The key of the sorted set. + cursor (str): The cursor that points to the next iteration of results. A value of "0" indicates the start of + the search. + match (Optional[str]): The match filter is applied to the result of the command and will only include + strings that match the pattern specified. If the sorted set is large enough for scan commands to return + only a subset of the sorted set then there could be a case where the result is empty although there are + items that match the pattern specified. This is due to the default `COUNT` being `10` which indicates + that it will only fetch and match `10` items from the list. + count (Optional[int]): `COUNT` is a just a hint for the command for how many elements to fetch from the + sorted set. `COUNT` could be ignored until the sorted set is large enough for the `SCAN` commands to + represent the results as compact single-allocation packed encoding. + + Returns: + List[Union[str, List[str]]]: An `Array` of the `cursor` and the subset of the sorted set held by `key`. + The first element is always the `cursor` for the next iteration of results. `0` will be the `cursor` + returned on the last iteration of the sorted set. The second element is always an `Array` of the subset + of the sorted set held in `key`. The `Array` in the second element is always a flattened series of + `String` pairs, where the value is at even indices and the score is at odd indices. + """ + args = [key, cursor] + if match is not None: + args += ["MATCH", match] + if count is not None: + args += ["COUNT", str(count)] + + return self.append_command(RequestType.ZScan, args) + + def hscan( + self: TTransaction, + key: str, + cursor: str, + match: Optional[str] = None, + count: Optional[int] = None, + ) -> TTransaction: + """ + Iterates incrementally over a hash. + + See https://valkey.io/commands/hscan for more details. + + Args: + key (str): The key of the set. + cursor (str): The cursor that points to the next iteration of results. A value of "0" indicates the start of + the search. + match (Optional[str]): The match filter is applied to the result of the command and will only include + strings that match the pattern specified. If the hash is large enough for scan commands to return only a + subset of the hash then there could be a case where the result is empty although there are items that + match the pattern specified. This is due to the default `COUNT` being `10` which indicates that it will + only fetch and match `10` items from the list. + count (Optional[int]): `COUNT` is a just a hint for the command for how many elements to fetch from the hash. + `COUNT` could be ignored until the hash is large enough for the `SCAN` commands to represent the results + as compact single-allocation packed encoding. + + Returns: + List[Union[str, List[str]]]: An `Array` of the `cursor` and the subset of the hash held by `key`. + The first element is always the `cursor` for the next iteration of results. `0` will be the `cursor` + returned on the last iteration of the hash. The second element is always an `Array` of the subset of the + hash held in `key`. The `Array` in the second element is always a flattened series of `String` pairs, + where the value is at even indices and the score is at odd indices. + """ + args = [key, cursor] + if match is not None: + args += ["MATCH", match] + if count is not None: + args += ["COUNT", str(count)] + + return self.append_command(RequestType.HScan, args) + def lcs( self: TTransaction, key1: str, diff --git a/python/python/tests/test_async_client.py b/python/python/tests/test_async_client.py index 0524ffcd3c..d253c72d12 100644 --- a/python/python/tests/test_async_client.py +++ b/python/python/tests/test_async_client.py @@ -8185,6 +8185,7 @@ async def test_sscan(self, redis_client: GlideClusterClient): set(next_result[result_collection_index]) ) result_values.update(next_result[result_collection_index]) + result = next_result result_cursor = next_result_cursor assert set(num_members).issubset(result_values) assert set(char_members).issubset(result_values) @@ -8216,6 +8217,233 @@ async def test_sscan(self, redis_client: GlideClusterClient): with pytest.raises(RequestError): await redis_client.sscan(key2, initial_cursor, count=-1) + @pytest.mark.parametrize("cluster_mode", [True, False]) + @pytest.mark.parametrize("protocol", [ProtocolVersion.RESP2, ProtocolVersion.RESP3]) + async def test_zscan(self, redis_client: GlideClusterClient): + key1 = f"{{key}}-1{get_random_string(5)}" + key2 = f"{{key}}-2{get_random_string(5)}" + initial_cursor = "0" + result_cursor_index = 0 + result_collection_index = 1 + default_count = 20 + num_map = {} + num_map_with_str_scores = {} + for i in range(50000): # Use large dataset to force an iterative cursor. + num_map.update({"value " + str(i): i}) + num_map_with_str_scores.update({"value " + str(i): str(i)}) + char_map = {"a": 0, "b": 1, "c": 2, "d": 3, "e": 4} + char_map_with_str_scores = { + "a": "0", + "b": "1", + "c": "2", + "d": "3", + "e": "4", + } + + convert_list_to_dict = lambda list: { + list[i]: list[i + 1] for i in range(0, len(list), 2) + } + + # Empty set + result = await redis_client.zscan(key1, initial_cursor) + assert result[result_cursor_index] == initial_cursor.encode() + assert result[result_collection_index] == [] + + # Negative cursor + result = await redis_client.zscan(key1, "-1") + assert result[result_cursor_index] == initial_cursor.encode() + assert result[result_collection_index] == [] + + # Result contains the whole set + assert await redis_client.zadd(key1, char_map) == len(char_map) + result = await redis_client.zscan(key1, initial_cursor) + result_collection = result[result_collection_index] + assert result[result_cursor_index] == initial_cursor.encode() + assert len(result_collection) == len(char_map) * 2 + assert convert_list_to_dict(result_collection) == cast( + list, convert_string_to_bytes_object(char_map_with_str_scores) + ) + + result = await redis_client.zscan(key1, initial_cursor, match="a") + result_collection = result[result_collection_index] + assert result[result_cursor_index] == initial_cursor.encode() + assert convert_list_to_dict(result_collection) == {b"a": b"0"} + + # Result contains a subset of the key + assert await redis_client.zadd(key1, num_map) == len(num_map) + full_result_map = {} + result = result = cast( + list, + convert_bytes_to_string_object( + await redis_client.zscan(key1, initial_cursor) + ), + ) + result_cursor = str(result[result_cursor_index]) + result_iteration_collection: dict[str, str] = convert_list_to_dict( + result[result_collection_index] + ) + full_result_map.update(result_iteration_collection) + + # 0 is returned for the cursor of the last iteration. + while result_cursor != "0": + next_result = cast( + list, + convert_bytes_to_string_object( + await redis_client.zscan(key1, result_cursor) + ), + ) + next_result_cursor = next_result[result_cursor_index] + assert next_result_cursor != result_cursor + + next_result_collection = convert_list_to_dict( + next_result[result_collection_index] + ) + assert result_iteration_collection != next_result_collection + + full_result_map.update(next_result_collection) + result_iteration_collection = next_result_collection + result_cursor = next_result_cursor + assert (num_map_with_str_scores | char_map_with_str_scores) == full_result_map + + # Test match pattern + result = await redis_client.zscan(key1, initial_cursor, match="*") + assert result[result_cursor_index] != b"0" + assert len(result[result_collection_index]) >= default_count + + # Test count + result = await redis_client.zscan(key1, initial_cursor, count=20) + assert result[result_cursor_index] != b"0" + assert len(result[result_collection_index]) >= 20 + + # Test count with match returns a non-empty list + result = await redis_client.zscan(key1, initial_cursor, match="1*", count=20) + assert result[result_cursor_index] != b"0" + assert len(result[result_collection_index]) >= 0 + + # Exceptions + # Non-set key + assert await redis_client.set(key2, "test") == OK + with pytest.raises(RequestError): + await redis_client.zscan(key2, initial_cursor) + with pytest.raises(RequestError): + await redis_client.zscan(key2, initial_cursor, match="test", count=20) + + # Negative count + with pytest.raises(RequestError): + await redis_client.zscan(key2, initial_cursor, count=-1) + + @pytest.mark.parametrize("cluster_mode", [True, False]) + @pytest.mark.parametrize("protocol", [ProtocolVersion.RESP2, ProtocolVersion.RESP3]) + async def test_hscan(self, redis_client: GlideClusterClient): + key1 = f"{{key}}-1{get_random_string(5)}" + key2 = f"{{key}}-2{get_random_string(5)}" + initial_cursor = "0" + result_cursor_index = 0 + result_collection_index = 1 + default_count = 20 + num_map = {} + for i in range(50000): # Use large dataset to force an iterative cursor. + num_map.update({"field " + str(i): "value " + str(i)}) + char_map = { + "field a": "value a", + "field b": "value b", + "field c": "value c", + "field d": "value d", + "field e": "value e", + } + + convert_list_to_dict = lambda list: { + list[i]: list[i + 1] for i in range(0, len(list), 2) + } + + # Empty set + result = await redis_client.hscan(key1, initial_cursor) + assert result[result_cursor_index] == initial_cursor.encode() + assert result[result_collection_index] == [] + + # Negative cursor + result = await redis_client.hscan(key1, "-1") + assert result[result_cursor_index] == initial_cursor.encode() + assert result[result_collection_index] == [] + + # Result contains the whole set + assert await redis_client.hset(key1, char_map) == len(char_map) + result = await redis_client.hscan(key1, initial_cursor) + result_collection = result[result_collection_index] + assert result[result_cursor_index] == initial_cursor.encode() + assert len(result_collection) == len(char_map) * 2 + assert convert_list_to_dict(result_collection) == cast( + dict, convert_string_to_bytes_object(char_map) + ) + + result = await redis_client.hscan(key1, initial_cursor, match="field a") + result_collection = result[result_collection_index] + assert result[result_cursor_index] == initial_cursor.encode() + assert convert_list_to_dict(result_collection) == {b"field a": b"value a"} + + # Result contains a subset of the key + assert await redis_client.hset(key1, num_map) == len(num_map) + full_result_map = {} + result = result = cast( + list, + convert_bytes_to_string_object( + await redis_client.hscan(key1, initial_cursor) + ), + ) + result_cursor = str(result[result_cursor_index]) + result_iteration_collection: dict[str, str] = convert_list_to_dict( + result[result_collection_index] + ) + full_result_map.update(result_iteration_collection) + + # 0 is returned for the cursor of the last iteration. + while result_cursor != "0": + next_result = cast( + list, + convert_bytes_to_string_object( + await redis_client.hscan(key1, result_cursor) + ), + ) + next_result_cursor = next_result[result_cursor_index] + assert next_result_cursor != result_cursor + + next_result_collection = convert_list_to_dict( + next_result[result_collection_index] + ) + assert result_iteration_collection != next_result_collection + + full_result_map.update(next_result_collection) + result_iteration_collection = next_result_collection + result_cursor = next_result_cursor + assert (num_map | char_map) == full_result_map + + # Test match pattern + result = await redis_client.hscan(key1, initial_cursor, match="*") + assert result[result_cursor_index] != b"0" + assert len(result[result_collection_index]) >= default_count + + # Test count + result = await redis_client.hscan(key1, initial_cursor, count=20) + assert result[result_cursor_index] != b"0" + assert len(result[result_collection_index]) >= 20 + + # Test count with match returns a non-empty list + result = await redis_client.hscan(key1, initial_cursor, match="1*", count=20) + assert result[result_cursor_index] != b"0" + assert len(result[result_collection_index]) >= 0 + + # Exceptions + # Non-hash key + assert await redis_client.set(key2, "test") == OK + with pytest.raises(RequestError): + await redis_client.hscan(key2, initial_cursor) + with pytest.raises(RequestError): + await redis_client.hscan(key2, initial_cursor, match="test", count=20) + + # Negative count + with pytest.raises(RequestError): + await redis_client.hscan(key2, initial_cursor, count=-1) + @pytest.mark.asyncio class TestScripts: diff --git a/python/python/tests/test_transaction.py b/python/python/tests/test_transaction.py index 121ade1a5a..c7cf93638a 100644 --- a/python/python/tests/test_transaction.py +++ b/python/python/tests/test_transaction.py @@ -237,6 +237,10 @@ async def transaction_test( ) transaction.hdel(key4, [key, key2]) args.append(2) + transaction.hscan(key4, "0") + args.append([b"0", [key3.encode(), b"10.5"]]) + transaction.hscan(key4, "0", match="*", count=10) + args.append([b"0", [key3.encode(), b"10.5"]]) transaction.hrandfield(key4) args.append(key3_bytes) transaction.hrandfield_count(key4, 1) @@ -385,6 +389,10 @@ async def transaction_test( args.append([b"three"]) transaction.zrandmember_withscores(key8, 1) args.append([[b"three", 3.0]]) + transaction.zscan(key8, "0") + args.append([b"0", [b"three", b"3"]]) + transaction.zscan(key8, "0", match="*", count=20) + args.append([b"0", [b"three", b"3"]]) transaction.zpopmax(key8) args.append({b"three": 3.0}) transaction.zpopmin(key8)