feat: add signing of public URL (#407)

This PR is part of [Issue
#19363](apify/apify-core#19363), which updates
`KeyValueStore.getPublicUrl(recordKey)` to generate signed links using
HMAC.

**This PR**:
- creates signature and appends it to the public URL of the record in KV
store

P.S. Before merging, we need to wait for the release of Crawlee, so we
can update version of Crawlee.

P.P.S It's hard to test the changes, since master branch of SDK and
master branch of Crawlee are out of sync. Crawlee has some breaking
changes in version 6.0, which are not yet addressed in master branch of
SDK.

Previous PR that adds storageObject to Crawlee Python is
[here](apify/crawlee-python#993)

Same PR in SDK JS is
[here](apify/apify-sdk-js#358)

Author: danpoletaev

src/apify/_crypto.py

@@ -1,6 +1,9 @@
 from __future__ import annotations
 
 import base64
+import hashlib
+import hmac
+import string
 from typing import Any
 
 from cryptography.exceptions import InvalidTag as InvalidTagException
@@ -153,3 +156,38 @@ def decrypt_input_secrets(private_key: rsa.RSAPrivateKey, input_data: Any) -> An
                 )
 
     return input_data
+
+
+CHARSET = string.digits + string.ascii_letters
+
+
+def encode_base62(num: int) -> str:
+    """Encode the given number to base62."""
+    if num == 0:
+        return CHARSET[0]
+
+    res = ''
+    while num > 0:
+        num, remainder = divmod(num, 62)
+        res = CHARSET[remainder] + res
+    return res
+
+
+@ignore_docs
+def create_hmac_signature(secret_key: str, message: str) -> str:
+    """Generate an HMAC signature and encodes it using Base62. Base62 encoding reduces the signature length.
+
+    HMAC signature is truncated to 30 characters to make it shorter.
+
+    Args:
+        secret_key: Secret key used for signing signatures.
+        message: Message to be signed.
+
+    Returns:
+        Base62 encoded signature.
+    """
+    signature = hmac.new(secret_key.encode('utf-8'), message.encode('utf-8'), hashlib.sha256).hexdigest()[:30]
+
+    decimal_signature = int(signature, 16)
+
+    return encode_base62(decimal_signature)

src/apify/apify_storage_client/_key_value_store_client.py

@@ -4,10 +4,13 @@
 from typing import TYPE_CHECKING, Any
 
 from typing_extensions import override
+from yarl import URL
 
 from crawlee.storage_clients._base import KeyValueStoreClient as BaseKeyValueStoreClient
 from crawlee.storage_clients.models import KeyValueStoreListKeysPage, KeyValueStoreMetadata, KeyValueStoreRecord
 
+from apify._crypto import create_hmac_signature
+
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
     from contextlib import AbstractAsyncContextManager
@@ -89,6 +92,18 @@ async def get_public_url(self, key: str) -> str:
         Args:
             key: The key for which the URL should be generated.
         """
-        public_api_url = self._api_public_base_url
+        if self._client.resource_id is None:
+            raise ValueError('resource_id cannot be None when generating a public URL')
+
+        public_url = (
+            URL(self._api_public_base_url) / 'v2' / 'key-value-stores' / self._client.resource_id / 'records' / key
+        )
+
+        key_value_store = await self.get()
+
+        if key_value_store is not None and isinstance(key_value_store.model_extra, dict):
+            url_signing_secret_key = key_value_store.model_extra.get('urlSigningSecretKey')
+            if url_signing_secret_key:
+                public_url = public_url.with_query(signature=create_hmac_signature(url_signing_secret_key, key))
 
-        return f'{public_api_url}/v2/key-value-stores/{self._client.resource_id}/records/{key}'
+        return str(public_url)

tests/integration/test_actor_key_value_store.py

@@ -201,19 +201,28 @@ async def test_generate_public_url_for_kvs_record(
     run_actor: RunActorFunction,
 ) -> None:
     async def main() -> None:
-        from typing import cast
-
-        from apify.apify_storage_client._key_value_store_client import KeyValueStoreClient
+        from apify._crypto import create_hmac_signature
 
         async with Actor:
             public_api_url = Actor.config.api_public_base_url
             default_store_id = Actor.config.default_key_value_store_id
+            record_key = 'public-record-key'
 
             store = await Actor.open_key_value_store()
-            record_url = await cast(KeyValueStoreClient, store._resource_client).get_public_url('dummy')
-            print(record_url)
 
-            assert record_url == f'{public_api_url}/v2/key-value-stores/{default_store_id}/records/dummy'
+            assert isinstance(store.storage_object.model_extra, dict)
+            url_signing_secret_key = store.storage_object.model_extra.get('urlSigningSecretKey')
+            assert url_signing_secret_key is not None
+
+            await store.set_value(record_key, {'exposedData': 'test'}, 'application/json')
+
+            record_url = await store.get_public_url(record_key)
+
+            signature = create_hmac_signature(url_signing_secret_key, record_key)
+            assert (
+                record_url
+                == f'{public_api_url}/v2/key-value-stores/{default_store_id}/records/{record_key}?signature={signature}'
+            )
 
     actor = await make_actor(label='kvs-get-public-url', main_func=main)
     run_result = await run_actor(actor)

tests/unit/test_crypto.py

@@ -4,7 +4,15 @@
 
 import pytest
 
-from apify._crypto import _load_public_key, crypto_random_object_id, load_private_key, private_decrypt, public_encrypt
+from apify._crypto import (
+    _load_public_key,
+    create_hmac_signature,
+    crypto_random_object_id,
+    encode_base62,
+    load_private_key,
+    private_decrypt,
+    public_encrypt,
+)
 
 # NOTE: Uses the same keys as in:
 # https://github.com/apify/apify-shared-js/blob/master/test/crypto.test.ts
@@ -105,3 +113,25 @@ def test_crypto_random_object_id_length_and_charset() -> None:
     long_random_object_id = crypto_random_object_id(1000)
     for char in long_random_object_id:
         assert char in 'abcdefghijklmnopqrstuvwxyzABCEDFGHIJKLMNOPQRSTUVWXYZ0123456789'
+
+
+@pytest.mark.parametrize(('test_input', 'expected'), [(0, '0'), (10, 'a'), (999999999, '15FTGf')])
+def test_encode_base62(test_input: int, expected: str) -> None:
+    assert encode_base62(test_input) == expected
+
+
+# This test ensures compatibility with the JavaScript version of the same method.
+# https://github.com/apify/apify-shared-js/blob/master/packages/utilities/src/hmac.ts
+def test_create_valid_hmac_signature() -> None:
+    # This test uses the same secret key and message as in JS tests.
+    secret_key = 'hmac-secret-key'
+    message = 'hmac-message-to-be-authenticated'
+    assert create_hmac_signature(secret_key, message) == 'pcVagAsudj8dFqdlg7mG'
+
+
+def test_create_same_hmac() -> None:
+    # This test uses the same secret key and message as in JS tests.
+    secret_key = 'hmac-same-secret-key'
+    message = 'hmac-same-message-to-be-authenticated'
+    assert create_hmac_signature(secret_key, message) == 'FYMcmTIm3idXqleF1Sw5'
+    assert create_hmac_signature(secret_key, message) == 'FYMcmTIm3idXqleF1Sw5'