Merge pull request #23 from thewebscraping/develop

Develop

Author: thewebscraping

requirements-dev.txt

@@ -10,16 +10,15 @@ setuptools~=75.3.0
 twine~=6.0.1
 
 # Tests & Linting
-pre-commit==3.7.0
+pre-commit
+pytest-cov
+Werkzeug
 black==24.3.0
 coverage[toml]==7.6.1
-pre-commit==3.7.0
 isort==5.13.2
 flake8==7.1.1
 mypy==1.11.2
 pytest==8.3.3
 pytest-asyncio==0.24.0
-pytest-cov==6.0.0
 pytest_httpserver==1.1.0
-Werkzeug==3.1.3
 tox==4.23.2

setup.cfg

@@ -3,11 +3,12 @@ long_description = file: README.md
 long_description_content_type = text/markdown
 license = MIT
 license_file = LICENSE
-python_requires = >=3.9
+python_requires = >=3.8
 install_requires =
     chardet ~= 5.2.0
     requests ~= 2.32.3
     tqdm ~= 4.67.1
+    idna ~= 3.10
 classifiers =
     Development Status :: 5 - Production/Stable
     Intended Audience :: Developers
@@ -17,6 +18,7 @@ classifiers =
     Operating System :: OS Independent
     Programming Language :: Python
     Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11

setup.py

@@ -9,7 +9,7 @@
 
 BASE_DIR = os.path.dirname(__file__)
 CURRENT_PYTHON = sys.version_info[:2]
-REQUIRED_PYTHON = (3, 9)
+REQUIRED_PYTHON = (3, 8)
 
 if CURRENT_PYTHON < REQUIRED_PYTHON:
     sys.stderr.write(

tls_requests/models/encoders.py

@@ -2,7 +2,7 @@
 import os
 from io import BufferedReader, BytesIO, TextIOWrapper
 from mimetypes import guess_type
-from typing import Any, AsyncIterator, Iterator, Mapping, TypeVar
+from typing import Any, AsyncIterator, Dict, Iterator, Mapping, Tuple, TypeVar
 from urllib.parse import urlencode
 
 from tls_requests.types import (BufferTypes, ByteOrStr, RequestData,
@@ -75,7 +75,7 @@ def render(self, chunk_size: int = 65_536) -> Iterator[bytes]:
         yield self.render_headers()
         yield from self.render_data(chunk_size)
 
-    def get_headers(self) -> dict[bytes, bytes]:
+    def get_headers(self) -> Dict[bytes, bytes]:
         self._headers[b"Content-Disposition"] = self.render_parts()
         content_type = getattr(self, "content_type", None)
         if content_type:
@@ -101,7 +101,7 @@ def __init__(self, name: str, value: RequestFileValue):
         super(FileField, self).__init__(name, value)
         self.filename, self._buffer, self.content_type = self.unpack(value)
 
-    def unpack(self, value: RequestFileValue) -> tuple[str, BufferTypes, str]:
+    def unpack(self, value: RequestFileValue) -> Tuple[str, BufferTypes, str]:
         filename, content_type = None, None
         if isinstance(value, tuple):
             if len(value) > 1:

tls_requests/models/headers.py

@@ -1,7 +1,7 @@
 from abc import ABC
 from collections.abc import Mapping, MutableMapping
 from enum import Enum
-from typing import Any, ItemsView, KeysView, Literal, ValuesView
+from typing import Any, ItemsView, KeysView, List, Literal, Tuple, ValuesView
 
 from tls_requests.types import ByteOrStr, HeaderTypes
 from tls_requests.utils import to_str
@@ -63,7 +63,7 @@ def update(self, headers: HeaderTypes) -> "Headers":  # noqa
     def copy(self) -> "Headers":
         return self.__class__(self._items.copy(), alias=self.alias)  # noqa
 
-    def _prepare_items(self, headers: HeaderTypes) -> list[tuple[str, Any]]:
+    def _prepare_items(self, headers: HeaderTypes) -> List[Tuple[str, Any]]:
         if headers is None:
             return []
         if isinstance(headers, self.__class__):
@@ -88,7 +88,7 @@ def _normalize_key(self, key: ByteOrStr) -> str:
 
         return key.lower()
 
-    def _normalize_value(self, value) -> list[str]:
+    def _normalize_value(self, value) -> List[str]:
         if isinstance(value, dict):
             raise TypeError
 
@@ -102,7 +102,7 @@ def _normalize_value(self, value) -> list[str]:
 
         return [to_str(value)]
 
-    def _normalize(self, key, value) -> tuple[str, list[str]]:
+    def _normalize(self, key, value) -> Tuple[str, List[str]]:
         return self._normalize_key(key), self._normalize_value(value)
 
     def __setitem__(self, key, value) -> None:

tls_requests/models/libraries.py

@@ -7,7 +7,7 @@
 from dataclasses import dataclass, field, fields
 from pathlib import Path
 from platform import machine
-from typing import Optional
+from typing import List, Optional
 
 import requests
 from tqdm import tqdm
@@ -86,7 +86,7 @@ class ReleaseAsset(BaseRelease):
 class Release(BaseRelease):
     name: Optional[str] = None
     tag_name: Optional[str] = None
-    assets: list[ReleaseAsset] = field(default_factory=list)
+    assets: List[ReleaseAsset] = field(default_factory=list)
 
     @classmethod
     def from_kwargs(cls, **kwargs):
@@ -274,7 +274,7 @@ def find(cls) -> str:
                 return fp
 
     @classmethod
-    def find_all(cls) -> list[str]:
+    def find_all(cls) -> List[str]:
         return [
             src
             for src in glob.glob(os.path.join(BIN_DIR, r"*"))

tls_requests/models/tls.py

@@ -2,7 +2,7 @@
 import uuid
 from dataclasses import asdict, dataclass, field
 from dataclasses import fields as get_fields
-from typing import Any, Optional, TypeVar, Union
+from typing import Any, List, Mapping, Optional, Set, TypeVar, Union
 
 from tls_requests.models.encoders import StreamEncoder
 from tls_requests.models.libraries import TLSLibrary
@@ -178,7 +178,7 @@ class _BaseConfig:
     """Base configuration for TLSSession"""
 
     @classmethod
-    def model_fields_set(cls) -> set[str]:
+    def model_fields_set(cls) -> Set[str]:
         return {
             model_field.name
             for model_field in get_fields(cls)
@@ -341,19 +341,19 @@ class CustomTLSClientConfig(_BaseConfig):
 
     """
 
-    alpnProtocols: list[str] = None
-    alpsProtocols: list[str] = None
+    alpnProtocols: List[str] = None
+    alpsProtocols: List[str] = None
     certCompressionAlgo: str = None
     connectionFlow: int = None
-    h2Settings: list[str] = None
-    h2SettingsOrder: list[str] = None
-    headerPriority: list[str] = None
+    h2Settings: List[str] = None
+    h2SettingsOrder: List[str] = None
+    headerPriority: List[str] = None
     ja3String: str = None
-    keyShareCurves: list[str] = None
-    priorityFrames: list[str] = None
-    pseudoHeaderOrder: list[str] = None
-    supportedSignatureAlgorithms: list[str] = None
-    supportedVersions: list[str] = None
+    keyShareCurves: List[str] = None
+    priorityFrames: List[str] = None
+    pseudoHeaderOrder: List[str] = None
+    supportedSignatureAlgorithms: List[str] = None
+    supportedVersions: List[str] = None
 
 
 @dataclass
@@ -424,19 +424,19 @@ class TLSConfig(_BaseConfig):
     """
 
     catchPanics: bool = False
-    certificatePinningHosts: dict[str, str] = field(default_factory=dict)
+    certificatePinningHosts: Mapping[str, str] = field(default_factory=dict)
     customTlsClient: Optional[CustomTLSClientConfig] = None
     followRedirects: bool = False
     forceHttp1: bool = False
-    headerOrder: list[str] = field(default_factory=list)
-    headers: dict[str, str] = field(default_factory=dict)
+    headerOrder: List[str] = field(default_factory=list)
+    headers: Mapping[str, str] = field(default_factory=dict)
     insecureSkipVerify: bool = False
     isByteRequest: bool = False
     isByteResponse: bool = True
     isRotatingProxy: bool = False
     proxyUrl: str = ""
     requestBody: Union[str, bytes, bytearray, None] = None
-    requestCookies: list[TLSRequestCookiesConfig] = field(default_factory=list)
+    requestCookies: List[TLSRequestCookiesConfig] = field(default_factory=list)
     requestMethod: MethodTypes = None
     requestUrl: Optional[str] = None
     sessionId: str = field(default_factory=lambda: str(uuid.uuid4()))
@@ -471,7 +471,7 @@ def to_dict(self) -> dict:
     def copy_with(
         self,
         session_id: str = None,
-        headers: dict[str, str] = None,
+        headers: Mapping[str, str] = None,
         cookies: TLSCookiesTypes = None,
         method: MethodTypes = None,
         url: URLTypes = None,
@@ -515,7 +515,7 @@ def copy_with(
     def from_kwargs(
         cls,
         session_id: str = None,
-        headers: dict[str, str] = None,
+        headers: Mapping[str, str] = None,
         cookies: TLSCookiesTypes = None,
         method: MethodTypes = None,
         url: URLTypes = None,

tls_requests/models/urls.py

@@ -53,7 +53,7 @@ def __init__(self, params: URLParamTypes = None, **kwargs):
 
     @property
     def params(self) -> str:
-        return quote(str(self))
+        return str(self)
 
     def update(self, params: URLParamTypes = None, **kwargs):
         self._data.update(self._prepare(params, **kwargs))

tls_requests/types.py

@@ -3,8 +3,8 @@
 """
 
 from http.cookiejar import CookieJar
-from typing import (IO, TYPE_CHECKING, Any, BinaryIO, Callable, List, Literal,
-                    Mapping, Optional, Sequence, Tuple, Union)
+from typing import (IO, TYPE_CHECKING, Any, BinaryIO, Callable, Dict, List,
+                    Literal, Mapping, Optional, Sequence, Set, Tuple, Union)
 from uuid import UUID
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -28,9 +28,9 @@
             Union[str, bytes],
             Union[
                 URL_ALLOWED_PARAMS,
-                list[URL_ALLOWED_PARAMS],
-                tuple[URL_ALLOWED_PARAMS],
-                set[URL_ALLOWED_PARAMS],
+                List[URL_ALLOWED_PARAMS],
+                Tuple[URL_ALLOWED_PARAMS],
+                Set[URL_ALLOWED_PARAMS],
             ],
         ],
     ]
@@ -43,7 +43,7 @@
 TLSSession = Union["TLSSession", None]
 TLSSessionId = Union[str, UUID]
 TLSPayload = Union[dict, str, bytes, bytearray]
-TLSCookiesTypes = Optional[List[dict[str, str]]]
+TLSCookiesTypes = Optional[List[Dict[str, str]]]
 TLSIdentifierTypes = Literal[
     "chrome_103",
     "chrome_104",
@@ -104,12 +104,7 @@
         "Headers",
         Mapping[str, str],
         Mapping[bytes, bytes],
-        List[list[str, str]],
-        List[list[bytes, bytes]],
-        List[tuple[str, str]],
-        List[tuple[bytes, bytes]],
-        List[set[str, str]],
-        List[set[bytes, bytes]],
+        List[Union[List[Union[str, bytes]], Tuple[Union[str, bytes]], Set[Union[str, bytes]]]],
     ]
 ]
 
@@ -119,12 +114,7 @@
         CookieJar,
         Mapping[str, str],
         Mapping[bytes, bytes],
-        List[list[str, str]],
-        List[list[bytes, bytes]],
-        List[tuple[str, str]],
-        List[tuple[bytes, bytes]],
-        List[set[str, str]],
-        List[set[bytes, bytes]],
+        List[Union[List[Union[str, bytes]], Tuple[Union[str, bytes]], Set[Union[str, bytes]]]],
     ]
 ]
 

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py{39,310,311,312,313}-default
+envlist = py{38,39,310,311,312,313}-default
 
 [testenv]
 deps = -r requirements-dev.txt