CHIA-590: Add in connect timeout to DL http download (#18339)

* Add in connect timeout to DL http download

* Update mock function

* Consolidate to using aiohttp.ClientTimeout

* Revert "Consolidate to using aiohttp.ClientTimeout"

This reverts commit db3e77c.

* Consolidate to using aiohttp.ClientTimeout

* Fix up default value with proper use of default_factory

chia/_tests/core/data_layer/test_data_store.py

@@ -1357,7 +1357,7 @@ async def mock_http_download(
         filename: str,
         proxy_url: str,
         server_info: ServerInfo,
-        timeout: int,
+        timeout: aiohttp.ClientTimeout,
         log: logging.Logger,
     ) -> None:
         if error:
@@ -1373,7 +1373,7 @@ async def mock_http_download(
             root_hashes=[bytes32.random(seeded_random)],
             server_info=sinfo,
             client_foldername=tmp_path,
-            timeout=15,
+            timeout=aiohttp.ClientTimeout(total=15, sock_connect=5),
             log=log,
             proxy_url="",
             downloader=None,
@@ -1396,7 +1396,7 @@ async def mock_http_download(
             root_hashes=[bytes32.random(seeded_random)],
             server_info=sinfo,
             client_foldername=tmp_path,
-            timeout=15,
+            timeout=aiohttp.ClientTimeout(total=15, sock_connect=5),
             log=log,
             proxy_url="",
             downloader=None,
@@ -1903,7 +1903,7 @@ async def test_insert_from_delta_file_correct_file_exists(
         root_hashes=root_hashes,
         server_info=sinfo,
         client_foldername=tmp_path,
-        timeout=15,
+        timeout=aiohttp.ClientTimeout(total=15, sock_connect=5),
         log=log,
         proxy_url="",
         downloader=None,
@@ -1962,7 +1962,7 @@ async def test_insert_from_delta_file_incorrect_file_exists(
         root_hashes=[incorrect_root_hash],
         server_info=sinfo,
         client_foldername=tmp_path,
-        timeout=15,
+        timeout=aiohttp.ClientTimeout(total=15, sock_connect=5),
         log=log,
         proxy_url="",
         downloader=None,

chia/data_layer/data_layer.py

@@ -3,6 +3,7 @@
 import asyncio
 import contextlib
 import dataclasses
+import functools
 import json
 import logging
 import os
@@ -124,6 +125,9 @@ class DataLayer:
     _wallet_rpc: Optional[WalletRpcClient] = None
     subscription_lock: asyncio.Lock = dataclasses.field(default_factory=asyncio.Lock)
     subscription_update_concurrency: int = 5
+    client_timeout: aiohttp.ClientTimeout = dataclasses.field(
+        default_factory=functools.partial(aiohttp.ClientTimeout, total=45, sock_connect=5)
+    )
 
     @property
     def server(self) -> ChiaServer:
@@ -185,6 +189,9 @@ def create(
             maximum_full_file_count=config.get("maximum_full_file_count", 1),
             subscription_update_concurrency=config.get("subscription_update_concurrency", 5),
             unsubscribe_data_queue=[],
+            client_timeout=aiohttp.ClientTimeout(
+                total=config.get("client_timeout", 45), sock_connect=config.get("connect_timeout", 5)
+            ),
         )
 
         self.db_path.parent.mkdir(parents=True, exist_ok=True)
@@ -586,7 +593,6 @@ async def fetch_and_validate(self, store_id: bytes32) -> None:
                 max_generation=singleton_record.generation,
             )
             try:
-                timeout = self.config.get("client_timeout", 15)
                 proxy_url = self.config.get("proxy_url", None)
                 success = await insert_from_delta_file(
                     self.data_store,
@@ -595,7 +601,7 @@ async def fetch_and_validate(self, store_id: bytes32) -> None:
                     [record.root for record in reversed(to_download)],
                     server_info,
                     self.server_files_location,
-                    timeout,
+                    self.client_timeout,
                     self.log,
                     proxy_url,
                     await self.get_downloader(store_id, url),

chia/data_layer/download_data.py

@@ -145,7 +145,7 @@ async def insert_from_delta_file(
     root_hashes: List[bytes32],
     server_info: ServerInfo,
     client_foldername: Path,
-    timeout: int,
+    timeout: aiohttp.ClientTimeout,
     log: logging.Logger,
     proxy_url: str,
     downloader: Optional[PluginRemote],
@@ -164,7 +164,14 @@ async def insert_from_delta_file(
             if downloader is None:
                 # use http downloader - this raises on any error
                 try:
-                    await http_download(client_foldername, filename, proxy_url, server_info, timeout, log)
+                    await http_download(
+                        client_foldername,
+                        filename,
+                        proxy_url,
+                        server_info,
+                        timeout,
+                        log,
+                    )
                 except (asyncio.TimeoutError, aiohttp.ClientError):
                     new_server_info = await data_store.server_misses_file(store_id, server_info, timestamp)
                     log.info(
@@ -252,7 +259,7 @@ async def http_download(
     filename: str,
     proxy_url: str,
     server_info: ServerInfo,
-    timeout: int,
+    timeout: aiohttp.ClientTimeout,
     log: logging.Logger,
 ) -> None:
     """
@@ -262,7 +269,10 @@ async def http_download(
     async with aiohttp.ClientSession() as session:
         headers = {"accept-encoding": "gzip"}
         async with session.get(
-            server_info.url + "/" + filename, headers=headers, timeout=timeout, proxy=proxy_url
+            server_info.url + "/" + filename,
+            headers=headers,
+            timeout=timeout,
+            proxy=proxy_url,
         ) as resp:
             resp.raise_for_status()
             size = int(resp.headers.get("content-length", 0))

chia/util/initial-config.yaml

@@ -615,7 +615,8 @@ data_layer:
   # The location where the server files will be stored.
   server_files_location: "data_layer/db/server_files_location_CHALLENGE"
   # The timeout for the client to download a file from a server
-  client_timeout: 15
+  client_timeout: 45
+  connect_timeout: 5
   # If you need use a proxy for download data you can use this setting sample
   # proxy_url: http://localhost:8888