Provide User-Agent and Accept headers in URLFile requests

python/cog/types.py

@@ -278,7 +278,13 @@ def __wrapped__(self) -> Any:
             # is that the book keeping for closing the response needs to be
             # handled elsewhere. There's probably a better design for this
             # in the long term.
-            res = urllib.request.urlopen(url)  # noqa: S310
+            from . import __version__ as cog_version
+
+            req = urllib.request.Request(  # noqa: S310
+                url,
+                headers={"User-agent": f"cog/{cog_version}", "Accept": "*/*"},
+            )
+            res = urllib.request.urlopen(req)  # noqa: S310
             object.__setattr__(self, "__target__", res)
 
             return res

python/tests/server/test_clients.py

@@ -138,7 +138,9 @@ async def test_upload_files_with_url_file(urlopen_mock, respx_mock):
 
     assert uploader.call_count == 1
     assert urlopen_mock.call_count == 1
-    assert urlopen_mock.call_args[0][0] == "https://example.com/cdn/my_file.txt"
+    assert (
+        urlopen_mock.call_args[0][0].full_url == "https://example.com/cdn/my_file.txt"
+    )
 
 
 @pytest.mark.asyncio
@@ -164,4 +166,6 @@ async def test_upload_files_with_url_file_with_retry(urlopen_mock, respx_mock):
 
     assert uploader.call_count == 3
     assert urlopen_mock.call_count == 1
-    assert urlopen_mock.call_args[0][0] == "https://example.com/cdn/my_file.txt"
+    assert (
+        urlopen_mock.call_args[0][0].full_url == "https://example.com/cdn/my_file.txt"
+    )

python/tests/test_types.py

@@ -1,11 +1,12 @@
 import io
 import pickle
 import urllib.request
-from urllib.response import addinfourl
 from email.message import Message
 from unittest import mock
+from urllib.response import addinfourl
 
 import pytest
+from cog import __version__
 from cog.types import Secret, URLFile, get_filename_from_url, get_filename_from_urlopen
 
 
@@ -26,6 +27,19 @@ def test_urlfile_protocol_validation():
         URLFile("data:text/plain,hello")
 
 
+@mock.patch("urllib.request.urlopen", return_value=file_fixture("hello world"))
+def test_urlfile_headers(mock_urlopen: mock.Mock):
+    u = URLFile("https://example.com/some-path", filename="my_file.txt")
+    u.read()
+
+    assert mock_urlopen.call_count == 1
+
+    req: urllib.request.Request = mock_urlopen.call_args[0][0]
+    assert req.full_url == "https://example.com/some-path"
+    assert req.headers.get("User-agent") == f"cog/{__version__}"
+    assert req.headers.get("Accept") == "*/*"
+
+
 @mock.patch("urllib.request.urlopen", return_value=file_fixture("hello world"))
 def test_urlfile_custom_filename(mock_urlopen):
     u = URLFile("https://example.com/some-path", filename="my_file.txt")