check for bad utf

CHANGELOG.md

@@ -2,7 +2,7 @@
 
 ## Unreleased
 
-- dump encoded filename in exception where there is an error in decoding it [ref](https://github.com/boredazfcuk/docker-icloudpd/issues/641)
+- dump encoded filename in exception when there is an error in decoding it [#935](https://github.com/icloud-photos-downloader/icloud_photos_downloader/issues/935) and [ref](https://github.com/boredazfcuk/docker-icloudpd/issues/641)
 
 ## 1.23.1 (2024-08-22)
 

src/foundation/__init__.py

@@ -38,7 +38,9 @@ def _internal(inp: bytes) -> str:
 T_out = TypeVar("T_out")
 
 
-def _capture_param_in_exception(func: Callable[[T_in], T_out]) -> Callable[[T_in], T_out]:
+def wrap_param_in_exception(func: Callable[[T_in], T_out]) -> Callable[[T_in], T_out]:
+    """Decorator to wrap excpetion into ValueError with the dump of the input parameter"""
+
     def _internal(input: T_in) -> T_out:
         try:
             return func(input)

src/pyicloud_ipd/services/photos.py

@@ -11,7 +11,7 @@
 import typing
 
 from requests import Response
-from foundation import _capture_param_in_exception, bytes_decode
+from foundation import wrap_param_in_exception, bytes_decode
 from pyicloud_ipd.asset_version import AssetVersion
 from pyicloud_ipd.exceptions import PyiCloudServiceNotActivatedException
 from pyicloud_ipd.exceptions import PyiCloudAPIResponseException
@@ -613,8 +613,8 @@ def id(self) -> str:
     def filename(self) -> str:
         fields = self._master_record['fields']
         if 'filenameEnc' in fields and 'value' in fields['filenameEnc']:
-            _decode_base64 = _capture_param_in_exception(base64.b64decode)
-            _decode_bytes = _capture_param_in_exception(bytes_decode('utf-8'))
+            _decode_base64 = wrap_param_in_exception(base64.b64decode)
+            _decode_bytes = wrap_param_in_exception(bytes_decode('utf-8'))
             _filename = compose(
                 self._service.filename_cleaner,
                 compose(

tests/test_download_photos.py

@@ -2086,7 +2086,7 @@ def test_download_raw_photos_policy_as_is(self) -> None:
 
         assert result.exit_code == 0
 
-    def test_download_bad_filename_encoding(self) -> None:
+    def test_download_bad_filename_base64_encoding(self) -> None:
         base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3])
 
         files_to_create = [
@@ -2105,7 +2105,7 @@ def test_download_bad_filename_encoding(self) -> None:
             self.assertEqual,
             self.vcr_path,
             base_dir,
-            "listing_photos_bad_filename_encoding.yml",
+            "listing_photos_bad_filename_base64_encoding.yml",
             files_to_create,
             files_to_download,
             [
@@ -2124,4 +2124,44 @@ def test_download_bad_filename_encoding(self) -> None:
         )
 
         self.assertIsInstance(result.exception, ValueError)
-        # ValueError("Invalid Input: 'aS9uIHY6YQBsKmk/ZFxwPGE+dCJofC5KUE'")
+        # self.assertEqual(result.exception, ValueError("Invalid Input: 'aS9uIHY6YQBsKmk/ZFxwPGE+dCJofC5KUE'"))
+
+    def test_download_bad_filename_utf8_encoding(self) -> None:
+        base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3])
+
+        files_to_create = [
+            ("2018/07/30", "IMG_7408.JPG", 1151066),
+            ("2018/07/30", "IMG_7407.JPG", 656257),
+        ]
+
+        files_to_download: List[Tuple[str, str]] = [
+            # <>:"/\|?*  -- windows
+            # / & \0x00 -- linux
+            # aS9uIHY6YQBsKmk/ZFxwPGE+dCJofC5KUE -> abcdefgh
+            # ("2018/07/31", "i_n v_a_l_i_d_p_a_t_h_.JPG")
+        ]
+
+        data_dir, result = run_icloudpd_test(
+            self.assertEqual,
+            self.vcr_path,
+            base_dir,
+            "listing_photos_bad_filename_utf8_encoding.yml",
+            files_to_create,
+            files_to_download,
+            [
+                "--username",
+                "jdoe@gmail.com",
+                "--password",
+                "password1",
+                "--recent",
+                "5",
+                "--skip-videos",
+                "--skip-live-photos",
+                "--no-progress-bar",
+                "--threads-num",
+                "1",
+            ],
+        )
+
+        self.assertIsInstance(result.exception, ValueError)
+        # self.assertEqual(result.exception, ValueError("Invalid Input: b'i\\xb7\\x1dy\\xf8!'"))

tests/test_download_photos_id.py

@@ -2074,7 +2074,7 @@ def test_download_raw_photos_policy_as_is_name_id7(self) -> None:
 
         assert result.exit_code == 0
 
-    def test_download_bad_filename_encoding_name_id7(self) -> None:
+    def test_download_bad_filename_base64_encoding_name_id7(self) -> None:
         base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3])
 
         files_to_create = [
@@ -2093,7 +2093,7 @@ def test_download_bad_filename_encoding_name_id7(self) -> None:
             self.assertEqual,
             self.vcr_path,
             base_dir,
-            "listing_photos_bad_filename_encoding.yml",
+            "listing_photos_bad_filename_base64_encoding.yml",
             files_to_create,
             files_to_download,
             [
@@ -2106,12 +2106,50 @@ def test_download_bad_filename_encoding_name_id7(self) -> None:
                 "--skip-videos",
                 "--skip-live-photos",
                 "--no-progress-bar",
-                "--threads-num",
-                "1",
                 "--file-match-policy",
                 "name-id7",
             ],
         )
 
         self.assertIsInstance(result.exception, ValueError)
         # ValueError("Invalid Input: 'aS9uIHY6YQBsKmk/ZFxwPGE+dCJofC5KUE'")
+
+    def test_download_bad_filename_utf8_encoding_name_id7(self) -> None:
+        base_dir = os.path.join(self.fixtures_path, inspect.stack()[0][3])
+
+        files_to_create = [
+            ("2018/07/30", "IMG_7408_QVI4T2l.JPG", 1151066),
+            ("2018/07/30", "IMG_7407_QVovd0F.JPG", 656257),
+        ]
+
+        files_to_download: List[Tuple[str, str]] = [
+            # <>:"/\|?*  -- windows
+            # / & \0x00 -- linux
+            # aS9uIHY6YQBsKmk/ZFxwPGE+dCJofC5KUE -> abcdefgh
+            # ("2018/07/31", "i_n v_a_l_i_d_p_a_t_h_.JPG")
+        ]
+
+        data_dir, result = run_icloudpd_test(
+            self.assertEqual,
+            self.vcr_path,
+            base_dir,
+            "listing_photos_bad_filename_utf8_encoding.yml",
+            files_to_create,
+            files_to_download,
+            [
+                "--username",
+                "jdoe@gmail.com",
+                "--password",
+                "password1",
+                "--recent",
+                "5",
+                "--skip-videos",
+                "--skip-live-photos",
+                "--no-progress-bar",
+                "--file-match-policy",
+                "name-id7",
+            ],
+        )
+
+        self.assertIsInstance(result.exception, ValueError)
+        # self.assertEqual(result.exception, ValueError("Invalid Input: b'i\\xb7\\x1dy\\xf8!'"))