Switch to chardet for stable behavior

charset_normalizer has stability issues Reference: jawah/charset_normalizer#520 Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>
aboutcode-org · Sep 5, 2024 · 1203172 · 1203172
1 parent 9127939
commit 1203172
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 5 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -1,7 +1,7 @@
 attrs==22.2.0
 beautifulsoup4==4.11.2
 certifi==2022.12.7
-charset-normalizer==3.0.1
+chardet==5.2.0
 click==8.1.3
 idna==3.4
 pip==23.0

diff --git a/setup.cfg b/setup.cfg
@@ -41,9 +41,9 @@ python_requires = >=3.8
 
 install_requires =
     attrs >= 18.1, !=20.1.0
-    Beautifulsoup4 >= 4.0.0
+    Beautifulsoup4[chardet] >= 4.0.0
     click >= 6.7, !=7.0
-    requests >= 2.7.0
+    requests[use_chardet_on_py3] >= 2.7.0
     saneyaml >= 0.5.2
     text_unidecode >= 1.0
 

diff --git a/tests/test_paths.py b/tests/test_paths.py
@@ -76,8 +76,11 @@ def test_safe_path_posix_style_french_char(self):
 
     def test_safe_path_posix_style_chinese_char(self):
         test = paths.safe_path(b'/includes/webform.compon\xd2\xaants.inc/')
-        expected = 'includes/webform.componNSnts.inc'
-        assert test == expected
+        expected = [
+            'includes/webform.componNSnts.inc',
+            'includes/webform.componS_nts.inc',
+        ]
+        assert test in expected
 
     def test_safe_path_windows_style_dots(self):
         test = paths.safe_path('\\includes\\..\\webform.components.inc\\')