Merge pull request #12 from Eossu/develop

Develop

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [2020.07.15 - 2020.07.15
+
+- Fixed a bug where the calculated total size of the compression was wrong.
+
 ## [2020.07.13] - 2020.07.13
 
 - Add the posibility to batched compressions instead of adding a full list to the CompressedJsonList.

src/gzip_utils/_compressed_json_list.py

@@ -123,7 +123,7 @@ def get_compressed_json_list(self, json_data: MutableSequence[Union[str, dict]])
         self._gzip_stream.write(b"[")
 
         for org_data in json_data:
-            if self._compress(org_data):
+            if self._compress(org_data):  # pragma: no cover
                 json_data.remove(org_data)
 
         self._gzip_stream.write(b"]")
@@ -156,7 +156,7 @@ def _compress(self, org_data: Union[str, dict]) -> bool:
         return True
 
     def _check_compression(self, data_bytes: bytes) -> bool:
-        stream_size = self._gzip_stream.size  # type: ignore
+        stream_size = self._byte_stream.getbuffer().nbytes
         if (stream_size + len(data_bytes) + self._unzipped_chars) > self._max_compressed_size:
             self._gzip_stream.flush()
             self._unzipped_chars = 0 + self._gzip_metadata_size

tests/conftest.py

@@ -45,12 +45,12 @@
 
 @pytest.fixture
 def str_list() -> MutableSequence[str]:
-    return json_data_str
+    return json_data_str.copy()
 
 
 @pytest.fixture
 def dict_list() -> MutableSequence[dict]:
-    return json_data_dict
+    return json_data_dict.copy()
 
 
 @pytest.fixture

tests/test_compressed_json_list.py

@@ -42,7 +42,7 @@ def test_compress_str_list(comp: CompressedJsonList, str_list: MutableSequence[U
 
     assert comp.compressed_size == len(data)
     assert comp.compression_ratio > 0
-    assert comp.uncompressed_size == 352
+    assert comp.uncompressed_size == 440
     assert comp.compression_limit == 400
 
 
@@ -60,7 +60,7 @@ def test_compress_dict_list(comp: CompressedJsonList, dict_list: MutableSequence
 
     assert comp.compressed_size == len(data)
     assert comp.compression_ratio > 0
-    assert comp.uncompressed_size == 352
+    assert comp.uncompressed_size == 440
     assert comp.compression_limit == 400
 
 
@@ -78,7 +78,7 @@ def test_compress_mixed_list(comp: CompressedJsonList, mixed_list: MutableSequen
 
     assert comp.compressed_size == len(data)
     assert comp.compression_ratio > 0
-    assert comp.uncompressed_size == 352
+    assert comp.uncompressed_size == 440
     assert comp.compression_limit == 400
 
 
@@ -99,6 +99,18 @@ def test_raise_value_error(comp: CompressedJsonList):
         comp.get_compressed_json_list(data)  # type: ignore
 
 
+def test_compress_list_get_full(comp: CompressedJsonList, str_list: MutableSequence[Union[str, dict]]):
+
+    comp.compression_limit = 300
+
+    compressed = list()
+    while len(str_list) > 0:
+        data = comp.get_compressed_json_list(str_list)
+        compressed.append(data)
+
+    assert len(compressed) > 1
+
+
 # ------------------------------------------------------------------------------------------------------------------------------
 # Compress a list of strings og dicts
 # ------------------------------------------------------------------------------------------------------------------------------