Merge pull request #96 from sergey-dryabzhinsky/issue-95-update-zstd-155

For issues #95 and #94

.github/workflows/build-wheels.yml

@@ -120,7 +120,7 @@ jobs:
             python -m unittest discover -v -s {package}
 
       - name: Build ${{ matrix.os.name }} wheels and test (new)
-        uses: joerick/cibuildwheel@v2.11.1
+        uses: joerick/cibuildwheel@v2.12.1
         if: matrix.cibw.group == 'new'
         with:
           output-dir: dist

PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 1.1
 Name: zstd
-Version: 1.5.4.1
+Version: 1.5.5.1
 Summary: Simple python bindings to Yann Collet ZSTD compression library
 Home-page: https://github.com/sergey-dryabzhinsky/python-zstd
 Author: Sergey Dryabzhinsky
 Author-email: sergey.dryabzhinsky@gmail.com
 License: BSD
-Download-URL: https://github.com/sergey-dryabzhinsky/python-zstd/archive/v1.5.4.1.tar.gz
+Download-URL: https://github.com/sergey-dryabzhinsky/python-zstd/archive/v1.5.5.1.tar.gz
 Description: Simple ZSTandarD bindings for Python
 Keywords: zstd,zstandard,compression
 Platform: POSIX

README.rst

@@ -2,7 +2,7 @@
 python-zstd
 =============
 
-.. |releaseW| image:: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/build-wheels.yml/badge.svg?branch=v1.5.4.0
+.. |releaseW| image:: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/build-wheels.yml/badge.svg?branch=v1.5.5.1
     :target: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/build-wheels.yml
 
 .. |masterW| image:: https://github.com/sergey-dryabzhinsky/python-zstd/actions/workflows/build-wheels.yml/badge.svg
@@ -136,13 +136,15 @@ ZSTD_compress (data[, level, threads]): string|bytes
   Max number of threads:
   - 32bit system: 64
   - 64bit system: 256
-  If provided bigger number - siletly set maximum number (since 1.5.4.1)
+  If provided bigger number - silemtly set maximum number (since 1.5.4.1)
 
   Since: 0.1
 
 ZSTD_uncompress (data): string|bytes
   Function, decompress input compressed data block, return decompressed block, or raises Error.
 
+  Support compressed data with multiple/concatenated frames (blocks) (since 1.5.5.1).
+
   Params:
 
   * **data**: string|bytes - input compressed data block, length limited by 2Gb by Python API

setup.py

@@ -8,7 +8,7 @@
 from setuptools.command.build_ext import build_ext
 
 # ZSTD version
-VERSION = (1, 5, 4,)
+VERSION = (1, 5, 5,)
 VERSION_STR = ".".join([str(x) for x in VERSION])
 
 # Package version

src/python-zstd.c

@@ -138,9 +138,9 @@ static PyObject *py_zstd_uncompress(PyObject* self, PyObject *args)
 {
 
     PyObject    *result;
-    const char  *source;
-    Py_ssize_t  source_size;
-    uint64_t    dest_size;
+    const char  *source, *src;
+    Py_ssize_t  source_size, ss, seek_frame;
+    uint64_t    dest_size, frame_size;
     char        error = 0;
     size_t      cSize;
 
@@ -157,18 +157,41 @@ static PyObject *py_zstd_uncompress(PyObject* self, PyObject *args)
         PyErr_Format(ZstdError, "Input data invalid or missing content size in frame header.");
         return NULL;
     }
+
+	// Find real dest_size across multiple frames
+	ss = source_size;
+	seek_frame = ss - 1;
+	src = source;
+	while (seek_frame < ss) {
+		seek_frame = ZSTD_findFrameCompressedSize(src, ss);
+		if (ZSTD_isError(seek_frame)) break;
+		src += seek_frame;
+		ss -= seek_frame;
+		if (ss <=0) break;
+		frame_size = (uint64_t) ZSTD_getFrameContentSize(src, ss);
+		if (ZSTD_isError(frame_size)) break;
+		dest_size += frame_size;
+	}
+
     result = PyBytes_FromStringAndSize(NULL, dest_size);
 
     if (result != NULL) {
         char *dest = PyBytes_AS_STRING(result);
 
         Py_BEGIN_ALLOW_THREADS
+		// get real dest_size
         cSize = ZSTD_decompress(dest, dest_size, source, source_size);
         Py_END_ALLOW_THREADS
 
         if (ZSTD_isError(cSize)) {
-            PyErr_Format(ZstdError, "Decompression error: %s", ZSTD_getErrorName(cSize));
-            error = 1;
+			const char *errStr = ZSTD_getErrorName(cSize);
+/*			if (strstr(errStr, "buffer is too small") != NULL) {
+				// reroll decompression with bigger buffer
+				error = 2;
+			} else {*/
+            	PyErr_Format(ZstdError, "Decompression error: %s", errStr);
+            	error = 1;
+//			}
         } else if (cSize != dest_size) {
             PyErr_Format(ZstdError, "Decompression error: length mismatch -> decomp %lu != %lu [header]", (uint64_t)cSize, dest_size);
             error = 1;
@@ -180,6 +203,10 @@ static PyObject *py_zstd_uncompress(PyObject* self, PyObject *args)
         result = NULL;
     }
 
+	if (!error) {
+    	Py_SET_SIZE(result, cSize);
+	}
+
     return result;
 }
 

tests/base.py

@@ -40,10 +40,10 @@ def raise_skip(msg):
 
 class BaseTestZSTD(unittest.TestCase):
 
-    VERSION = "1.5.4"
-    VERSION_INT = 10504
+    VERSION = "1.5.5"
+    VERSION_INT = 10505
     VERSION_INT_MIN = 1 * 100*100 + 0 * 1*100 + 0
-    PKG_VERSION = "1.5.4.1"
+    PKG_VERSION = "1.5.5.1"
 
     def helper_version(self):
         self.assertEqual(self.PKG_VERSION, zstd.version())
@@ -131,3 +131,16 @@ def helper_compression_empty_string(self):
 
         for level in range(0, 20):
             self.assertEqual(data, zstd.decompress(zstd.compress(data, level + 1)))
+
+    def helper_compression_multiple_blocks(self):
+        # https://github.com/sergey-dryabzhinsky/python-zstd/issues/94
+        # An conctenaed blocks should be able to be decompressed
+        if sys.hexversion < 0x03000000:
+            import codecs
+            data = codecs.decode("28b52ffd200631000068656c6c6f0a28b52ffd2006310000776f726c640a", 'hex_codec')
+            odata = "hello\nworld\n"
+        else:
+            data = bytes.fromhex("28b52ffd200631000068656c6c6f0a28b52ffd2006310000776f726c640a")
+            odata = b"hello\nworld\n"
+
+        self.assertEqual(odata, zstd.decompress(data))

tests/test_compress.py

@@ -39,9 +39,13 @@ def test_compression_level6(self):
 
     def test_compression_level20(self):
         BaseTestZSTD.helper_compression_level20(self)
-    
+
     def test_compression_empty_string(self):
         BaseTestZSTD.helper_compression_empty_string(self)
 
+    def test_compression_multiple_blocks(self):
+        BaseTestZSTD.helper_compression_multiple_blocks(self)
+
+
 if __name__ == '__main__':
     unittest.main()