Port zlib streams to zlib 1.3.2

.github/workflows/build.yml

@@ -319,12 +319,12 @@ jobs:
           - toolset: clang-win
             cxxstd: 14,17,20,latest
             addrmd: 32
-            os: windows-2025
+            os: windows-2025-no-aws
             cxxflags: "linkflags=/safeseh:no"
           - toolset: clang-win
             cxxstd: 14,17,20,latest
             addrmd: 64
-            os: windows-2025
+            os: windows-2025-no-aws
           - toolset: gcc
             cxxstd: "11,14"
             addrmd: 64

CHANGELOG.md

@@ -3,6 +3,12 @@ Version 362:
 * `http::ext_list` does not carry parameters from the previous extension into one that has none
 * `detail::base64::decoded_size` accounts for unpadded input
 * Fixed the `@tparam` name in the `buffers_iterator_type` documentation
+* `zlib::deflate_stream` and `zlib::inflate_stream` are ported from zlib 1.3.2
+* `zlib::deflate_stream::reset` rejects `windowBits` 8
+* `zlib::deflate_stream::write` returns `error::stream_error` for null buffer pointers
+* `zlib::deflate_stream` uses `memLevel` 8 by default
+* `zlib::inflate_stream::write` returns `error::stream_error` for null buffer pointers
+* `zlib::inflate_stream::write` errors from corrupt input are sticky until the stream is reset
 
 --------------------------------------------------------------------------------
 

include/boost/beast/websocket/detail/impl_base.hpp

@@ -24,6 +24,7 @@
 #include <boost/beast/core/error.hpp>
 #include <boost/beast/core/detail/clamp.hpp>
 #include <boost/asio/buffer.hpp>
+#include <algorithm>
 #include <cstdint>
 #include <memory>
 #include <stdexcept>
@@ -197,7 +198,23 @@ struct impl_base<true>
            (role == role_type::server &&
                 pmd_config_.client_no_context_takeover))
         {
-            pmd_->zi.clear();
+            /*  Not resetting the inflater for now, see
+                https://github.com/boostorg/beast/issues/3118
+
+                - RFC 7692 7.2.2 lets the receiver keep or drop the window;
+                  only the sender must drop it (7.2.1).
+                - Keeping it tolerates peers that agree to no context
+                  takeover but still refer back. Beast has done this since
+                  Boost 1.70: 8ea282e turned zi.reset() into zi.clear(),
+                  which was a no-op until the zlib 1.3.2 port.
+                - zi.reset() rejects such peers and keeps the window
+                  allocated; zi.clear() also frees it between messages
+                  (the memory saving the parameter exists for), at the
+                  cost of one allocation per message.
+                - Most implementations reset; Chromium, Firefox and a few
+                  libraries keep it. No known peer depends on keeping it.
+            */
+            // pmd_->zi.clear();
         }
     }
 
@@ -285,13 +302,20 @@ struct impl_base<true>
         {
             detail::pmd_normalize(pmd_config_);
             pmd_.reset(::new pmd_type);
+            /*  permessage-deflate allows a window of 8 bits (256 bytes),
+                which deflate does not support; zlib rejects it for raw
+                streams. Compress with a 9-bit window instead. This is
+                safe for the peer, because match distances never exceed
+                the window size minus 262, so a 256-byte inflate window
+                is never exceeded. The inflater keeps the negotiated size.
+            */
             if(role == role_type::client)
             {
                 pmd_->zi.reset(
                     pmd_config_.server_max_window_bits);
                 pmd_->zo.reset(
                     pmd_opts_.compLevel,
-                    pmd_config_.client_max_window_bits,
+                    (std::max)(pmd_config_.client_max_window_bits, 9),
                     pmd_opts_.memLevel,
                     zlib::Strategy::normal);
             }
@@ -301,7 +325,7 @@ struct impl_base<true>
                     pmd_config_.client_max_window_bits);
                 pmd_->zo.reset(
                     pmd_opts_.compLevel,
-                    pmd_config_.server_max_window_bits,
+                    (std::max)(pmd_config_.server_max_window_bits, 9),
                     pmd_opts_.memLevel,
                     zlib::Strategy::normal);
             }

include/boost/beast/zlib/deflate_stream.hpp

@@ -26,7 +26,7 @@ namespace zlib {
 
 // This is a derivative work based on Zlib, copyright below:
 /*
-    Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler
+    Copyright (C) 1995-2026 Jean-loup Gailly and Mark Adler
 
     This software is provided 'as-is', without any express or implied
     warranty.  In no event will the authors be held liable for any damages
@@ -94,13 +94,17 @@ class deflate_stream
         @param level Compression level from 0 to 9.
 
         @param windowBits The base two logarithm of the window size, or the
-        history buffer. It should be in the range 9..15.
+        history buffer. It should be in the range 9..15. A value of 8 is
+        rejected, as zlib does for raw deflate streams, because deflate
+        does not support a 256-byte window.
 
         @param memLevel How much memory should be allocated for the internal
         compression state, with level from from 1 to 9.
 
         @param strategy Strategy to tune the compression algorithm.
 
+        @throws std::invalid_argument if a parameter is out of range.
+
         @note Any unprocessed input or pending output from
         previous calls are discarded.
     */
@@ -305,18 +309,40 @@ class deflate_stream
         compression strategy. The interpretation of level and strategy
         is as in @ref reset. This can be used to switch between compression
         and straight copy of the input data, or to switch to a different kind
-        of input data requiring a different strategy. If the compression level
-        is changed, the input available so far is compressed with the old level
-        (and may be flushed); the new level will take effect only at the next
-        call of @ref write.
+        of input data requiring a different strategy. If the compression
+        approach (which is a function of the level) or the strategy is
+        changed, and if there have been any calls to @ref write since the
+        stream was reset, then the input available so far is compressed
+        with the old level and strategy using `write` with `Flush::block`.
+        There are three approaches for the compression levels 0, 1..3, and
+        4..9 respectively. The new level and strategy will take effect at
+        the next call of @ref write.
+
+        If a `write` with `Flush::block` is performed by `params`, and it
+        does not have enough output space to complete, then the parameter
+        change will not take effect. In this case, `params` can be called
+        again with the same parameters and more output space to try again.
+
+        In order to assure a change in the parameters on the first try, the
+        stream should be flushed using @ref write with `Flush::block` or
+        another flush request until `zs.avail_out` is not zero, before
+        calling `params`. Then no more input data should be provided before
+        the call to `params`. If this is done, the old level and strategy
+        will be applied to the data compressed before `params`, and the new
+        level and strategy will be applied to the data compressed after
+        `params`.
 
         Before the call of `params`, the stream state must be set as for a
-        call of @ref write, since the currently available input may have to be
-        compressed and flushed. In particular, `zs.avail_out` must be non-zero.
-
-        @return `Z_OK` if success, `Z_STREAM_ERROR` if the source stream state
-        was inconsistent or if a parameter was invalid, `error::need_buffers`
-        if `zs.avail_out` was zero.
+        call of @ref write, since the currently available input may have to
+        be compressed and flushed.
+
+        @return `error::stream_error` if the source stream state was
+        inconsistent or if a parameter was invalid, or `error::need_buffers`
+        if there was not enough output space to complete the compression of
+        the available input data before a change in the strategy or
+        approach. Note that in the case of `error::need_buffers`, the
+        parameters are not changed. `error::need_buffers` is not fatal, in
+        which case `params` can be retried with more output space.
     */
     void
     params(
@@ -335,11 +361,8 @@ class deflate_stream
         output. The bytes not provided would be due to the available
         output space having being consumed. The number of bits of output
         not provided are between 0 and 7, where they await more bits to
-        join them in order to fill out a full byte. If pending or bits
+        join them in order to fill out a full byte. If `value` or `bits`
         are `nullptr`, then those values are not set.
-
-        @return `Z_OK` if success, or `Z_STREAM_ERROR` if the source
-        stream state was inconsistent.
     */
     void
     pending(unsigned *value, int *bits)
@@ -359,7 +382,8 @@ class deflate_stream
         the output.
 
         @return `error::need_buffers` if there was not enough room in
-        the internal buffer to insert the bits.
+        the internal buffer to insert the bits, or if `bits` is negative
+        or greater than 16.
     */
     void
     prime(int bits, int value, error_code& ec)
@@ -383,30 +407,37 @@ class deflate_stream
 std::size_t
 deflate_upper_bound(std::size_t bytes);
 
-/*  For the default windowBits of 15 and memLevel of 8, this function returns
-    a close to exact, as well as small, upper bound on the compressed size.
-    They are coded as constants here for a reason--if the #define's are
-    changed, then this function needs to be changed as well.  The return
-    value for 15 and 8 only works for those exact settings.
-
-    For any setting other than those defaults for windowBits and memLevel,
-    the value returned is a conservative worst case for the maximum expansion
-    resulting from using fixed blocks instead of stored blocks, which deflate
-    can emit on compressed data for some combinations of the parameters.
-
-    This function could be more sophisticated to provide closer upper bounds for
-    every combination of windowBits and memLevel.  But even the conservative
-    upper bound of about 14% expansion does not seem onerous for output buffer
-    allocation.
+/*  Without the stream parameters, the tight bound for the default settings
+    cannot be used. This returns what zlib's deflateBound returns when it is
+    not given a stream: the larger of two worst case bounds, plus the size
+    of a wrapper.
+
+    The ~4% bound derives from the overhead of stored blocks of 127 bytes
+    (the worst case memLevel == 1), five bytes of header for each block.
+    The larger expansion of ~13% results from a window size less than or
+    equal to the symbols buffer size (windowBits <= memLevel + 7). In that
+    case some of the data being compressed may have slid out of the sliding
+    window, impeding a stored block from being emitted. Then the only choice
+    is a fixed or dynamic block, where a fixed block limits the maximum
+    expansion to 9 bits per 8-bit byte, plus 10 bits for every block. The
+    smallest block size for which this can occur is 255 (memLevel == 2).
+
+    Shifts are used to approximate divisions, for speed.
 */
 inline
 std::size_t
 deflate_upper_bound(std::size_t bytes)
 {
-    return bytes +
-        ((bytes + 7) >> 3) +
-        ((bytes + 63) >> 6) + 5 +
-        6;
+    std::size_t fixedlen = bytes + (bytes >> 3) + (bytes >> 8) +
+        (bytes >> 9) + 4;
+    if(fixedlen < bytes)
+        fixedlen = static_cast<std::size_t>(-1);
+    std::size_t storelen = bytes + (bytes >> 5) + (bytes >> 7) +
+        (bytes >> 11) + 7;
+    if(storelen < bytes)
+        storelen = static_cast<std::size_t>(-1);
+    std::size_t const bound = fixedlen > storelen ? fixedlen : storelen;
+    return bound + 18 < bound ? static_cast<std::size_t>(-1) : bound + 18;
 }
 
 } // zlib

include/boost/beast/zlib/detail/bitstream.hpp

@@ -95,6 +95,11 @@ class bitstream
     void
     fill_8(FwdIt& it);
 
+    // fill 8 bits, returns false if there is no input
+    template<class FwdIt>
+    bool
+    fill_8(FwdIt& first, FwdIt const& last);
+
     // fill 16 bits, unchecked
     template<class FwdIt>
     void
@@ -147,6 +152,18 @@ fill_8(FwdIt& it)
     n_ += 8;
 }
 
+template<class FwdIt>
+bool
+bitstream::
+fill_8(FwdIt& first, FwdIt const& last)
+{
+    if(first == last)
+        return false;
+    v_ += static_cast<value_type>(*first++) << n_;
+    n_ += 8;
+    return true;
+}
+
 template<class FwdIt>
 void
 bitstream::

include/boost/beast/zlib/detail/deflate_stream.hpp

@@ -8,7 +8,7 @@
 //
 // This is a derivative work based on Zlib, copyright below:
 /*
-    Copyright (C) 1995-2018 Jean-loup Gailly and Mark Adler
+    Copyright (C) 1995-2026 Jean-loup Gailly and Mark Adler
 
     This software is provided 'as-is', without any express or implied
     warranty.  In no event will the authors be held liable for any damages
@@ -42,7 +42,6 @@
 #include <boost/beast/zlib/detail/ranges.hpp>
 #include <boost/assert.hpp>
 #include <boost/config.hpp>
-#include <boost/optional.hpp>
 #include <boost/throw_exception.hpp>
 #include <cstdint>
 #include <cstdlib>
@@ -110,7 +109,7 @@ class deflate_stream
     static std::uint8_t constexpr max_mem_level = 9;
 
     // Default memLevel
-    static std::uint8_t constexpr def_mem_level = max_mem_level;
+    static std::uint8_t constexpr def_mem_level = 8;
 
     /*  Note: the deflate() code requires max_lazy >= minMatch and max_chain >= 4
         For deflate_fast() (levels <= 3) good is ignored and lazy has a different
@@ -126,6 +125,9 @@ class deflate_stream
     // Matches of length 3 are discarded if their distance exceeds kTooFar
     static std::size_t constexpr kTooFar = 4096;
 
+    // Maximum stored block length in deflate format (not including header)
+    static unsigned constexpr kMaxStored = 65535;
+
     /*  Minimum amount of lookahead, except at the end of the input file.
         See deflate.c for comments about the minMatch+1.
     */
@@ -259,8 +261,12 @@ class deflate_stream
         pending_buf_size_;          // size of pending_buf
     Byte* pending_out_;             // next pending byte to output to the stream
     uInt pending_;                  // nb of bytes in the pending buffer
-    boost::optional<Flush>
-        last_flush_;                // value of flush param for previous deflate call
+
+    /*  Value of the flush parameter for the previous call to write,
+        or -1 to avoid returning need_buffers on the next call, or -2
+        if write has not been called since the stream was reset.
+    */
+    int last_flush_;
 
     uInt w_size_;                   // LZ77 window size (32K by default)
     uInt w_bits_;                   // log2(w_size)  (8..16)
@@ -617,8 +623,7 @@ class deflate_stream
     BOOST_BEAST_DECL std::size_t doUpperBound (std::size_t sourceLen) const;
     BOOST_BEAST_DECL void doTune              (int good_length, int max_lazy, int nice_length, int max_chain);
     BOOST_BEAST_DECL void doParams            (z_params& zs, int level, Strategy strategy, error_code& ec);
-    BOOST_BEAST_DECL void doWrite             (z_params& zs, boost::optional<Flush> flush, error_code& ec);
-    BOOST_BEAST_DECL void doDictionary        (Byte const* dict, uInt dictLength, error_code& ec);
+    BOOST_BEAST_DECL void doWrite             (z_params& zs, Flush flush, error_code& ec);
     BOOST_BEAST_DECL void doPrime             (int bits, int value, error_code& ec);
     BOOST_BEAST_DECL void doPending           (unsigned* value, int* bits);
 
@@ -637,7 +642,6 @@ class deflate_stream
     BOOST_BEAST_DECL int  detect_data_type    ();
     BOOST_BEAST_DECL void bi_windup           ();
     BOOST_BEAST_DECL void bi_flush            ();
-    BOOST_BEAST_DECL void copy_block          (char *buf, unsigned len, int header);
 
     BOOST_BEAST_DECL void tr_init             ();
     BOOST_BEAST_DECL void tr_align            ();
@@ -647,6 +651,7 @@ class deflate_stream
     BOOST_BEAST_DECL void tr_tally_lit        (std::uint8_t c, bool& flush);
 
     BOOST_BEAST_DECL void tr_flush_block      (z_params& zs, char *buf, std::uint32_t stored_len, int last);
+    BOOST_BEAST_DECL void slide_hash          ();
     BOOST_BEAST_DECL void fill_window         (z_params& zs);
     BOOST_BEAST_DECL void flush_pending       (z_params& zs);
     BOOST_BEAST_DECL void flush_block         (z_params& zs, bool last);