Update Brotli to v1.0.0

19dc93 (01 Jun 2017) to c60563 (21 Sep 2017) Bug: Change-Id: Ice95736dbaf3574fc4bb75289c60723ef019a16b Reviewed-on: https://chromium-review.googlesource.com/677395 Commit-Queue: Eugene Kliuchnikov <eustas@chromium.org> Reviewed-by: Kenichi Ishibashi <bashi@chromium.org> Cr-Commit-Position: refs/heads/master@{#503724}
marcosholgado · Sep 22, 2017 · 805bc58 · 805bc58
1 parent 1245654
commit 805bc58
Show file tree

Hide file tree

Showing 33 changed files with 431 additions and 543 deletions.
diff --git a/third_party/brotli/README.chromium b/third_party/brotli/README.chromium
@@ -1,6 +1,6 @@
 Name: Brotli
 URL: https://github.com/google/brotli
-Version: 19dc934e391752b78f78eca60c2c1ca5f6ac647b
+Version: c60563591a9a86196f19987c81dde4384a088861
 License: MIT
 License File: LICENSE
 Security Critical: yes
@@ -14,8 +14,7 @@ Local Modifications:
 - This only includes the contents of c/ directory, the README.md and the LICENSE
   files.
 - Auxiliary fuzzer runners removed from fuzz/
-- Line 19 of enc/cluster_inc.h was modified to eliminate build error on Windows.
-- Line 700 of tools/brotli.c was modified to eliminate build error on Windows.
 - common/dictionary.bin: Removed.
+- enc/hash_longest_match_quickly_inc.h:223: calm down MSVC
 - BUILD.gn: Added.
 - brotli.gni: Added.
diff --git a/third_party/brotli/README.md b/third_party/brotli/README.md
@@ -8,21 +8,22 @@ and 2nd order context modeling, with a compression ratio comparable to the best
 currently available general-purpose compression methods. It is similar in speed
 with deflate but offers more dense compression.
 
-The specification of the Brotli Compressed Data Format is defined in [RFC 7932](https://www.ietf.org/rfc/rfc7932.txt).
+The specification of the Brotli Compressed Data Format is defined in [RFC 7932](https://tools.ietf.org/html/rfc7932).
 
 Brotli is open-sourced under the MIT License, see the LICENSE file.
 
 Brotli mailing list:
 https://groups.google.com/forum/#!forum/brotli
 
-[![Build Status](https://travis-ci.org/google/brotli.svg?branch=master)](https://travis-ci.org/google/brotli)
+[![TravisCI Build Status](https://travis-ci.org/google/brotli.svg?branch=master)](https://travis-ci.org/google/brotli)
+[![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/github/google/brotli?branch=master&svg=true)](https://ci.appveyor.com/project/szabadka/brotli)
 
 ### Build instructions
 
 #### Autotools-style CMake
 
 [configure-cmake](https://github.com/nemequ/configure-cmake) is an
-autotools-style configure script for CMake-based projects.
+autotools-style configure script for CMake-based projects (not supported on Windows).
 
 The basic commands to build, test and install brotli are:
 
@@ -32,52 +33,49 @@ The basic commands to build, test and install brotli are:
     $ make test
     $ make install
 
-To build static libraries use `--disable-shared-libs` argument:
-
-    $ mkdir out-static && cd out-static
-    $ ../configure-cmake --disable-shared-libs
-    $ make install
-
 #### Bazel
 
 See [Bazel](http://www.bazel.build/)
 
 #### CMake
 
-The basic commands to build, test and install brotli are:
+The basic commands to build and install brotli are:
 
     $ mkdir out && cd out
-    $ cmake ..
-    $ make
-    $ make test
-    $ make install
+    $ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=./installed ..
+    $ cmake --build . --config Release --target install
 
-You can use other [CMake](https://cmake.org/) configuration. For example, to
-build static libraries:
-
-    $ mkdir out-static && cd out-static
-    $ cmake .. -DBUILD_SHARED_LIBS=OFF
-    $ make
+You can use other [CMake](https://cmake.org/) configuration.
 
 #### Premake5
 
 See [Premake5](https://premake.github.io/)
 
 #### Python
 
-To install the Python module from source, run the following:
+To install the latest release of the Python module, run the following:
+
+    $ pip install brotli
+
+To install the tip-of-the-tree version, run:
 
-    $ python setup.py install
+    $ pip install --upgrade git+https://github.com/google/brotli
 
-See the [Python readme](python/README.md) for more details on testing
-and development.
+See the [Python readme](python/README.md) for more details on installing
+from source, development, and testing.
 
 ### Benchmarks
 * [Squash Compression Benchmark](https://quixdb.github.io/squash-benchmark/) / [Unstable Squash Compression Benchmark](https://quixdb.github.io/squash-benchmark/unstable/)
 * [Large Text Compression Benchmark](http://mattmahoney.net/dc/text.html)
 * [Lzturbo Benchmark](https://sites.google.com/site/powturbo/home/benchmark)
 
 ### Related projects
+> **Disclaimer:** Brotli authors take no responsibility for the third party projects mentioned in this section.
+
 Independent [decoder](https://github.com/madler/brotli) implementation by Mark Adler, based entirely on format specification.
 
 JavaScript port of brotli [decoder](https://github.com/devongovett/brotli.js). Could be used directly via `npm install brotli`
+
+Hand ported [decoder / encoder](https://github.com/dominikhlbg/BrotliHaxe) in haxe by Dominik Homberger. Output source code: JavaScript, PHP, Python, Java and C#
+
+7Zip [plugin](https://github.com/mcmilk/7-Zip-Zstd)
diff --git a/third_party/brotli/common/constants.h b/third_party/brotli/common/constants.h
@@ -38,6 +38,8 @@
                                      BROTLI_MAX_NDIRECT +              \
                                      (BROTLI_MAX_DISTANCE_BITS <<      \
                                       (BROTLI_MAX_NPOSTFIX + 1)))
+/* Distance that is guaranteed to be representable in any stream. */
+#define BROTLI_MAX_DISTANCE 0x3FFFFFC
 
 /* 7.1. Context modes and context ID lookup for literals */
 /* "context IDs for literals are in the range of 0..63" */

diff --git a/third_party/brotli/dec/decode.c b/third_party/brotli/dec/decode.c
@@ -39,6 +39,11 @@ extern "C" {
 #define HUFFMAN_TABLE_BITS 8U
 #define HUFFMAN_TABLE_MASK 0xff
 
+/* We need the slack region for the following reasons:
+    - doing up to two 16-byte copies for fast backward copying
+    - inserting transformed dictionary word (5 prefix + 24 base + 8 suffix) */
+static const uint32_t kRingBufferWriteAheadSlack = 42;
+
 static const uint8_t kCodeLengthCodeOrder[BROTLI_CODE_LENGTH_CODES] = {
   1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 };
@@ -52,6 +57,17 @@ static const uint8_t kCodeLengthPrefixValue[16] = {
   0, 4, 3, 2, 0, 4, 3, 1, 0, 4, 3, 2, 0, 4, 3, 5,
 };
 
+BROTLI_BOOL BrotliDecoderSetParameter(
+    BrotliDecoderState* state, BrotliDecoderParameter p, uint32_t value) {
+  switch (p) {
+    case BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION:
+      state->canny_ringbuffer_allocation = !!value ? 0 : 1;
+      return BROTLI_TRUE;
+
+    default: return BROTLI_FALSE;
+  }
+}
+
 BrotliDecoderState* BrotliDecoderCreateInstance(
     brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
   BrotliDecoderState* state = 0;
@@ -1210,7 +1226,9 @@ static BrotliDecoderErrorCode BROTLI_NOINLINE WriteRingBuffer(
   BROTLI_LOG_UINT(to_write);
   BROTLI_LOG_UINT(num_written);
   s->partial_pos_out += num_written;
-  if (total_out) *total_out = s->partial_pos_out - (size_t)s->custom_dict_size;
+  if (total_out) {
+    *total_out = s->partial_pos_out;
+  }
   if (num_written < to_write) {
     if (s->ringbuffer_size == (1 << s->window_bits) || force) {
       return BROTLI_DECODER_NEEDS_MORE_OUTPUT;
@@ -1242,22 +1260,16 @@ static void BROTLI_NOINLINE WrapRingBuffer(BrotliDecoderState* s) {
 
    Last two bytes of ring-buffer are initialized to 0, so context calculation
    could be done uniformly for the first two and all other positions.
-
-   Custom dictionary, if any, is copied to the end of ring-buffer.
 */
 static BROTLI_BOOL BROTLI_NOINLINE BrotliEnsureRingBuffer(
     BrotliDecoderState* s) {
-  /* We need the slack region for the following reasons:
-      - doing up to two 16-byte copies for fast backward copying
-      - inserting transformed dictionary word (5 prefix + 24 base + 8 suffix) */
-  static const int kRingBufferWriteAheadSlack = 42;
   uint8_t* old_ringbuffer = s->ringbuffer;
   if (s->ringbuffer_size == s->new_ringbuffer_size) {
     return BROTLI_TRUE;
   }
 
-  s->ringbuffer = (uint8_t*)BROTLI_ALLOC(s, (size_t)(s->new_ringbuffer_size +
-      kRingBufferWriteAheadSlack));
+  s->ringbuffer = (uint8_t*)BROTLI_ALLOC(s, (size_t)(s->new_ringbuffer_size) +
+      kRingBufferWriteAheadSlack);
   if (s->ringbuffer == 0) {
     /* Restore previous value. */
     s->ringbuffer = old_ringbuffer;
@@ -1266,13 +1278,7 @@ static BROTLI_BOOL BROTLI_NOINLINE BrotliEnsureRingBuffer(
   s->ringbuffer[s->new_ringbuffer_size - 2] = 0;
   s->ringbuffer[s->new_ringbuffer_size - 1] = 0;
 
-  if (!old_ringbuffer) {
-    if (s->custom_dict) {
-      memcpy(s->ringbuffer, s->custom_dict, (size_t)s->custom_dict_size);
-      s->partial_pos_out = (size_t)s->custom_dict_size;
-      s->pos = s->custom_dict_size;
-    }
-  } else {
+  if (!!old_ringbuffer) {
     memcpy(s->ringbuffer, old_ringbuffer, (size_t)s->pos);
     BROTLI_FREE(s, old_ringbuffer);
   }
@@ -1361,16 +1367,20 @@ static void BROTLI_NOINLINE BrotliCalculateRingBufferSize(
   }
 
   if (!s->ringbuffer) {
-    /* Custom dictionary counts as a "virtual" output. */
-    output_size = s->custom_dict_size;
+    output_size = 0;
   } else {
     output_size = s->pos;
   }
   output_size += s->meta_block_remaining_len;
   min_size = min_size < output_size ? output_size : min_size;
 
-  while ((new_ringbuffer_size >> 1) >= min_size) {
-    new_ringbuffer_size >>= 1;
+  if (!!s->canny_ringbuffer_allocation) {
+    /* Reduce ring buffer size to save memory when server is unscrupulous.
+       In worst case memory usage might be 1.5x bigger for a short period of
+       ring buffer reallocation.*/
+    while ((new_ringbuffer_size >> 1) >= min_size) {
+      new_ringbuffer_size >>= 1;
+    }
   }
 
   s->new_ringbuffer_size = new_ringbuffer_size;
@@ -1735,14 +1745,14 @@ static BROTLI_INLINE BrotliDecoderErrorCode ProcessCommandsInternal(
   /* Apply copy of LZ77 back-reference, or static dictionary reference if
   the distance is larger than the max LZ77 distance */
   if (s->distance_code > s->max_distance) {
+    int address = s->distance_code - s->max_distance - 1;
     if (i >= BROTLI_MIN_DICTIONARY_WORD_LENGTH &&
         i <= BROTLI_MAX_DICTIONARY_WORD_LENGTH) {
       int offset = (int)s->dictionary->offsets_by_length[i];
-      int word_id = s->distance_code - s->max_distance - 1;
       uint32_t shift = s->dictionary->size_bits_by_length[i];
       int mask = (int)BitMask(shift);
-      int word_idx = word_id & mask;
-      int transform_idx = word_id >> shift;
+      int word_idx = address & mask;
+      int transform_idx = address >> shift;
       /* Compensate double distance-ring-buffer roll. */
       s->dist_rb_idx += s->distance_context;
       offset += word_idx * i;
@@ -1925,7 +1935,13 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
     if (result != BROTLI_DECODER_SUCCESS) { /* Error, needs more input/output */
       if (result == BROTLI_DECODER_NEEDS_MORE_INPUT) {
         if (s->ringbuffer != 0) { /* Pro-actively push output. */
-          WriteRingBuffer(s, available_out, next_out, total_out, BROTLI_TRUE);
+          BrotliDecoderErrorCode intermediate_result = WriteRingBuffer(s,
+              available_out, next_out, total_out, BROTLI_TRUE);
+          /* WriteRingBuffer checks s->meta_block_remaining_len validity. */
+          if ((int)intermediate_result < 0) {
+            result = intermediate_result;
+            break;
+          }
         }
         if (s->buffer_length != 0) { /* Used with internal buffer. */
           if (br->avail_in == 0) { /* Successfully finished read transaction. */
@@ -1999,11 +2015,6 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
         }
         /* Maximum distance, see section 9.1. of the spec. */
         s->max_backward_distance = (1 << s->window_bits) - BROTLI_WINDOW_GAP;
-        /* Limit custom dictionary size. */
-        if (s->custom_dict_size >= s->max_backward_distance) {
-          s->custom_dict += s->custom_dict_size - s->max_backward_distance;
-          s->custom_dict_size = s->max_backward_distance;
-        }
 
         /* Allocate memory for both block_type_trees and block_len_trees. */
         s->block_type_trees = (HuffmanCode*)BROTLI_ALLOC(s,
@@ -2300,16 +2311,11 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
   return SaveErrorCode(s, result);
 }
 
-void BrotliDecoderSetCustomDictionary(
-    BrotliDecoderState* s, size_t size, const uint8_t* dict) {
-  if (size > (1u << 24)) {
-    return;
-  }
-  s->custom_dict = dict;
-  s->custom_dict_size = (int)size;
-}
-
 BROTLI_BOOL BrotliDecoderHasMoreOutput(const BrotliDecoderState* s) {
+  /* After unrecoverable error remaining output is considered nonsensical. */
+  if ((int)s->error_code < 0) {
+    return BROTLI_FALSE;
+  }
   return TO_BROTLI_BOOL(
       s->ringbuffer != 0 && UnwrittenBytes(s, BROTLI_FALSE) != 0);
 }
@@ -2319,17 +2325,20 @@ const uint8_t* BrotliDecoderTakeOutput(BrotliDecoderState* s, size_t* size) {
   size_t available_out = *size ? *size : 1u << 24;
   size_t requested_out = available_out;
   BrotliDecoderErrorCode status;
-  if (s->ringbuffer == 0) {
+  if ((s->ringbuffer == 0) || ((int)s->error_code < 0)) {
     *size = 0;
     return 0;
   }
   WrapRingBuffer(s);
   status = WriteRingBuffer(s, &available_out, &result, 0, BROTLI_TRUE);
+  /* Either WriteRingBuffer returns those "success" codes... */
   if (status == BROTLI_DECODER_SUCCESS ||
       status == BROTLI_DECODER_NEEDS_MORE_OUTPUT) {
     *size = requested_out - available_out;
   } else {
-    /* This might happen if previous decoder error code was ignored. */
+    /* ... or stream is broken. Normally this should be caught by
+       BrotliDecoderDecompressStream, this is just a safeguard. */
+    if ((int)status < 0) SaveErrorCode(s, status);
     *size = 0;
     result = 0;
   }

diff --git a/third_party/brotli/dec/state.c b/third_party/brotli/dec/state.c
@@ -83,11 +83,12 @@ void BrotliDecoderStateInitWithCustomAllocators(BrotliDecoderState* s,
   s->distance_hgroup.codes = NULL;
   s->distance_hgroup.htrees = NULL;
 
-  s->custom_dict = NULL;
-  s->custom_dict_size = 0;
-
   s->is_last_metablock = 0;
+  s->is_uncompressed = 0;
+  s->is_metadata = 0;
   s->should_wrap_ringbuffer = 0;
+  s->canny_ringbuffer_allocation = 1;
+
   s->window_bits = 0;
   s->max_distance = 0;
   s->dist_rb[0] = 16;

diff --git a/third_party/brotli/dec/state.h b/third_party/brotli/dec/state.h
@@ -172,7 +172,7 @@ struct BrotliDecoderStateStruct {
   uint32_t space;
 
   HuffmanCode table[32];
-  /* List of of symbol chains. */
+  /* List of heads of symbol chains. */
   uint16_t* symbol_lists;
   /* Storage from symbol_lists. */
   uint16_t symbols_lists_array[BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1 +
@@ -197,10 +197,6 @@ struct BrotliDecoderStateStruct {
   uint32_t mtf_upper_bound;
   uint32_t mtf[64 + 1];
 
-  /* For custom dictionaries */
-  const uint8_t* custom_dict;
-  int custom_dict_size;
-
   /* less used attributes are in the end of this struct */
   /* States inside function calls */
   BrotliRunningMetablockHeaderState substate_metablock_header;
@@ -215,6 +211,7 @@ struct BrotliDecoderStateStruct {
   unsigned int is_uncompressed : 1;
   unsigned int is_metadata : 1;
   unsigned int should_wrap_ringbuffer : 1;
+  unsigned int canny_ringbuffer_allocation : 1;
   unsigned int size_nibbles : 8;
   uint32_t window_bits;
 

diff --git a/third_party/brotli/enc/backward_references.c b/third_party/brotli/enc/backward_references.c
@@ -48,6 +48,8 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
 #define EXPAND_CAT(a, b) CAT(a, b)
 #define CAT(a, b) a ## b
 #define FN(X) EXPAND_CAT(X, HASHER())
+#define EXPORT_FN(X) EXPAND_CAT(X, EXPAND_CAT(PREFIX(), HASHER()))
+#define PREFIX() N
 
 #define HASHER() H2
 /* NOLINTNEXTLINE(build/include) */
@@ -94,6 +96,8 @@ static BROTLI_INLINE size_t ComputeDistanceCode(size_t distance,
 #include "./backward_references_inc.h"
 #undef HASHER
 
+#undef PREFIX
+#undef EXPORT_FN
 #undef FN
 #undef CAT
 #undef EXPAND_CAT
@@ -113,11 +117,11 @@ void BrotliCreateBackwardReferences(const BrotliDictionary* dictionary,
   switch (params->hasher.type) {
 #define CASE_(N)                                                  \
     case N:                                                       \
-      CreateBackwardReferencesH ## N(dictionary,                  \
+      CreateBackwardReferencesNH ## N(dictionary,                 \
           kStaticDictionaryHash, num_bytes, position, ringbuffer, \
           ringbuffer_mask, params, hasher, dist_cache,            \
           last_insert_len, commands, num_commands, num_literals); \
-      break;
+      return;
     FOR_GENERIC_HASHERS(CASE_)
 #undef CASE_
     default: