Certificate Transparency: Code for unpacking EV cert hashes whitelist

Re-submission of https://codereview.chromium.org/462543002/ (note that the Fingerprint256 changes have been broken off to a separate change).

Note for the build cop: Please attempt to contact me if there's a need to roll back.

BUG=339128

Review URL: https://codereview.chromium.org/547603002

Cr-Commit-Position: refs/heads/master@{#301642}

chrome/browser/component_updater/ev_whitelist_component_installer.cc

@@ -13,6 +13,7 @@
 #include "base/lazy_instance.h"
 #include "base/logging.h"
 #include "base/path_service.h"
+#include "chrome/browser/net/packed_ct_ev_whitelist.h"
 #include "components/component_updater/component_updater_paths.h"
 #include "content/public/browser/browser_thread.h"
 #include "net/ssl/ssl_config_service.h"
@@ -65,16 +66,9 @@ void EVWhitelistComponentInstallerTraits::ComponentReady(
   VLOG(1) << "Component ready, version " << version.GetString() << " in "
           << path.value();
 
-  // TODO(eranm): Uncomment once https://codereview.chromium.org/462543002/
-  // is in.
-  /*
   const base::FilePath whitelist_file = GetInstalledPath(path);
-  base::Callback<void(void)> set_cb =
-      base::Bind(&net::ct::SetEVWhitelistFromFile, whitelist_file);
   content::BrowserThread::PostBlockingPoolTask(
-      FROM_HERE,
-      set_cb);
-      */
+      FROM_HERE, base::Bind(&SetEVWhitelistFromFile, whitelist_file));
 }
 
 bool EVWhitelistComponentInstallerTraits::VerifyInstallation(

chrome/browser/net/bit_stream_reader.cc

@@ -0,0 +1,63 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/net/bit_stream_reader.h"
+
+#include "base/big_endian.h"
+#include "base/logging.h"
+#include "base/numerics/safe_conversions.h"
+
+namespace internal {
+
+BitStreamReader::BitStreamReader(const base::StringPiece& source)
+    : source_(source), current_byte_(0), current_bit_(7) {
+  DCHECK_LT(source_.length(), UINT32_MAX);
+}
+
+bool BitStreamReader::ReadUnaryEncoding(uint64_t* out) {
+  if (BitsLeft() == 0)
+    return false;
+
+  *out = 0;
+  while ((BitsLeft() > 0) && ReadBit())
+    ++(*out);
+
+  return true;
+}
+
+bool BitStreamReader::ReadBits(uint8_t num_bits, uint64_t* out) {
+  DCHECK_LE(num_bits, 64);
+
+  if (BitsLeft() < num_bits)
+    return false;
+
+  *out = 0;
+  for (uint8_t i = 0; i < num_bits; ++i)
+    (*out) |= (static_cast<uint64_t>(ReadBit()) << (num_bits - (i + 1)));
+
+  return true;
+}
+
+uint64_t BitStreamReader::BitsLeft() const {
+  if (current_byte_ == source_.length())
+    return 0;
+  DCHECK_GT(source_.length(), current_byte_);
+  return (source_.length() - (current_byte_ + 1)) * 8 + current_bit_ + 1;
+}
+
+uint8_t BitStreamReader::ReadBit() {
+  DCHECK_GT(BitsLeft(), 0u);
+  DCHECK(current_bit_ < 8 && current_bit_ >= 0);
+  uint8_t res =
+      (source_.data()[current_byte_] & (1 << current_bit_)) >> current_bit_;
+  current_bit_--;
+  if (current_bit_ < 0) {
+    current_byte_++;
+    current_bit_ = 7;
+  }
+
+  return res;
+}
+
+}  // namespace internal

chrome/browser/net/bit_stream_reader.h

@@ -0,0 +1,58 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_NET_BIT_STREAM_READER_H_
+#define CHROME_BROWSER_NET_BIT_STREAM_READER_H_
+
+#include <stdint.h>
+
+#include "base/strings/string_piece.h"
+
+namespace internal {
+
+// A class for reading individual bits from a packed buffer. Bits are read
+// MSB-first from the stream.
+// It is limited to 64-bit reads, 4GB streams and is inefficient as a design
+// choice. This class should not be used frequently.
+//
+// It is meant for data that is is packed across bytes, necessitating the need
+// to read a variable number of bits across a byte boundary.
+class BitStreamReader {
+ public:
+  explicit BitStreamReader(const base::StringPiece& source);
+
+  // Reads unary-encoded number into |out|. Returns true if
+  // there was at least one bit to read, false otherwise.
+  bool ReadUnaryEncoding(uint64_t* out);
+
+  // Reads |num_bits| (up to 64) into |out|. |out| is filled from the MSB to the
+  // LSB. If |num_bits| is less than 64, the most significant |64 - num_bits|
+  // bits are unused and left as zeros. Returns true if the stream had the
+  // requested |num_bits|, false otherwise.
+  bool ReadBits(uint8_t num_bits, uint64_t* out);
+
+  // Returns the number of bits left in the stream.
+  uint64_t BitsLeft() const;
+
+ private:
+  // Reads a single bit. Within a byte, the bits are read from the MSB to the
+  // LSB.
+  uint8_t ReadBit();
+
+  const base::StringPiece source_;
+
+  // Index of the byte currently being read from.
+  size_t current_byte_;
+
+  // Index of the last bit read within |current_byte_|. Since bits are read
+  // from the MSB to the LSB, this value is initialized to 7 and decremented
+  // after each read.
+  int8 current_bit_;
+
+  DISALLOW_COPY_AND_ASSIGN(BitStreamReader);
+};
+
+}  // namespace internal
+
+#endif  // CHROME_BROWSER_NET_BIT_STREAM_READER_H_

chrome/browser/net/bit_stream_reader_unittest.cc

@@ -0,0 +1,95 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/net/bit_stream_reader.h"
+
+#include <algorithm>
+#include <string>
+
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace internal {
+
+const uint8_t kSomeData[] = {0xd5, 0xe2, 0xaf, 0xe5, 0xbb, 0x10, 0x7c, 0xd1};
+
+TEST(BitStreamReaderTest, CanReadSingleByte) {
+  BitStreamReader reader(
+      base::StringPiece(reinterpret_cast<const char*>(kSomeData), 1));
+  uint64_t v(0);
+
+  EXPECT_EQ(8u, reader.BitsLeft());
+  EXPECT_TRUE(reader.ReadBits(8, &v));
+  EXPECT_EQ(UINT64_C(0xd5), v);
+
+  EXPECT_FALSE(reader.ReadBits(1, &v));
+  EXPECT_EQ(0u, reader.BitsLeft());
+}
+
+TEST(BitStreamReaderTest, CanReadSingleBits) {
+  const uint64_t expected_bits[] = {
+      1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0};
+  BitStreamReader reader(
+      base::StringPiece(reinterpret_cast<const char*>(kSomeData), 2));
+  EXPECT_EQ(16u, reader.BitsLeft());
+  uint64_t v(0);
+
+  for (int i = 0; i < 16; ++i) {
+    EXPECT_TRUE(reader.ReadBits(1, &v));
+    EXPECT_EQ(expected_bits[i], v);
+  }
+  EXPECT_EQ(0u, reader.BitsLeft());
+}
+
+TEST(BitStreamReaderTest, CanReadBitGroups) {
+  BitStreamReader reader(
+      base::StringPiece(reinterpret_cast<const char*>(kSomeData), 3));
+  EXPECT_EQ(24u, reader.BitsLeft());
+  uint64_t v(0);
+  uint64_t res(0);
+
+  EXPECT_TRUE(reader.ReadBits(5, &v));
+  res |= v << 19;
+  EXPECT_EQ(19u, reader.BitsLeft());
+  EXPECT_TRUE(reader.ReadBits(13, &v));
+  res |= v << 6;
+  EXPECT_EQ(6u, reader.BitsLeft());
+  EXPECT_TRUE(reader.ReadBits(6, &v));
+  res |= v;
+  EXPECT_EQ(UINT64_C(0xd5e2af), res);
+
+  EXPECT_FALSE(reader.ReadBits(1, &v));
+}
+
+TEST(BitStreamReaderTest, CanRead64Bit) {
+  BitStreamReader reader(
+      base::StringPiece(reinterpret_cast<const char*>(kSomeData), 8));
+  EXPECT_EQ(64u, reader.BitsLeft());
+  uint64_t v(0);
+
+  EXPECT_TRUE(reader.ReadBits(64, &v));
+  EXPECT_EQ(UINT64_C(0xd5e2afe5bb107cd1), v);
+}
+
+TEST(BitStreamReaderTest, CanReadUnaryEncodedNumbers) {
+  internal::BitStreamReader reader(
+      base::StringPiece(reinterpret_cast<const char*>(kSomeData), 3));
+  const uint64_t expected_values[] = {2, 1, 1, 4, 0, 0, 1, 1, 1, 4};
+  uint64_t v(0);
+  for (int i = 0; i < 10; ++i) {
+    EXPECT_TRUE(reader.ReadUnaryEncoding(&v));
+    EXPECT_EQ(expected_values[i], v) << "Values differ at position " << i;
+  }
+}
+
+TEST(BitStreamReaderTest, CannotReadFromEmptyStream) {
+  BitStreamReader reader(base::StringPiece(
+      reinterpret_cast<const char*>(kSomeData), static_cast<size_t>(0u)));
+  uint64_t v(0);
+
+  EXPECT_EQ(0u, reader.BitsLeft());
+  EXPECT_FALSE(reader.ReadBits(1, &v));
+  EXPECT_FALSE(reader.ReadUnaryEncoding(&v));
+}
+
+}  // namespace internal

chrome/browser/net/packed_ct_ev_whitelist.cc

@@ -0,0 +1,138 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/net/packed_ct_ev_whitelist.h"
+
+#include <string.h>
+
+#include <algorithm>
+
+#include "base/big_endian.h"
+#include "base/files/file_util.h"
+#include "base/lazy_instance.h"
+#include "base/logging.h"
+#include "chrome/browser/net/bit_stream_reader.h"
+#include "content/public/browser/browser_thread.h"
+#include "net/ssl/ssl_config_service.h"
+
+namespace {
+const uint8_t kCertHashLengthBits = 64;  // 8 bytes
+const uint8_t kCertHashLength = kCertHashLengthBits / 8;
+const uint64_t kGolombMParameterBits = 47;  // 2^47
+
+void SetNewEVWhitelistInSSLConfigService(
+    const scoped_refptr<net::ct::EVCertsWhitelist>& new_whitelist) {
+  net::SSLConfigService::SetEVCertsWhitelist(new_whitelist);
+}
+
+int TruncatedHashesComparator(const void* v1, const void* v2) {
+  const uint64_t& h1(*(static_cast<const uint64_t*>(v1)));
+  const uint64_t& h2(*(static_cast<const uint64_t*>(v2)));
+  if (h1 < h2)
+    return -1;
+  else if (h1 > h2)
+    return 1;
+  return 0;
+}
+}  // namespace
+
+void SetEVWhitelistFromFile(const base::FilePath& compressed_whitelist_file) {
+  VLOG(1) << "Setting EV whitelist from file: "
+          << compressed_whitelist_file.value();
+  std::string compressed_list;
+  if (!base::ReadFileToString(compressed_whitelist_file, &compressed_list)) {
+    VLOG(1) << "Failed reading from " << compressed_whitelist_file.value();
+    return;
+  }
+
+  scoped_refptr<net::ct::EVCertsWhitelist> new_whitelist(
+      new PackedEVCertsWhitelist(compressed_list));
+  if (!new_whitelist->IsValid()) {
+    VLOG(1) << "Failed uncompressing EV certs whitelist.";
+    return;
+  }
+
+  base::Closure assign_cb =
+      base::Bind(SetNewEVWhitelistInSSLConfigService, new_whitelist);
+  content::BrowserThread::PostTask(
+      content::BrowserThread::IO, FROM_HERE, assign_cb);
+}
+
+bool PackedEVCertsWhitelist::UncompressEVWhitelist(
+    const std::string& compressed_whitelist,
+    std::vector<uint64_t>* uncompressed_list) {
+  internal::BitStreamReader reader(base::StringPiece(
+      compressed_whitelist.data(), compressed_whitelist.size()));
+  std::vector<uint64_t> result;
+
+  VLOG(1) << "Uncompressing EV whitelist of size "
+          << compressed_whitelist.size();
+  uint64_t curr_hash(0);
+  if (!reader.ReadBits(kCertHashLengthBits, &curr_hash)) {
+    VLOG(1) << "Failed reading first hash.";
+    return false;
+  }
+  result.push_back(curr_hash);
+  // M is the tunable parameter used by the Golomb coding.
+  static const uint64_t kGolombParameterM = static_cast<uint64_t>(1)
+                                            << kGolombMParameterBits;
+
+  while (reader.BitsLeft() > kGolombMParameterBits) {
+    uint64_t read_prefix = 0;
+    if (!reader.ReadUnaryEncoding(&read_prefix)) {
+      VLOG(1) << "Failed reading unary-encoded prefix.";
+      return false;
+    }
+    if (read_prefix > (UINT64_MAX / kGolombParameterM)) {
+      VLOG(1) << "Received value that would cause overflow: " << read_prefix;
+      return false;
+    }
+
+    uint64_t r = 0;
+    if (!reader.ReadBits(kGolombMParameterBits, &r)) {
+      VLOG(1) << "Failed reading " << kGolombMParameterBits << " bits.";
+      return false;
+    }
+    DCHECK_LT(r, kGolombParameterM);
+
+    uint64_t curr_diff = read_prefix * kGolombParameterM + r;
+    curr_hash += curr_diff;
+
+    result.push_back(curr_hash);
+  }
+
+  uncompressed_list->swap(result);
+  return true;
+}
+
+PackedEVCertsWhitelist::PackedEVCertsWhitelist(
+    const std::string& compressed_whitelist)
+    : is_whitelist_valid_(false) {
+  if (!UncompressEVWhitelist(compressed_whitelist, &whitelist_)) {
+    whitelist_.clear();
+    return;
+  }
+
+  is_whitelist_valid_ = true;
+}
+
+PackedEVCertsWhitelist::~PackedEVCertsWhitelist() {
+}
+
+bool PackedEVCertsWhitelist::ContainsCertificateHash(
+    const std::string& certificate_hash) const {
+  DCHECK(!whitelist_.empty());
+  uint64_t hash_to_lookup;
+
+  base::ReadBigEndian(certificate_hash.data(), &hash_to_lookup);
+  return bsearch(&hash_to_lookup,
+                 &whitelist_[0],
+                 whitelist_.size(),
+                 kCertHashLength,
+                 TruncatedHashesComparator) != NULL;
+}
+
+bool PackedEVCertsWhitelist::IsValid() const {
+  return is_whitelist_valid_;
+}