Adding python code to generate resource name hash to index mapping.

This adds a build step after generated_resources.h is generated. This step
parses generated_resources.h, extracting all the IDS_* names, index pairs and
produces a mapping from a hash of each name to its index.

This is the first step to implement a mechanism to allow us to override UI
strings with server provided strings from the variations_service.

Design doc for this feature: https://docs.google.com/a/google.com/document/d/1UCQCZzF0Ox9dyD3R66_Fn1RYfckF9BTVjM0o5u4YaWg/edit?usp=sharing

BUG=370033

Review URL: https://codereview.chromium.org/246123007

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@269880 0039d316-1c4b-4281-b951-d872f2087c98

chrome/browser/metrics/variations/generate_resources_map.py

@@ -0,0 +1,181 @@
+#!/usr/bin/python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import collections
+import hashlib
+import operator
+import os
+import re
+import sys
+
+
+RESOURCE_EXTRACT_REGEX = re.compile('^#define (\S*) (\d*)$', re.MULTILINE)
+
+class Error(Exception):
+  """Base error class for all exceptions in generated_resources_map."""
+
+
+class HashCollisionError(Error):
+  """Multiple resource names hash to the same value."""
+
+
+Resource = collections.namedtuple("Resource", ['hash', 'name', 'index'])
+
+
+def _HashName(name):
+  """Returns the hash id for a name.
+
+  Args:
+    name: The name to hash.
+
+  Returns:
+    An int that is at most 32 bits.
+  """
+  md5hash = hashlib.md5()
+  md5hash.update(name)
+  return int(md5hash.hexdigest()[:8], 16)
+
+
+def _GetNameIndexPairsIter(string_to_scan):
+  """Gets an iterator of the resource name and index pairs of the given string.
+
+  Scans the input string for lines of the form "#define NAME INDEX" and returns
+  an iterator over all matching (NAME, INDEX) pairs.
+
+  Args:
+    string_to_scan: The input string to scan.
+
+  Yields:
+    A tuple of name and index.
+  """
+  for match in RESOURCE_EXTRACT_REGEX.finditer(string_to_scan):
+    yield match.group(1, 2)
+
+
+def _GetResourceListFromString(resources_content):
+  """Produces a list of |Resource| objects from a string.
+
+  The input string conaints lines of the form "#define NAME INDEX". The returned
+  list is sorted primarily by hash, then name, and then index.
+
+  Args:
+    resources_content: The input string to process, contains lines of the form
+        "#define NAME INDEX".
+
+  Returns:
+    A sorted list of |Resource| objects.
+  """
+  resources = [Resource(_HashName(name), name, index) for name, index in
+               _GetNameIndexPairsIter(resources_content)]
+
+  # The default |Resource| order makes |resources| sorted by the hash, then
+  # name, then index.
+  resources.sort()
+
+  return resources
+
+
+def _CheckForHashCollisions(sorted_resource_list):
+  """Checks a sorted list of |Resource| objects for hash collisions.
+
+  Args:
+    sorted_resource_list: A sorted list of |Resource| objects.
+
+  Returns:
+    A set of all |Resource| objects with collisions.
+  """
+  collisions = set()
+  for i in xrange(len(sorted_resource_list) - 1):
+    resource = sorted_resource_list[i]
+    next_resource = sorted_resource_list[i+1]
+    if resource.hash == next_resource.hash:
+      collisions.add(resource)
+      collisions.add(next_resource)
+
+  return collisions
+
+
+def _GenDataArray(
+    resources, entry_pattern, array_name, array_type, data_getter):
+  """Generates a C++ statement defining a literal array containing the hashes.
+
+  Args:
+    resources: A sorted list of |Resource| objects.
+    entry_pattern: A pattern to be used to generate each entry in the array. The
+        pattern is expected to have a place for data and one for a comment, in
+        that order.
+    array_name: The name of the array being generated.
+    array_type: The type of the array being generated.
+    data_getter: A function that gets the array data from a |Resource| object.
+
+  Returns:
+    A string containing a C++ statement defining the an array.
+  """
+  lines = [entry_pattern % (data_getter(r), r.name) for r in resources]
+  pattern = """const %(type)s %(name)s[] = {
+%(content)s
+};
+"""
+  return pattern % {'type': array_type,
+                    'name': array_name,
+                    'content': '\n'.join(lines)}
+
+
+def _GenerateFileContent(resources_content):
+  """Generates the .cc content from the given generated_resources.h content.
+
+  Args:
+    resources_content: The input string to process, contains lines of the form
+        "#define NAME INDEX".
+
+  Returns:
+    .cc file content defining the kResourceHashes and kResourceIndices arrays.
+  """
+  hashed_tuples = _GetResourceListFromString(resources_content)
+
+  collisions = _CheckForHashCollisions(hashed_tuples)
+  if collisions:
+    error_message = "\n".join(
+        ["hash: %i, name: %s" % (i[0], i[1]) for i in sorted(collisions)])
+    error_message = ("\nThe following names had hash collisions "
+                     "(sorted by the hash value):\n%s\n" %(error_message))
+    raise HashCollisionError(error_message)
+
+  hashes_array = _GenDataArray(
+      hashed_tuples, "    %iU,  // %s", 'kResourceHashes', 'uint32_t',
+      operator.attrgetter('hash'))
+  indices_array = _GenDataArray(
+      hashed_tuples, "    %s,  // %s", 'kResourceIndices', 'int',
+      operator.attrgetter('index'))
+
+  return (
+      "// This file was generated by generate_resources_map.py. Do not edit.\n"
+      "\n\n"
+      "#include "
+      "\"chrome/browser/metrics/variations/generated_resources_map.h\"\n\n"
+      "namespace chrome_variations {\n\n"
+      "%s"
+      "\n"
+      "%s"
+      "\n"
+      "}  // namespace chrome_variations\n") % (hashes_array, indices_array)
+
+
+def main(resources_file, map_file):
+  generated_resources_h = ""
+  with open(resources_file, "r") as resources:
+    generated_resources_h = resources.read()
+
+  if len(generated_resources_h) == 0:
+    raise Error("No content loaded for %s." % (resources_file))
+
+  file_content = _GenerateFileContent(generated_resources_h)
+
+  with open(map_file, "w") as generated_file:
+    generated_file.write(file_content)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1], sys.argv[2]))

chrome/browser/metrics/variations/generate_resources_map_unittest.py

@@ -0,0 +1,91 @@
+#!/usr/bin/python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unittests for generate_resources_map.py"""
+
+import unittest
+
+import generate_resources_map
+
+
+class GenerateResourcesMapUnittest(unittest.TestCase):
+  TEST_INPUT = """
+// This file is automatically generated by GRIT. Do not edit.
+
+#pragma once
+
+#define IDS_BOOKMARKS_NO_ITEMS 12500
+#define IDS_BOOKMARK_BAR_IMPORT_LINK 12501
+#define IDS_BOOKMARK_GROUP_FROM_IE 12502
+#define IDS_BOOKMARK_GROUP_FROM_FIREFOX 12503
+"""
+
+  def testGetResourceListFromString(self):
+    expected_tuples = [(301430091, "IDS_BOOKMARKS_NO_ITEMS", "12500"),
+                       (2654138887, "IDS_BOOKMARK_BAR_IMPORT_LINK", "12501"),
+                       (2894469061, "IDS_BOOKMARK_GROUP_FROM_IE", "12502"),
+                       (3847176170, "IDS_BOOKMARK_GROUP_FROM_FIREFOX", "12503")]
+    expected = [generate_resources_map.Resource(*t) for t in expected_tuples]
+
+    actual_tuples = generate_resources_map._GetResourceListFromString(
+        self.TEST_INPUT)
+
+    self.assertEqual(expected_tuples, actual_tuples)
+
+
+  def testCheckForHashCollisions(self):
+    collisions_tuples = [(123, "IDS_FOO", "12500"),
+                         (456, "IDS_BAR", "12501"),
+                         (456, "IDS_BAZ", "12502"),
+                         (890, "IDS_QUX", "12503"),
+                         (899, "IDS_NO", "12504"),
+                         (899, "IDS_YES", "12505")]
+    list_with_collisions = [generate_resources_map.Resource(*t)
+                            for t in collisions_tuples]
+
+    expected_collision_tuples = [(456, "IDS_BAR", "12501"),
+                                 (456, "IDS_BAZ", "12502"),
+                                 (899, "IDS_NO", "12504"),
+                                 (899, "IDS_YES", "12505")]
+    expected_collisions = [generate_resources_map.Resource(*t)
+                          for t in expected_collision_tuples]
+
+    actual_collisions = sorted(
+        generate_resources_map._CheckForHashCollisions(list_with_collisions))
+    actual_collisions
+
+    self.assertEqual(expected_collisions, actual_collisions)
+
+  def testGenerateFileContent(self):
+    expected = (
+        """// This file was generated by generate_resources_map.py. Do not edit.
+
+
+#include "chrome/browser/metrics/variations/generated_resources_map.h"
+
+namespace chrome_variations {
+
+const uint32_t kResourceHashes[] = {
+    301430091U,  // IDS_BOOKMARKS_NO_ITEMS
+    2654138887U,  // IDS_BOOKMARK_BAR_IMPORT_LINK
+    2894469061U,  // IDS_BOOKMARK_GROUP_FROM_IE
+    3847176170U,  // IDS_BOOKMARK_GROUP_FROM_FIREFOX
+};
+
+const int kResourceIndices[] = {
+    12500,  // IDS_BOOKMARKS_NO_ITEMS
+    12501,  // IDS_BOOKMARK_BAR_IMPORT_LINK
+    12502,  // IDS_BOOKMARK_GROUP_FROM_IE
+    12503,  // IDS_BOOKMARK_GROUP_FROM_FIREFOX
+};
+
+}  // namespace chrome_variations
+""")
+    actual = generate_resources_map._GenerateFileContent(self.TEST_INPUT)
+
+    self.assertEqual(expected, actual)
+
+if __name__ == '__main__':
+  unittest.main()

chrome/browser/metrics/variations/generated_resources_map.h

@@ -0,0 +1,31 @@
+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_METRICS_VARIATIONS_GENERATED_RESOURCES_MAP_H_
+#define CHROME_BROWSER_METRICS_VARIATIONS_GENERATED_RESOURCES_MAP_H_
+
+#include "base/basictypes.h"
+
+namespace chrome_variations {
+
+// This file provides a mapping from hashes of generated resource names to their
+// IDs. This mapping is achieved by having two arrays: |kResourceHashes|, a
+// sorted array of resource name hashes; and |kResourceIndices|, an array of
+// resource indices in the same order as |kResourceHashes|. So, if
+// generated_resources.h contains |#define IDS_FOO 12345|, then for some index i
+// kResourceHashes[i] = HASH("IDS_FOO") and kResourceIndices[i] = 12345.
+
+// The definitions of the arrays are generated by generate_resources_map.py from
+// the content of generated_resources.h.
+
+// A sorted array of hashed generated resource names.
+extern const uint32_t kResourceHashes[];
+
+// An array of generated resource indices. The order of this array corresponds
+// to the order of |kResourceHashes|.
+extern const int kResourceIndices[];
+
+}  // namespace chrome_variations
+
+#endif  // CHROME_BROWSER_METRICS_VARIATIONS_GENERATED_RESOURCES_MAP_H_

chrome/chrome_browser.gypi

@@ -17,6 +17,7 @@
         'chrome_resources.gyp:chrome_extra_resources',
         'chrome_resources.gyp:chrome_resources',
         'chrome_resources.gyp:chrome_strings',
+        'chrome_resources.gyp:chrome_strings_map',
         'chrome_resources.gyp:platform_locale_settings',
         'chrome_resources.gyp:theme_resources',
         'common',
@@ -1217,6 +1218,7 @@
         'browser/metrics/tracking_synchronizer.cc',
         'browser/metrics/tracking_synchronizer.h',
         'browser/metrics/tracking_synchronizer_observer.h',
+        'browser/metrics/variations/generated_resources_map.h',
         'browser/metrics/variations/variations_http_header_provider.cc',
         'browser/metrics/variations/variations_http_header_provider.h',
         'browser/metrics/variations/variations_registry_syncer_win.cc',
@@ -2562,6 +2564,10 @@
         '<(grit_out_dir)/grit/component_extension_resources_map.cc',
         '<(grit_out_dir)/grit/theme_resources_map.cc',
         '<(SHARED_INTERMEDIATE_DIR)/ui/ui_resources/grit/ui_resources_map.cc',
+
+        # This file is generated by
+        # chrome/browser/metrics/variations/generate_resources_map.py
+        '<(SHARED_INTERMEDIATE_DIR)/chrome/browser/metrics/variations/generated_resources_map.cc',
       ],
       'conditions': [
         ['OS != "ios"', {

chrome/chrome_resources.gyp

@@ -257,6 +257,29 @@
       ],
       'includes': [ '../build/grit_target.gypi' ],
     },
+    {
+      'target_name': 'chrome_strings_map',
+      'type': 'none',
+      'dependencies': [ 'chrome_strings', ],
+      'actions': [
+        {
+          'action_name': 'generate_resources_map',
+          'inputs': [
+            '<(grit_out_dir)/grit/generated_resources.h'
+          ],
+          'outputs': [
+            '<(SHARED_INTERMEDIATE_DIR)/chrome/browser/metrics/variations/generated_resources_map.cc',
+          ],
+          'action': [
+            'python',
+            'browser/metrics/variations/generate_resources_map.py',
+            '<(grit_out_dir)/grit/generated_resources.h',
+            '<(SHARED_INTERMEDIATE_DIR)/chrome/browser/metrics/variations/generated_resources_map.cc'
+          ],
+          'message': 'Generating generated resources map.',
+        }
+      ],
+    },
     {
       'target_name': 'platform_locale_settings',
       'type': 'none',