From ef6dd6e1fc6084bf49ed44e9a60c53365fc438e5 Mon Sep 17 00:00:00 2001 From: Toni Barzic Date: Sat, 15 May 2021 02:49:11 +0000 Subject: [PATCH] Set encoding when calling open() in few tools/ python scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The script is failing with UnicodeDecodeError when building Chrome on Chrome OS builders. BUG=1208896 Change-Id: I3d3cd0928e38362bec79ec3085d16bb329f9b0e8 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2894764 Commit-Queue: Toni Baržić Commit-Queue: Nico Weber Reviewed-by: Nico Weber Cr-Commit-Position: refs/heads/master@{#883250} --- tools/flags/generate_expired_list.py | 5 +- tools/flags/list_flags.py | 4 +- tools/json_to_struct/json_to_struct.py | 95 ++++++++++++++------------ 3 files changed, 56 insertions(+), 48 deletions(-) diff --git a/tools/flags/generate_expired_list.py b/tools/flags/generate_expired_list.py index 33d7edccb534f6..f199aab62a973a 100755 --- a/tools/flags/generate_expired_list.py +++ b/tools/flags/generate_expired_list.py @@ -28,7 +28,8 @@ def get_chromium_version(): """Parses the Chromium version out of //chrome/VERSION.""" - with open(os.path.join(ROOT_PATH, 'chrome', 'VERSION')) as f: + with open(os.path.join(ROOT_PATH, 'chrome', 'VERSION'), + encoding='utf-8') as f: for line in f.readlines(): key, value = line.strip().split('=') if key == 'MAJOR': @@ -118,7 +119,7 @@ def main(): return output = gen_expiry_file(sys.argv[0], sys.argv[1]) - with open(sys.argv[2], 'w') as f: + with open(sys.argv[2], 'w', encoding='utf-8') as f: f.write(output) diff --git a/tools/flags/list_flags.py b/tools/flags/list_flags.py index ad693b7abfd20b..e43cdf86556b44 100755 --- a/tools/flags/list_flags.py +++ b/tools/flags/list_flags.py @@ -27,7 +27,7 @@ def load_metadata(): flags_path = os.path.join(ROOT_PATH, 'chrome', 'browser', 'flag-metadata.json') - return json5.load(open(flags_path)) + return json5.load(open(flags_path, encoding='utf-8')) def keep_expired_by(flags, mstone): @@ -89,7 +89,7 @@ def find_unused(flags): 'chrome/browser/about_flags.cc', 'ios/chrome/browser/flags/about_flags.mm', ] - flag_files_data = [open(f, 'rb').read().decode('utf-8') for f in FLAG_FILES] + flag_files_data = [open(f, 'rb', encoding='utf-8').read() for f in FLAG_FILES] unused_flags = [] for flag in flags: # Search for the name in quotes. diff --git a/tools/json_to_struct/json_to_struct.py b/tools/json_to_struct/json_to_struct.py index f3668d06528505..c98d3cd8c382ec 100755 --- a/tools/json_to_struct/json_to_struct.py +++ b/tools/json_to_struct/json_to_struct.py @@ -58,6 +58,7 @@ import json from datetime import datetime +import io import os.path import sys import optparse @@ -73,7 +74,7 @@ import struct_generator import element_generator -HEAD = """// Copyright %d The Chromium Authors. All rights reserved. +HEAD = u"""// Copyright %d The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. @@ -87,8 +88,9 @@ def _GenerateHeaderGuard(h_filename): """Generates the string used in #ifndef guarding the header file. """ - result = re.sub('[%s\\\\.]' % os.sep, '_', h_filename.upper()) - return re.sub('^_*', '', result) + '_' # Remove leading underscores. + result = re.sub(u'[%s\\\\.]' % os.sep, u'_', h_filename.upper()) + return re.sub(u'^_*', '', result) + u'_' # Remove leading underscores. + def _GenerateH(basepath, fileroot, head, namespace, schema, description): """Generates the .h file containing the definition of the structure specified @@ -105,50 +107,52 @@ def _GenerateH(basepath, fileroot, head, namespace, schema, description): this file. """ - h_filename = fileroot + '.h' - with open(os.path.join(basepath, h_filename), 'w') as f: + h_filename = fileroot + u'.h' + with io.open(os.path.join(basepath, h_filename), 'w', encoding='utf-8') as f: f.write(head) header_guard = _GenerateHeaderGuard(h_filename) - f.write('#ifndef %s\n' % header_guard) - f.write('#define %s\n' % header_guard) - f.write('\n') + f.write(u'#ifndef %s\n' % header_guard) + f.write(u'#define %s\n' % header_guard) + f.write(u'\n') - f.write('#include \n') - f.write('\n') + f.write(u'#include \n') + f.write(u'\n') - for header in schema.get('headers', []): - f.write('#include "%s"\n' % header) - f.write('\n') + for header in schema.get(u'headers', []): + f.write(u'#include "%s"\n' % header) + f.write(u'\n') if namespace: - f.write('namespace %s {\n' % namespace) - f.write('\n') + f.write(u'namespace %s {\n' % namespace) + f.write(u'\n') f.write(struct_generator.GenerateStruct( schema['type_name'], schema['schema'])) - f.write('\n') + f.write(u'\n') for var_name, value in description.get('int_variables', {}).items(): - f.write('extern const int %s;\n' % var_name) - f.write('\n') + f.write(u'extern const int %s;\n' % var_name) + f.write(u'\n') for element_name, element in description['elements'].items(): - f.write('extern const %s %s;\n' % (schema['type_name'], element_name)) + f.write(u'extern const %s %s;\n' % (schema['type_name'], element_name)) if 'generate_array' in description: - f.write('\n') - f.write('extern const %s* const %s[];\n' % (schema['type_name'], - description['generate_array']['array_name'])) - f.write('extern const size_t %s;\n' % - (description['generate_array']['array_name'] + 'Length')) + f.write(u'\n') + f.write( + u'extern const %s* const %s[];\n' % + (schema['type_name'], description['generate_array']['array_name'])) + f.write(u'extern const size_t %s;\n' % + (description['generate_array']['array_name'] + u'Length')) if namespace: - f.write('\n') - f.write('} // namespace %s\n' % namespace) + f.write(u'\n') + f.write(u'} // namespace %s\n' % namespace) + + f.write(u'\n') + f.write(u'#endif // %s\n' % header_guard) - f.write('\n') - f.write( '#endif // %s\n' % header_guard) def _GenerateCC(basepath, fileroot, head, namespace, schema, description): """Generates the .cc file containing the static initializers for the @@ -165,39 +169,42 @@ def _GenerateCC(basepath, fileroot, head, namespace, schema, description): this file. """ - with open(os.path.join(basepath, fileroot + '.cc'), 'w') as f: + with io.open(os.path.join(basepath, fileroot + '.cc'), 'w', + encoding='utf-8') as f: f.write(head) - f.write('#include "%s"\n' % (fileroot + '.h')) - f.write('\n') + f.write(u'#include "%s"\n' % (fileroot + u'.h')) + f.write(u'\n') if namespace: - f.write('namespace %s {\n' % namespace) - f.write('\n') + f.write(u'namespace %s {\n' % namespace) + f.write(u'\n') f.write(element_generator.GenerateElements(schema['type_name'], schema['schema'], description)) if 'generate_array' in description: - f.write('\n') - f.write('const %s* const %s[] = {\n' % (schema['type_name'], - description['generate_array']['array_name'])) + f.write(u'\n') + f.write( + u'const %s* const %s[] = {\n' % + (schema['type_name'], description['generate_array']['array_name'])) for element_name, _ in description['elements'].items(): - f.write('\t&%s,\n' % element_name) - f.write('};\n') - f.write('const size_t %s = %d;\n' % - (description['generate_array']['array_name'] + 'Length', - len(description['elements']))) + f.write(u'\t&%s,\n' % element_name) + f.write(u'};\n') + f.write(u'const size_t %s = %d;\n' % + (description['generate_array']['array_name'] + u'Length', + len(description['elements']))) if namespace: - f.write('\n') - f.write('} // namespace %s\n' % namespace) + f.write(u'\n') + f.write(u'} // namespace %s\n' % namespace) + def _Load(filename): """Loads a JSON file int a Python object and return this object. """ # TODO(beaudoin): When moving to Python 2.7 use object_pairs_hook=OrderedDict. - with open(filename, 'r') as handle: + with io.open(filename, 'r', encoding='utf-8') as handle: result = json.loads(json_comment_eater.Nom(handle.read())) return result