mojo: use python3 to generate src files

This applies sort in some places and adds __lt__ in some classes for
that.

This doesn't generate deterministic mojo-module file dumped by python
pickle. But that file is not uploaded for test, so that is ignorable.

Bug: 1194274, 1205583, 1205585
Change-Id: Id820af101621229d587a92ae071341af3cf146b5
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2919072
Auto-Submit: Takuto Ikuta <tikuta@chromium.org>
Commit-Queue: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Nico Weber <thakis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#886958}

mojo/public/tools/bindings/BUILD.gn

@@ -2,12 +2,10 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import("//build/config/python.gni")
 import("//mojo/public/tools/bindings/mojom.gni")
 import("//third_party/jinja2/jinja2.gni")
 
-# TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds.
-python2_action("precompile_templates") {
+action("precompile_templates") {
   sources = mojom_generator_sources
   sources += [
     "$mojom_generator_root/generators/cpp_templates/enum_macros.tmpl",

mojo/public/tools/bindings/generators/js_templates/lite/mojom-lite.js.tmpl

@@ -12,7 +12,7 @@ goog.require('mojo.internal');
 {%- if interfaces %}
 goog.require('mojo.internal.interfaceSupport');
 {%- endif %}
-{% for kind in module.imported_kinds.values() %}
+{% for kind in module.imported_kinds.values()|sort %}
 goog.require('{{kind|lite_js_import_name}}');
 {%- endfor %}
 {% elif not for_bindings_internals  %}

mojo/public/tools/bindings/generators/js_templates/lite/mojom.m.js.tmpl

@@ -6,9 +6,9 @@
 import {mojo} from '{{bindings_library_path}}';
 {%- endif %}
 
-{% for path, kinds in js_module_imports.items() -%}
+{% for path, kinds in js_module_imports.items()|sort -%}
 import {
-{%- for kind in kinds %}
+{%- for kind in kinds|sort %}
 {%-   for item in kind|imports_for_kind %}
   {{item.name}} as {{item.alias}}
 {%-     if not loop.last -%},{% endif -%}

mojo/public/tools/bindings/mojom.gni

@@ -2,7 +2,6 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import("//build/config/python.gni")
 import("//third_party/closure_compiler/closure_args.gni")
 import("//third_party/closure_compiler/compile_js.gni")
 import("//third_party/protobuf/proto_library.gni")
@@ -683,8 +682,7 @@ template("mojom") {
       enabled_features += [ "is_win" ]
     }
 
-    # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds.
-    python2_action(parser_target_name) {
+    action(parser_target_name) {
       script = mojom_parser_script
       inputs = mojom_parser_sources + [ build_metadata_filename ]
       sources = sources_list
@@ -819,8 +817,7 @@ template("mojom") {
       }
     }
 
-    # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds.
-    python2_action(generator_cpp_message_ids_target_name) {
+    action(generator_cpp_message_ids_target_name) {
       script = mojom_generator_script
       inputs = mojom_generator_sources + jinja2_sources
       sources = sources_list
@@ -860,8 +857,7 @@ template("mojom") {
 
     generator_shared_target_name = "${target_name}_shared__generator"
 
-    # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds.
-    python2_action(generator_shared_target_name) {
+    action(generator_shared_target_name) {
       visibility = [ ":*" ]
       script = mojom_generator_script
       inputs = mojom_generator_sources + jinja2_sources
@@ -981,8 +977,7 @@ template("mojom") {
     generator_mojolpm_proto_target_name =
         "${target_name}_mojolpm_proto_generator"
 
-    # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds.
-    python2_action(generator_mojolpm_proto_target_name) {
+    action(generator_mojolpm_proto_target_name) {
       script = mojom_generator_script
       inputs = mojom_generator_sources + jinja2_sources
       sources = invoker.sources
@@ -1207,7 +1202,7 @@ template("mojom") {
       generator_target_name = "${target_name}${variant_suffix}__generator"
 
       # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds.
-      python2_action(generator_target_name) {
+      action(generator_target_name) {
         visibility = [ ":*" ]
         script = mojom_generator_script
         inputs = mojom_generator_sources + jinja2_sources
@@ -1389,8 +1384,7 @@ template("mojom") {
     write_file(_typemap_config_filename, _rebased_typemap_configs, "json")
     _mojom_target_name = target_name
 
-    # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds.
-    python2_action(_typemap_validator_target_name) {
+    action(_typemap_validator_target_name) {
       script = "$mojom_generator_root/validate_typemap_config.py"
       inputs = [ _typemap_config_filename ]
       outputs = [ _typemap_stamp_filename ]
@@ -1401,8 +1395,7 @@ template("mojom") {
       ]
     }
 
-    # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds.
-    python2_action(type_mappings_target_name) {
+    action(type_mappings_target_name) {
       inputs =
           mojom_generator_sources + jinja2_sources + [ _typemap_stamp_filename ]
       outputs = [ type_mappings_path ]
@@ -1583,8 +1576,7 @@ template("mojom") {
 
       java_generator_target_name = target_name + "_java__generator"
       if (sources_list != []) {
-        # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds.
-        python2_action(java_generator_target_name) {
+        action(java_generator_target_name) {
           script = mojom_generator_script
           inputs = mojom_generator_sources + jinja2_sources
           sources = sources_list
@@ -1624,8 +1616,7 @@ template("mojom") {
 
       java_srcjar_target_name = target_name + "_java_sources"
 
-      # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds.
-      python2_action(java_srcjar_target_name) {
+      action(java_srcjar_target_name) {
         script = "//build/android/gyp/zip.py"
         inputs = []
         if (output_file_base_paths != []) {
@@ -1686,8 +1677,7 @@ template("mojom") {
     if (sources_list != []) {
       generator_js_target_name = "${target_name}_js__generator"
 
-      # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds.
-      python2_action(generator_js_target_name) {
+      action(generator_js_target_name) {
         script = mojom_generator_script
         inputs = mojom_generator_sources + jinja2_sources
         sources = sources_list
@@ -1928,8 +1918,7 @@ template("mojom") {
       generator_ts_target_name =
           "${target_name}_${dependency_type.name}__ts__generator"
 
-      # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds.
-      python2_action(generator_ts_target_name) {
+      action(generator_ts_target_name) {
         script = mojom_generator_script
         inputs = mojom_generator_sources + jinja2_sources
         sources = sources_list
@@ -1996,8 +1985,7 @@ template("mojom") {
           "${target_name}_${dependency_type.name}__js__generator"
       generator_js_target_names += [ generator_js_target_name ]
 
-      # TODO(crbug.com/1194274): Investigate nondeterminism in Py3 builds.
-      python2_action(generator_js_target_name) {
+      action(generator_js_target_name) {
         script = "$mojom_generator_root/compile_typescript.py"
         sources = ts_outputs
         outputs = js_outputs

mojo/public/tools/mojom/mojom/generate/module.py

@@ -574,11 +574,18 @@ def qualified_name(self):
       prefix = self.module.GetNamespacePrefix()
     return '%s%s' % (prefix, self.mojom_name)
 
+  def _tuple(self):
+    return (self.mojom_name, self.native_only, self.fields, self.constants,
+            self.attributes)
+
   def __eq__(self, rhs):
-    return (isinstance(rhs, Struct) and
-            (self.mojom_name, self.native_only, self.fields, self.constants,
-             self.attributes) == (rhs.mojom_name, rhs.native_only, rhs.fields,
-                                  rhs.constants, rhs.attributes))
+    return isinstance(rhs, Struct) and self._tuple() == rhs._tuple()
+
+  def __lt__(self, rhs):
+    if not isinstance(self, type(rhs)):
+      return str(type(self)) < str(type(rhs))
+
+    return self._tuple() < rhs._tuple()
 
   def __hash__(self):
     return id(self)
@@ -690,10 +697,17 @@ def qualified_name(self):
       prefix = self.module.GetNamespacePrefix()
     return '%s%s' % (prefix, self.mojom_name)
 
+  def _tuple(self):
+    return (self.mojom_name, self.fields, self.attributes)
+
   def __eq__(self, rhs):
-    return (isinstance(rhs, Union) and
-            (self.mojom_name, self.fields,
-             self.attributes) == (rhs.mojom_name, rhs.fields, rhs.attributes))
+    return isinstance(rhs, Union) and self._tuple() == rhs._tuple()
+
+  def __lt__(self, rhs):
+    if not isinstance(self, type(rhs)):
+      return str(type(self)) < str(type(rhs))
+
+    return self._tuple() < rhs._tuple()
 
   def __hash__(self):
     return id(self)
@@ -1061,12 +1075,18 @@ def unlimited_message_size(self):
     return self.attributes.get(ATTRIBUTE_UNLIMITED_SIZE) \
         if self.attributes else False
 
+  def _tuple(self):
+    return (self.mojom_name, self.ordinal, self.parameters,
+            self.response_parameters, self.attributes)
+
   def __eq__(self, rhs):
-    return (isinstance(rhs, Method) and
-            (self.mojom_name, self.ordinal, self.parameters,
-             self.response_parameters,
-             self.attributes) == (rhs.mojom_name, rhs.ordinal, rhs.parameters,
-                                  rhs.response_parameters, rhs.attributes))
+    return isinstance(rhs, Method) and self._tuple() == rhs._tuple()
+
+  def __lt__(self, rhs):
+    if not isinstance(self, type(rhs)):
+      return str(type(self)) < str(type(rhs))
+
+    return self._tuple() < rhs._tuple()
 
 
 class Interface(ReferenceKind):
@@ -1199,11 +1219,18 @@ def qualified_name(self):
       prefix = self.module.GetNamespacePrefix()
     return '%s%s' % (prefix, self.mojom_name)
 
+  def _tuple(self):
+    return (self.mojom_name, self.methods, self.enums, self.constants,
+            self.attributes)
+
   def __eq__(self, rhs):
-    return (isinstance(rhs, Interface)
-            and (self.mojom_name, self.methods, self.enums, self.constants,
-                 self.attributes) == (rhs.mojom_name, rhs.methods, rhs.enums,
-                                      rhs.constants, rhs.attributes))
+    return isinstance(rhs, Interface) and self._tuple() == rhs._tuple()
+
+  def __lt__(self, rhs):
+    if not isinstance(self, type(rhs)):
+      return str(type(self)) < str(type(rhs))
+
+    return self._tuple() < rhs._tuple()
 
   @property
   def uuid(self):
@@ -1358,13 +1385,18 @@ def buildVersionFieldMap(enum):
 
     return True
 
+  def _tuple(self):
+    return (self.mojom_name, self.native_only, self.fields, self.attributes,
+            self.min_value, self.max_value, self.default_field)
+
   def __eq__(self, rhs):
-    return (isinstance(rhs, Enum) and
-            (self.mojom_name, self.native_only, self.fields, self.attributes,
-             self.min_value, self.max_value,
-             self.default_field) == (rhs.mojom_name, rhs.native_only,
-                                     rhs.fields, rhs.attributes, rhs.min_value,
-                                     rhs.max_value, rhs.default_field))
+    return isinstance(rhs, Enum) and self._tuple() == rhs._tuple()
+
+  def __lt__(self, rhs):
+    if not isinstance(self, type(rhs)):
+      return str(type(self)) < str(type(rhs))
+
+    return self._tuple() < rhs._tuple()
 
   def __hash__(self):
     return id(self)

mojo/public/tools/mojom/mojom_parser.py

@@ -139,7 +139,7 @@ def _EnsureInputLoaded(mojom_abspath, module_path, abs_paths, asts,
     # Already done.
     return
 
-  for dep_abspath, dep_path in dependencies[mojom_abspath]:
+  for dep_abspath, dep_path in sorted(dependencies[mojom_abspath]):
     if dep_abspath not in loaded_modules:
       _EnsureInputLoaded(dep_abspath, dep_path, abs_paths, asts, dependencies,
                          loaded_modules, module_metadata)
@@ -274,7 +274,7 @@ def _ParseMojoms(mojom_files,
     loaded_mojom_asts[mojom_abspath] = ast
 
   logging.info('Processing dependencies')
-  for mojom_abspath, ast in loaded_mojom_asts.items():
+  for mojom_abspath, ast in sorted(loaded_mojom_asts.items()):
     invalid_imports = []
     for imp in ast.import_list:
       import_abspath = _ResolveRelativeImportPath(imp.import_filename,