Bug 1361661 - Part 1: Generate headers with process data from Process…

…es.yaml. r=dexter

Adding the Gecko enums to Processes.yaml allows us to generate mappings from ProcessID to GeckoProcessType.
We generate string tables with the Telemetry process names, so we can use these names consistently throughout Telemetry.

CLOBBER

@@ -22,4 +22,5 @@
 # changes to stick? As of bug 928195, this shouldn't be necessary! Please
 # don't change CLOBBER for WebIDL changes any more.
 
-Bug 1340627 - clobber for Skia update
+Bug 1361661 - Update Telemetry build and headers.
+

toolkit/components/telemetry/Processes.yaml

@@ -6,12 +6,16 @@
 # For now this is only used to inform the data pipeline about new processes, but will be used to
 # generate headers with C++ data later (enums, strings, ...).
 parent:
+  gecko_enum: GeckoProcessType_Default
   description: This is the main process. It is also known as the parent or chrome process.
 content:
+  gecko_enum: GeckoProcessType_Content
   description: This is for processes web content is rendered in.
 extension:
+  gecko_enum: GeckoProcessType_Content
   description: >
     This is the WebExtension process. It is a re-used content process, with the data submitted
     separately to avoid skewing other content process Telemetry.
 gpu:
+  gecko_enum: GeckoProcessType_GPU
   description: This is the compositor or GPU process.

toolkit/components/telemetry/gen-process-data.py

@@ -0,0 +1,72 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Write out processes data for C++. The processes are defined
+# in a file provided as a command-line argument.
+
+from __future__ import print_function
+from shared_telemetry_utils import ParserError, load_processes
+
+import sys
+import collections
+
+# The banner/text at the top of the generated file.
+banner = """/* This file is auto-generated from Telemetry build scripts,
+   see gen-processes-data.py. */
+"""
+
+file_header = """\
+#ifndef mozilla_TelemetryProcessData_h
+#define mozilla_TelemetryProcessData_h
+
+#include "mozilla/TelemetryProcessEnums.h"
+
+namespace mozilla {
+namespace Telemetry {
+"""
+
+file_footer = """
+} // namespace Telemetry
+} // namespace mozilla
+#endif // mozilla_TelemetryProcessData_h"""
+
+
+def to_enum_label(name):
+    return name.title().replace('_', '')
+
+
+def write_processes_data(processes, output):
+    def p(line):
+        print(line, file=output)
+    processes = collections.OrderedDict(processes)
+
+    p("static GeckoProcessType ProcessIDToGeckoProcessType[%d] = {" % len(processes))
+    for i, (name, value) in enumerate(processes.iteritems()):
+        p("  /* %d: ProcessID::%s = */ %s," % (i, to_enum_label(name), value['gecko_enum']))
+    p("};")
+    p("")
+    p("static const char* const ProcessIDToString[%d] = {" % len(processes))
+    for i, (name, value) in enumerate(processes.iteritems()):
+        p("  /* %d: ProcessID::%s = */ \"%s\"," % (i, to_enum_label(name), name))
+    p("};")
+
+
+def main(output, *filenames):
+    if len(filenames) > 1:
+        raise Exception('We don\'t support loading from more than one file.')
+
+    try:
+        processes = load_yaml_file(filenames[0])
+
+        # Write the process data file.
+        print(banner, file=output)
+        print(file_header, file=output)
+        write_processes_data(processes, output)
+        print(file_footer, file=output)
+    except ParserError as ex:
+        print("\nError generating processes data:\n" + str(ex) + "\n")
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main(sys.stdout, *sys.argv[1:])

toolkit/components/telemetry/gen-process-enum.py

@@ -0,0 +1,66 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Write out processes data for C++. The processes are defined
+# in a file provided as a command-line argument.
+
+from __future__ import print_function
+from shared_telemetry_utils import ParserError, load_processes
+
+import sys
+import collections
+
+# The banner/text at the top of the generated file.
+banner = """/* This file is auto-generated from Telemetry build scripts,
+   see gen-processes-enum.py. */
+"""
+
+file_header = """\
+#ifndef mozilla_TelemetryProcessEnums_h
+#define mozilla_TelemetryProcessEnums_h
+
+namespace mozilla {
+namespace Telemetry {
+"""
+
+file_footer = """
+} // namespace Telemetry
+} // namespace mozilla
+#endif // mozilla_TelemetryProcessEnums_h"""
+
+
+def to_enum_label(name):
+    return name.title().replace('_', '')
+
+
+def write_processes_enum(processes, output):
+    def p(line):
+        print(line, file=output)
+    processes = collections.OrderedDict(processes)
+
+    p("enum class ProcessID : uint32_t {")
+    for i, (name, _) in enumerate(processes.iteritems()):
+        p("  %s = %d," % (to_enum_label(name), i))
+    p("  Count = %d" % len(processes))
+    p("};")
+
+
+def main(output, *filenames):
+    if len(filenames) > 1:
+        raise Exception('We don\'t support loading from more than one file.')
+
+    try:
+        processes = load_yaml_file(filenames[0])
+
+        # Write the process data file.
+        print(banner, file=output)
+        print(file_header, file=output)
+        write_processes_enum(processes, output)
+        print(file_footer, file=output)
+    except ParserError as ex:
+        print("\nError generating processes enums:\n" + str(ex) + "\n")
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main(sys.stdout, *sys.argv[1:])

toolkit/components/telemetry/moz.build

@@ -43,6 +43,7 @@ XPIDL_MODULE = 'telemetry'
 EXPORTS.mozilla += [
     '!TelemetryEventEnums.h',
     '!TelemetryHistogramEnums.h',
+    '!TelemetryProcessEnums.h',
     '!TelemetryScalarEnums.h',
     'ipc/TelemetryComms.h',
     'ipc/TelemetryIPC.h',
@@ -94,6 +95,8 @@ GENERATED_FILES = [
     'TelemetryEventEnums.h',
     'TelemetryHistogramData.inc',
     'TelemetryHistogramEnums.h',
+    'TelemetryProcessData.h',
+    'TelemetryProcessEnums.h',
     'TelemetryScalarData.h',
     'TelemetryScalarEnums.h',
 ]
@@ -139,5 +142,18 @@ event_enums = GENERATED_FILES['TelemetryEventEnums.h']
 event_enums.script = 'gen-event-enum.py'
 event_enums.inputs = event_files
 
+# Generate data from Processes.yaml
+processes_files = [
+    'Processes.yaml',
+]
+
+processes_enum = GENERATED_FILES['TelemetryProcessEnums.h']
+processes_enum.script = 'gen-process-enum.py'
+processes_enum.inputs = processes_files
+
+processes_data = GENERATED_FILES['TelemetryProcessData.h']
+processes_data.script = 'gen-process-data.py'
+processes_data.inputs = processes_files
+
 with Files('**'):
     BUG_COMPONENT = ('Toolkit', 'Telemetry')

toolkit/components/telemetry/shared_telemetry_utils.py

@@ -8,6 +8,7 @@
 from __future__ import print_function
 
 import re
+import yaml
 
 # This is a list of flags that determine which process a measurement is allowed
 # to record from.
@@ -130,3 +131,15 @@ def add_expiration_postfix(expiration):
         return expiration + "a1"
 
     return expiration
+
+
+def load_yaml_file(filename):
+    """ Load a YAML file from disk, throw a ParserError on failure."""
+    try:
+        with open(filename, 'r') as f:
+            return yaml.safe_load(f)
+    except IOError, e:
+        raise ParserError('Error opening ' + filename + ': ' + e.message)
+    except ValueError, e:
+        raise ParserError('Error parsing processes in {}: {}'
+                          .format(filename, e.message))