build: Generate TypeScript defs with Clutz

The output of Clutz is not directly usable.  We must perform several
transformations on the generated defs:

 - Remove the prefix clutz puts on all namespaces ("ಠ_ಠ.clutz")
 - Replace "GlobalObject" (from Clutz) with TypeScript-native "object"
 - Remove "Global" prefix from Date, Element, Event, and EventTarget
   and use the original definitions instead
 - Remove "protected", which appears on some fields, but is only
   supported on methods
 - Remove duplicate extension of EventTarget on IAdManager
 - Transform "Promise | null | undefined" return type into "Promise |
   void"
 - Fix the definitions of Error and shaka.util.Error
 - Remove an unnecessary second interface on SegmentIndex types
 - Remove definitions for Closure Compiler built-ins

We still need some basic TypeScript sample code that references Shaka
Player, so that we can try to compile it and automatically verify that
the defs work at every commit.

Issue shaka-project#1030

Change-Id: If394269205e57dbcce4e0f76f6ef582fba0afacb

build/build.py

@@ -281,10 +281,6 @@ def build_library(self, name, locales, force, is_debug):
     build_name = 'shaka-player.' + name
     closure = compiler.ClosureCompiler(self.include, build_name)
 
-    # Don't pass node modules to the extern generator.
-    local_include = set([f for f in self.include if 'node_modules' not in f])
-    generator = compiler.ExternGenerator(local_include, build_name)
-
     closure_opts = common_closure_opts + common_closure_defines
     if is_debug:
       closure_opts += debug_closure_opts + debug_closure_defines
@@ -294,7 +290,21 @@ def build_library(self, name, locales, force, is_debug):
     if not closure.compile(closure_opts, force):
       return False
 
-    if not generator.generate(force):
+    # Don't pass node modules to the extern generator.
+    local_include = set([f for f in self.include if 'node_modules' not in f])
+    extern_generator = compiler.ExternGenerator(local_include, build_name)
+
+    if not extern_generator.generate(force):
+      return False
+
+    generated_externs = [extern_generator.output]
+    shaka_externs = shakaBuildHelpers.get_all_js_files('externs')
+    if self.has_ui():
+      shaka_externs += shakaBuildHelpers.get_all_js_files('ui/externs')
+    ts_def_generator = compiler.TsDefGenerator(
+        generated_externs + shaka_externs, build_name)
+
+    if not ts_def_generator.generate(force):
       return False
 
     return True

build/compiler.py

@@ -224,6 +224,35 @@ def generate(self, force=False):
     return True
 
 
+class TsDefGenerator(object):
+  def __init__(self, source_files, build_name):
+    self.source_files = _canonicalize_source_files(source_files)
+    self.output = _get_source_path('dist/' + build_name + '.d.ts')
+
+  def generate(self, force=False):
+    """Generates externs for the files in |self.source_files|.
+
+    Args:
+      force: Generate the output even if the inputs have not changed.
+
+    Returns:
+      True on success; False on failure.
+    """
+    if not force and not _must_build(self.output, self.source_files):
+      return True
+
+    def_generator = _get_source_path('build/generateTsDefs.py')
+
+    cmd_line = [def_generator, '--output', self.output]
+    cmd_line += self.source_files
+
+    if shakaBuildHelpers.execute_get_code(cmd_line) != 0:
+      logging.error('TS defs generation failed')
+      return False
+
+    return True
+
+
 class Less(object):
   def __init__(self, main_source_file, all_source_files, output):
     # Less only takes one input file, but that input may import others.

build/generateTsDefs.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generates TypeScript defs from Closure externs.
+
+This uses the Clutz tool from https://github.com/angular/clutz and then performs
+additional transformations to make the generated defs independent of Clutz.
+"""
+
+import argparse
+import os
+import re
+import sys
+
+import shakaBuildHelpers
+
+
+def GenerateTsDefs(inputs, output):
+  """Generates TypeScript defs from Closure externs.
+
+  Args:
+    inputs: A list of paths to Closure extern files.
+    output: A path to a TypeScript def output file.
+  """
+  clutz = shakaBuildHelpers.get_node_binary('@teppeis/clutz', 'clutz')
+
+  command = clutz + [
+      '--closure_env', 'BROWSER',
+      '--externs',
+  ] + inputs + [
+      '-o', '-',
+  ]
+
+  # Get the output of Clutz, then transform it to make it independent of Clutz
+  # and usable directly in TypeScript projects.
+  contents = shakaBuildHelpers.execute_get_output(command)
+
+  # Remove the prefix clutz puts on all namespaces.
+  contents = contents.replace('\xe0\xb2\xa0_\xe0\xb2\xa0.clutz.', '')
+  # Replace "GlobalObject" (from Clutz) with TypeScript-native "object".
+  contents = re.sub(r'\bGlobalObject\b', 'object', contents)
+  # Remove "Global" from Clutz's Global{Date,Element,Event,EventTarget} and use
+  # their original definitions instead.
+  contents = re.sub(
+      r'\bGlobal(Date|Element|Event|EventTarget)\b', r'\1', contents)
+  # Remove "protected", which appears on some fields, and is only supported on
+  # methods.
+  contents = re.sub(r'^\s*protected ', '', contents, flags=re.MULTILINE)
+  # Clutz really likes EventTargets, so it wants IAdManager to extend it twice.
+  contents = contents.replace(
+      'extends EventTarget extends EventTarget',
+      'extends EventTarget')
+  # Some types allow you to return a Promise or nothing, but the Clutz output of
+  # "Promise | null | undefined" doesn't work in TypeScript.  We need to use
+  # "Promise | void" instead.
+  contents = contents.replace('| null | undefined', '| void')
+  # shaka.util.Error extends Error and implements shaka.extern.Error, but this
+  # confuses TypeScript when both are called "Error" in context of the namespace
+  # declaration for "shaka.util".  Therefore we need to declare this
+  # "GlobalError" type, which is already referenced by Clutz but never defined.
+  global_error_def = 'declare class GlobalError extends Error {}\n\n'
+  contents = global_error_def + contents
+  # There are some types that implement multiple interfaces, such as IReleasable
+  # and Iteratable.  Also, there are tools used inside Google that (for some
+  # reason) want to convert TS defs _BACK_ into Closure Compiler externs.  When
+  # that happens, these multiple implementors generate externs with broken
+  # @implements annotations.  Since this team can't control that process or
+  # those tools, we need to adjust our TS defs instead.  In these particular
+  # cases, thankfully, we can just remove the reference to the IReleasable
+  # interface, which is not needed outside the library.
+  # TODO: This only covers one very specific pattern, and could be brittle.
+  contents = contents.replace(
+      'implements shaka.util.IReleasable , ', 'implements ')
+  # Finally, Clutz includes a bunch of basic defs for a browser environment
+  # generated from Closure compiler's builtins.  Remove these.
+  sections = re.split(r'\n(?=// Generated from .*)', contents)
+  sections = filter(
+      lambda s: not s.startswith('// Generated from externs.zip'),
+      sections)
+  contents = '\n'.join(sections) + '\n'
+
+  license_header_path = os.path.join(
+      shakaBuildHelpers.get_source_base(), 'build/license-header')
+
+  with open(license_header_path, 'r') as f:
+    license_header = f.read()
+
+  with open(output, 'w') as f:
+    f.write(license_header)
+    f.write('\n')
+    f.write(contents)
+
+
+def CreateParser():
+  """Create the argument parser for this application."""
+  base = shakaBuildHelpers.get_source_base()
+
+  parser = argparse.ArgumentParser(
+      description=__doc__,
+      formatter_class=argparse.RawDescriptionHelpFormatter)
+
+  parser.add_argument(
+      '--output',
+      type=str,
+      help='The file path for TypeScripts defs output')
+
+  parser.add_argument(
+      'input',
+      type=str,
+      nargs='+',
+      help='The Closure extern files to be converted to TypeScript defs')
+
+  return parser
+
+
+def main(args):
+  parser = CreateParser()
+  args = parser.parse_args(args)
+  GenerateTsDefs(args.input, args.output)
+  return 0
+
+
+if __name__ == '__main__':
+  main(sys.argv[1:])

package.json

@@ -11,6 +11,7 @@
     }
   ],
   "devDependencies": {
+    "@teppeis/clutz": "^1.0.29-4c95e12.v20190929",
     "awesomplete": "^1.1.4",
     "babel-core": "^6.26.3",
     "babel-plugin-istanbul": "^4.1.6",