[build-script] Add option --infer to infer dependencies.

This causes build-script to use the conservative dependency information that I
committed. When one uses this option, it is assumed that one wants to also
install all built products.

Some notes:

1. I included an extra --install-all option so without --infer enabled
   one can enable this sort of install everything that we want to
   build behavior.

2. I added %cmake as a lit variable. I did this so I could specify in
   my build-system unit tests that on Linux they should use the just
   built cmake (if we built cmake due to an old cmake being on the
   system). Otherwise, the build system unit tests took way too
   long. These are meant to be dry-runs, so building this cmake again
   is just wasteful and doesn't make sense.

3. I unified how we handle cmark, llvm, swift with the rest of the
   build products by making them conditional on build_* variables, but
   to preserve current behavior I made it so that they are just
   enabled by default unlike things like
   llbuild/swiftpm/foundation/etc. This was necessary since previously
   we would just pass these flags to build-script-impl and
   build-script didn't know about them. Now I taught build-script
   about them so I can manipulate these skip-build-{cmark,llvm,swift}
   and then just pass them down to build-script-impl if appropriate
   rather than relying on build-script-impl to control if these are
   built.

Once this lands, I think we are at a good enough place with
build-script until we get rid of build-script-impl in terms of high
value QoI that will imnprove productivity. Once build-script-impl is
destroyed, we can start paring back what build-script itself does.

Author: gottesmm

test/lit.cfg

@@ -146,7 +146,7 @@ config.test_format = swift_test.SwiftTest(coverage_mode=config.coverage_mode,
 
 # suffixes: A list of file extensions to treat as test files.
 config.suffixes = ['.swift', '.ll', '.sil', '.gyb', '.m', '.c',
-                   '.swiftinterface', '.test-sh']
+                   '.swiftinterface', '.test-sh', '.test']
 
 # excludes: A list of directories to exclude from the testsuite. The 'Inputs'
 # subdirectories contain auxiliary inputs for various tests in their parent
@@ -162,6 +162,11 @@ config.test_source_root = os.path.dirname(__file__)
 # test_exec_root: The root path where tests should be run.
 swift_obj_root = getattr(config, 'swift_obj_root', None)
 
+# cmake. The path to the cmake executable we used to configure swift.
+assert(config.cmake)
+config.substitutions.append( ('%cmake',  config.cmake) )
+lit_config.note('Using cmake: ' + config.cmake)
+
 # Set llvm_{src,obj}_root for use by others.
 config.llvm_src_root = getattr(config, 'llvm_src_root', None)
 config.llvm_obj_root = getattr(config, 'llvm_obj_root', None)

test/lit.site.cfg.in

@@ -14,6 +14,7 @@ import os
 import platform
 import sys
 
+config.cmake = "@CMAKE_COMMAND@"
 config.llvm_src_root = "@LLVM_MAIN_SRC_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
 config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"

utils/build-script

@@ -35,6 +35,7 @@ from build_swift.build_swift.constants import SWIFT_SOURCE_ROOT
 
 import six
 
+from swift_build_support.swift_build_support import build_graph
 from swift_build_support.swift_build_support import products
 from swift_build_support.swift_build_support import shell
 from swift_build_support.swift_build_support import targets
@@ -375,6 +376,10 @@ class BuildScriptInvocation(object):
 
         self.build_libparser_only = args.build_libparser_only
 
+    @property
+    def install_all(self):
+        return self.args.install_all or self.args.infer_dependencies
+
     def build_ninja(self):
         if not os.path.exists(self.workspace.source_dir("ninja")):
             fatal_error(
@@ -548,20 +553,28 @@ class BuildScriptInvocation(object):
         # Currently we do not build external benchmarks by default.
         if args.build_external_benchmarks:
             impl_args += ["--skip-build-external-benchmarks=0"]
-        if not args.build_foundation:
-            impl_args += ["--skip-build-foundation"]
-        if not args.build_xctest:
-            impl_args += ["--skip-build-xctest"]
-        if not args.build_lldb:
-            impl_args += ["--skip-build-lldb"]
-        if not args.build_llbuild:
-            impl_args += ["--skip-build-llbuild"]
-        if not args.build_libcxx:
-            impl_args += ["--skip-build-libcxx"]
-        if not args.build_libdispatch:
-            impl_args += ["--skip-build-libdispatch"]
-        if not args.build_libicu:
-            impl_args += ["--skip-build-libicu"]
+
+        # Then add subproject install flags that either skip building them /or/
+        # if we are going to build them and install_all is set, we also install
+        # them.
+        conditional_subproject_configs = [
+            (args.build_cmark, "cmark"),
+            (args.build_llvm, "llvm"),
+            (args.build_swift, "swift"),
+            (args.build_foundation, "foundation"),
+            (args.build_xctest, "xctest"),
+            (args.build_lldb, "lldb"),
+            (args.build_llbuild, "llbuild"),
+            (args.build_libcxx, "libcxx"),
+            (args.build_libdispatch, "libdispatch"),
+            (args.build_libicu, "libicu")
+        ]
+        for (should_build, string_name) in conditional_subproject_configs:
+            if not should_build and not self.args.infer_dependencies:
+                impl_args += ["--skip-build-{}".format(string_name)]
+            elif self.install_all:
+                impl_args += ["--install-{}".format(string_name)]
+
         if args.build_swift_dynamic_stdlib:
             impl_args += ["--build-swift-dynamic-stdlib"]
         if args.build_swift_static_stdlib:
@@ -816,13 +829,16 @@ class BuildScriptInvocation(object):
         # FIXME: This is a weird division (returning a list of class objects),
         # but it matches the existing structure of the `build-script-impl`.
         impl_product_classes = []
-        impl_product_classes.append(products.CMark)
-        impl_product_classes.append(products.LLVM)
+        if self.args.build_cmark:
+            impl_product_classes.append(products.CMark)
+        if self.args.build_llvm:
+            impl_product_classes.append(products.LLVM)
         if self.args.build_libcxx:
             impl_product_classes.append(products.LibCXX)
         if self.args.build_libicu:
             impl_product_classes.append(products.LibICU)
-        impl_product_classes.append(products.Swift)
+        if self.args.build_swift:
+            impl_product_classes.append(products.Swift)
         if self.args.build_lldb:
             impl_product_classes.append(products.LLDB)
         if self.args.build_libdispatch:
@@ -868,6 +884,30 @@ class BuildScriptInvocation(object):
         for prod in product_classes:
             assert(not prod.is_build_script_impl_product())
 
+        # Now that we have our two lists of product_classes, if we are asked to
+        # infer dependencies, infer the dependencies now and then re-split the
+        # list.
+        if self.args.infer_dependencies:
+            combined = impl_product_classes + product_classes
+
+            # Now that we have produced the schedule, resplit. We require our
+            # dependencies to respect our build-script-impl property. This means
+            # that no build-script-impl products can have dependencies on
+            # non-build-script-impl products. Otherwise, it would be unsafe to
+            # re-order build-script-impl products in front of non
+            # build-script-impl products.
+            impl_product_classes = []
+            product_classes = []
+            is_darwin = platform.system() == 'Darwin'
+            final_schedule = build_graph.produce_scheduled_build(combined)[0]
+            for p in final_schedule:
+                if is_darwin and p.is_nondarwin_only_build_product():
+                    continue
+
+                if p.is_build_script_impl_product():
+                    impl_product_classes.append(p)
+                else:
+                    product_classes.append(p)
         return (impl_product_classes, product_classes)
 
     def execute(self):
@@ -956,7 +996,8 @@ class BuildScriptInvocation(object):
                     print("--- Running tests for %s ---" % product_name)
                     product.test(host_target)
                     print("--- Finished tests for %s ---" % product_name)
-                if product.should_install(host_target):
+                if product.should_install(host_target) or \
+                   (self.install_all and product.should_build(host_target)):
                     print("--- Installing %s ---" % product_name)
                     product.install(host_target)
 

utils/build_swift/build_swift/driver_arguments.py

@@ -345,6 +345,8 @@ def create_argument_parser():
            help='the path to install debug symbols into')
     option('--install-destdir', store_path,
            help='the path to use as the filesystem root for the installation')
+    option('--install-all', toggle_true,
+           help='Assume all built products should be installed')
 
     option(['-j', '--jobs'], store_int('build_jobs'),
            default=multiprocessing.cpu_count(),
@@ -545,6 +547,9 @@ def create_argument_parser():
     # -------------------------------------------------------------------------
     in_group('Options to select projects')
 
+    option('--infer', store_true('infer_dependencies'),
+           help='Infer any downstream dependencies from enabled projects')
+
     option(['-l', '--lldb'], store_true('build_lldb'),
            help='build LLDB')
 
@@ -1096,6 +1101,16 @@ def create_argument_parser():
 
     # -------------------------------------------------------------------------
     in_group('Build-script-impl arguments (for disambiguation)')
+
+    # We need to represent these options so that we can skip installing them if
+    # the user is running in install-all mode.
+    option('--skip-build-cmark', toggle_false('build_cmark'),
+           help='skip building cmark')
+    option('--skip-build-llvm', toggle_false('build_llvm'),
+           help='skip building llvm')
+    option('--skip-build-swift', toggle_false('build_swift'),
+           help='skip building swift')
+
     # We need to list --skip-test-swift explicitly because otherwise argparse
     # will auto-expand arguments like --skip-test-swift to the only known
     # argument --skip-test-swiftevolve.

utils/build_swift/tests/build_swift/test_presets.py

@@ -39,6 +39,7 @@
     'extra_swift_args': '',
     'install_destdir': '/tmp/install',
     'install_symroot': '/tmp/install/symroot',
+    'install_all': False,
     'install_toolchain_dir': '/tmp/install/toolchain',
     'install_prefix': '/usr',
     'installable_package': '/tmp/install/pkg',

utils/build_swift/tests/expected_options.py

@@ -61,6 +61,9 @@
     'build_cygwin': True,
     'build_external_benchmarks': False,
     'build_foundation': False,
+    'build_cmark': True,
+    'build_swift': True,
+    'build_llvm': True,
     'build_freebsd': True,
     'build_ios': True,
     'build_ios_device': False,
@@ -159,9 +162,11 @@
     'host_test': False,
     'only_executable_test': False,
     'only_non_executable_test': False,
+    'infer_dependencies': False,
     'install_prefix': targets.install_prefix(),
     'install_symroot': None,
     'install_destdir': None,
+    'install_all': False,
     'ios': False,
     'ios_all': False,
     'legacy_impl': False,
@@ -482,6 +487,7 @@ class BuildScriptImplOption(_BaseOption):
     SetTrueOption('-p', dest='build_swiftpm'),
 
     SetTrueOption('--legacy-impl', dest='legacy_impl'),
+    SetTrueOption('--infer', dest='infer_dependencies'),
 
     EnableOption('--android'),
     EnableOption('--build-external-benchmarks'),
@@ -537,6 +543,10 @@ class BuildScriptImplOption(_BaseOption):
     EnableOption('--watchos'),
     EnableOption('--xctest', dest='build_xctest'),
 
+    DisableOption('--skip-build-cmark', dest='build_cmark'),
+    DisableOption('--skip-build-llvm', dest='build_llvm'),
+    DisableOption('--skip-build-swift', dest='build_swift'),
+
     DisableOption('--skip-build-android', dest='build_android'),
     DisableOption('--skip-build-benchmarks', dest='build_benchmarks'),
     DisableOption('--skip-build-cygwin', dest='build_cygwin'),
@@ -632,6 +642,7 @@ class BuildScriptImplOption(_BaseOption):
     PathOption('--install-prefix'),
     PathOption('--install-symroot'),
     PathOption('--install-destdir'),
+    EnableOption('--install-all'),
     PathOption('--symbols-package'),
     PathOption('--cmake-c-launcher'),
     PathOption('--cmake-cxx-launcher'),

utils/swift_build_support/swift_build_support/cmake.py

@@ -272,7 +272,7 @@ def check_cmake_version(self, source_root, build_root):
             cmake_binary = 'cmake'
 
         installed_ver = self.installed_cmake_version(cmake_binary)
-        if installed_ver > self.cmake_source_version(cmake_source_dir):
+        if installed_ver >= self.cmake_source_version(cmake_source_dir):
             return
         else:
             # Build CMake from source and return the path to the executable.

validation-test/BuildSystem/infer_implies_install_all.test

@@ -0,0 +1,10 @@
+# RUN: %empty-directory(%t)
+# RUN: mkdir -p %t
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --infer --swiftpm --cmake %cmake 2>&1 | %FileCheck %s
+
+# CHECK: --- Installing cmark ---
+# CHECK: --- Installing llvm ---
+# CHECK: --- Installing swift ---
+# CHECK: --- Installing llbuild ---
+# CHECK: --- Building swiftpm ---
+# CHECK: --- Installing swiftpm ---

validation-test/BuildSystem/install_all.test

@@ -0,0 +1,7 @@
+# RUN: %empty-directory(%t)
+# RUN: mkdir -p %t
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --install-all --cmake %cmake 2>&1 | %FileCheck %s
+
+# CHECK: --- Installing cmark ---
+# CHECK: --- Installing llvm ---
+# CHECK: --- Installing swift ---

validation-test/BuildSystem/install_all_linux.test

@@ -0,0 +1,13 @@
+# RUN: %empty-directory(%t)
+# RUN: mkdir -p %t
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --install-all --llbuild --swiftpm --foundation --libdispatch --cmake %cmake 2>&1 | %FileCheck %s
+
+# REQUIRES: OS=linux-gnu
+
+# CHECK-DAG: --- Installing cmark ---
+# CHECK-DAG: --- Installing swift ---
+# CHECK-DAG: --- Installing llvm ---
+# CHECK-DAG: --- Installing llbuild ---
+# CHECK-DAG: --- Installing foundation ---
+# CHECK-DAG: --- Installing libdispatch ---
+# CHECK-DAG: --- Installing swiftpm ---

validation-test/BuildSystem/skip_cmark_swift_llvm.test

@@ -0,0 +1,33 @@
+# RUN: %empty-directory(%t)
+# RUN: mkdir -p %t
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --install-all --cmake %cmake --skip-build-cmark 2>&1 | %FileCheck --check-prefix=SKIP-CMARK-CHECK %s
+
+# RUN: %empty-directory(%t)
+# RUN: mkdir -p %t
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --install-all --cmake %cmake --skip-build-llvm 2>&1 | %FileCheck --check-prefix=SKIP-LLVM-CHECK %s
+
+# RUN: %empty-directory(%t)
+# RUN: mkdir -p %t
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --install-all --cmake %cmake --skip-build-swift 2>&1 | %FileCheck --check-prefix=SKIP-SWIFT-CHECK %s
+
+# SKIP-CMARK-CHECK-NOT: cmake --build {{.*}}cmark-
+# SKIP-CMARK-CHECK: cmake --build {{.*}}llvm-
+# SKIP-CMARK-CHECK: cmake --build {{.*}}swift-
+# SKIP-CMARK-CHECK-NOT: --- Installing cmark ---
+# SKIP-CMARK-CHECK: --- Installing llvm ---
+# SKIP-CMARK-CHECK: --- Installing swift ---
+
+# SKIP-LLVM-CHECK: cmake --build {{.*}}cmark-
+# SKIP-LLVM-CHECK-NOT: cmake --build {{.*}}llvm-
+# SKIP-LLVM-CHECK: cmake --build {{.*}}swift-
+# SKIP-LLVM-CHECK: --- Installing cmark ---
+# SKIP-LLVM-CHECK-NOT: --- Installing llvm ---
+# SKIP-LLVM-CHECK: --- Installing swift ---
+
+# SKIP-SWIFT-CHECK: cmake --build {{.*}}cmark-
+# SKIP-SWIFT-CHECK: cmake --build {{.*}}llvm-
+# SKIP-SWIFT-CHECK-NOT: cmake --build {{.*}}swift-
+# SKIP-SWIFT-CHECK: --- Installing cmark ---
+# SKIP-SWIFT-CHECK: --- Installing llvm ---
+# SKIP-SWIFT-CHECK-NOT: --- Installing swift ---
+

validation-test/lit.site.cfg.in

@@ -13,6 +13,7 @@
 import sys
 import platform
 
+config.cmake = "@CMAKE_COMMAND@"
 config.llvm_src_root = "@LLVM_MAIN_SRC_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
 config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"