Disable benchmark builds by default in build-script

utils/build-presets.ini

@@ -24,6 +24,7 @@ swift-install-components=compiler;clang-builtin-headers;stdlib;sdk-overlay;edito
 ios
 tvos
 watchos
+build-benchmarks
 test
 validation-test
 lit-args=-v
@@ -285,6 +286,7 @@ dash-dash
 #===------------------------------------------------------------------------===#
 
 [preset: buildbot_incremental_base]
+
 test
 validation-test
 lit-args=-v
@@ -303,9 +305,16 @@ build-ninja
 build-swift-stdlib-unittest-extra
 
 
-[preset: buildbot_incremental_base_all_platforms]
+[preset: buildbot_incremental_base_benchmarks]
+
 mixin-preset=buildbot_incremental_base
 
+build-benchmarks
+
+
+[preset: buildbot_incremental_base_all_platforms]
+mixin-preset=buildbot_incremental_base_benchmarks
+
 # Build standard library and SDK overlay for iOS device and simulator.
 
 ios
@@ -366,7 +375,7 @@ skip-test-playgroundsupport
 [preset: buildbot_incremental,tools=RA,stdlib=RA,XCTest]
 # We don't use buildbot_incremental_base_all_platforms because we don't need to
 # build for iOS/tvOS.
-mixin-preset=buildbot_incremental_base
+mixin-preset=buildbot_incremental_base_benchmarks
 
 # Build and test XCTest. On Linux, the build script is aware of the dependency
 # on Foundation, so that is built as well. On OS X, Foundation is built as part
@@ -397,7 +406,6 @@ dash-dash
 skip-test-cmark
 skip-test-swift
 skip-test-foundation
-skip-build-benchmarks
 
 [preset: buildbot_incremental_asan,tools=RDA,stdlib=RDA]
 mixin-preset=buildbot_incremental_base_all_platforms
@@ -598,6 +606,7 @@ skip-build-cmark
 [preset: buildbot_incremental_extra_swift_args,tools=RA,stdlib=RD]
 build-subdir=buildbot_incremental_extra_swift_args
 # Build standard library and SDK overlay for iOS device and simulator.
+build-benchmarks
 test
 validation-test
 extra-swift-args=%(extra_swift_args)s
@@ -639,6 +648,7 @@ mixin-preset=buildbot_incremental_asan,tools=RDA,stdlib=RDA
 #===------------------------------------------------------------------------===#
 
 [preset: buildbot_incremental_leaks]
+build-benchmarks
 compiler-vendor=apple
 dash-dash
 
@@ -704,6 +714,7 @@ swift-enable-ast-verifier=0
 [preset: mixin_linux_installation]
 mixin-preset=mixin_lightweight_assertions
 
+build-benchmarks
 llbuild
 swiftpm
 xctest
@@ -812,6 +823,7 @@ skip-test-lldb
 mixin-preset=buildbot_incremental_linux
 build-subdir=buildbot_linux
 release
+build-benchmarks
 test
 validation-test
 long-test
@@ -820,6 +832,7 @@ lit-args=-v
 
 [preset: buildbot_linux_armv7]
 release
+build-benchmarks
 llbuild
 lldb
 # Not all tests pass, currently
@@ -859,6 +872,7 @@ installable-package=%(installable_package)s
 [preset: buildbot_incremental_linux_base]
 assertions
 release
+build-benchmarks
 test
 validation-test
 lit-args=-v
@@ -915,6 +929,7 @@ build-subdir=buildbot_incremental_lsan
 release-debuginfo
 assertions
 enable-lsan
+build-benchmarks
 
 dash-dash
 
@@ -951,6 +966,7 @@ playgroundsupport
 # Build with debug info, this allows us to symbolicate crashes from
 # production builds.
 release-debuginfo
+build-benchmarks
 compiler-vendor=apple
 
 dash-dash
@@ -1077,7 +1093,6 @@ skip-test-playgroundsupport
 
 [preset: LLDB_Nested]
 dash-dash
-skip-build-benchmarks
 install-destdir=%(swift_install_destdir)s
 llvm-targets-to-build=X86;ARM;AArch64;PowerPC;SystemZ;Mips
 
@@ -1251,7 +1266,6 @@ skip-build-cmark
 skip-build-llvm
 skip-build-llbuild
 skip-build-swiftpm
-skip-build-benchmarks
 install-swift
 install-prefix=%(install_toolchain_dir)s/usr
 build-swift-examples=0
@@ -1358,7 +1372,6 @@ reconfigure
 verbose-build
 build-ninja
 build-swift-stdlib-unittest-extra
-skip-build-benchmarks
 
 [preset: mixin_buildbot_incremental,test=macOS,type=device]
 
@@ -1632,7 +1645,6 @@ validation-test
 skip-test-ios
 skip-test-tvos
 skip-test-watchos
-skip-build-benchmark
 verbose-build
 lit-args=-v
 reconfigure
@@ -1660,7 +1672,6 @@ build-swift-dynamic-stdlib=0
 build-swift-static-sdk-overlay=0
 build-swift-dynamic-sdk-overlay=0
 build-runtime-with-host-compiler=0
-skip-build-benchmarks
 llvm-include-tests=0
 llvm-enable-modules=0
 build-swift-examples=0

utils/build-script

@@ -570,11 +570,6 @@ class BuildScriptInvocation(object):
             impl_args += ["--skip-build-cmark",
                           "--skip-build-llvm",
                           "--skip-build-swift"]
-        if not args.build_benchmarks:
-            impl_args += ["--skip-build-benchmarks"]
-        # Currently we do not build external benchmarks by default.
-        if args.build_external_benchmarks:
-            impl_args += ["--skip-build-external-benchmarks=0"]
         if not args.build_foundation:
             impl_args += ["--skip-build-foundation"]
         if not args.build_xctest:
@@ -670,8 +665,6 @@ class BuildScriptInvocation(object):
             impl_args += ["--validation-test"]
         if args.long_test:
             impl_args += ["--long-test"]
-        if not args.benchmark:
-            impl_args += ["--skip-test-benchmarks"]
         if not args.test_optimized:
             impl_args += ["--skip-test-optimized"]
         if not args.test_optimize_for_size:
@@ -759,6 +752,16 @@ class BuildScriptInvocation(object):
                 os.path.abspath(args.coverage_db)
             ]
 
+        # Benchmark Suite
+        if args.build_benchmarks:
+            impl_args += ['--build-benchmarks']
+        if args.test_benchmarks:
+            impl_args += ['--test-benchmarks']
+
+        # NOTE: Currently we do not build external benchmarks by default
+        if args.build_external_benchmarks:
+            impl_args += ['--skip-build-external-benchmarks=0']
+
         # Compute the set of host-specific variables, which we pass through to
         # the build script via environment variables.
         host_specific_variables = self.compute_host_specific_variables()
@@ -889,7 +892,9 @@ class BuildScriptInvocation(object):
                     self.args.test or self.args.long_test):
                 print("Running Swift tests for: {}".format(
                     " ".join(config.swift_test_run_targets)))
-            if config.swift_benchmark_run_targets and self.args.benchmark:
+            if config.swift_benchmark_run_targets and \
+                    self.args.build_benchmarks and \
+                    self.args.test_benchmarks:
                 print("Running Swift benchmarks for: {}".format(
                     " ".join(config.swift_benchmark_run_targets)))
 

utils/build-script-impl

@@ -124,8 +124,6 @@ KNOWN_SETTINGS=(
     skip-build-foundation       ""               "set to skip building foundation"
     skip-build-libdispatch      ""               "set to skip building libdispatch"
     skip-build-libicu           ""               "set to skip building libicu"
-    skip-build-benchmarks       ""               "set to skip building Swift Benchmark Suite"
-    skip-build-external-benchmarks "1"            "set to skip building the external Swift Benchmark Suite. (skipped by default)"
     skip-build-playgroundlogger ""               "set to skip building PlaygroundLogger"
     skip-build-playgroundsupport ""              "set to skip building PlaygroundSupport"
     skip-test-cmark             ""               "set to skip testing CommonMark"
@@ -155,11 +153,13 @@ KNOWN_SETTINGS=(
     validation-test             "0"              "set to run the validation test suite"
     long-test                   "0"              "set to run the long test suite"
     test-paths                  ""               "run tests located in specific directories and/or files"
-    skip-test-benchmarks        ""               "set to skip running Swift Benchmark Suite"
     skip-test-optimized         ""               "set to skip testing the test suite in optimized mode"
     skip-test-optimize-for-size ""               "set to skip testing the test suite in optimize for size mode"
     skip-test-sourcekit         ""               "set to skip testing SourceKit"
     stress-test-sourcekit       ""               "set to run the stress-SourceKit target"
+    build-benchmarks            "0"               "set to build the Swift benchmark suite"
+    test-benchmarks             "0"               "set to test the Swift benchmark suite"
+    skip-build-external-benchmarks "1"            "set to skip building the external Swift benchmark suite. (skipped by default)"
     workspace                   "${HOME}/src"    "source directory containing llvm, clang, swift"
     enable-llvm-assertions      "1"              "set to enable llvm assertions"
     build-llvm                  "1"              "set to 1 to build LLVM and Clang"
@@ -1448,7 +1448,7 @@ function calculate_targets_for_host() {
         fi
         if [[ "${build_benchmark_this_target}" ]] && [[ "${is_in_build_list}" ]]; then
             SWIFT_BENCHMARK_TARGETS+=("swift-benchmark-${stdlib_deployment_target}")
-            if [[ $(not ${SKIP_TEST_BENCHMARK}) ]] ; then
+            if [[ ${TEST_BENCHMARKS} ]] ; then
               SWIFT_RUN_BENCHMARK_TARGETS+=("check-swift-benchmark-${stdlib_deployment_target}")
             fi
         fi
@@ -1457,7 +1457,7 @@ function calculate_targets_for_host() {
            [[ "${build_external_benchmark_this_target}" ]] &&
            [[ "${is_in_build_list}" ]] ; then
             SWIFT_BENCHMARK_TARGETS+=("swift-benchmark-${stdlib_deployment_target}-external")
-            if [[ $(not ${SKIP_TEST_BENCHMARK}) ]] ; then
+            if [[ ${TEST_BENCHMARKS} ]] ; then
                 SWIFT_RUN_BENCHMARK_TARGETS+=("check-swift-benchmark-${stdlib_deployment_target}-external")
             fi
         fi
@@ -1857,9 +1857,8 @@ for host in "${ALL_HOSTS[@]}"; do
         if [[ "${SWIFT_TEST_TARGETS[@]}" ]] && ! [[ "${SKIP_TEST_SWIFT}" ]]; then
             echo "Running Swift tests for: ${SWIFT_TEST_TARGETS[@]}"
         fi
-        if ! [[ "${SKIP_TEST_BENCHMARKS}" ]] &&
-                [[ "${SWIFT_RUN_BENCHMARK_TARGETS[@]}" ]] &&
-                ! [[ "${SKIP_TEST_BENCHMARK}" ]]; then
+        if [[ "${TEST_BENCHMARKS}" ]] &&
+                [[ "${SWIFT_RUN_BENCHMARK_TARGETS[@]}" ]]; then
             echo "Running Swift benchmarks for: ${SWIFT_RUN_BENCHMARK_TARGETS[@]}"
         fi
     fi
@@ -2125,7 +2124,7 @@ for host in "${ALL_HOSTS[@]}"; do
                     native_swift_tools_path="$(build_directory "${LOCAL_HOST}" swift)/bin"
                 else
                     # FIXME: Why is the next line not using false_true?
-                    build_perf_testsuite_this_time=$(true_false "$(not ${SKIP_BUILD_BENCHMARKS})")
+                    build_perf_testsuite_this_time=$(true_false "${BUILD_BENCHMARKS}")
                     build_external_perf_testsuite_this_time=$(false_true "${SKIP_BUILD_EXTERNAL_BENCHMARKS}")
                     build_tests_this_time=${SWIFT_INCLUDE_TESTS}
                 fi
@@ -2767,7 +2766,7 @@ for host in "${ALL_HOSTS[@]}"; do
                         )
                     fi
                 fi
-                if ! [[ "${SKIP_TEST_BENCHMARKS}" ]]; then
+                if [[ "${TEST_BENCHMARKS}" ]]; then
                     results_targets=(
                         "${results_targets[@]}"
                         "${SWIFT_RUN_BENCHMARK_TARGETS[@]}"

utils/build_swift/driver_arguments.py

@@ -691,17 +691,6 @@ def create_argument_parser():
            help='run tests located in specific directories and/or files '
                 '(implies --test and/or --validation-test)')
 
-    option(['-B', '--benchmark'], store_true,
-           help='run the Swift Benchmark Suite after building')
-    option('--benchmark-num-o-iterations', store_int,
-           default=3,
-           help='if the Swift Benchmark Suite is run after building, run N '
-                'iterations with -O')
-    option('--benchmark-num-onone-iterations', store_int,
-           default=3,
-           help='if the Swift Benchmark Suite is run after building, run N '
-                'iterations with -Onone')
-
     option('--skip-test-osx', toggle_false('test_osx'),
            help='skip testing Swift stdlibs for Mac OS X')
     option('--skip-test-linux', toggle_false('test_linux'),
@@ -774,12 +763,6 @@ def create_argument_parser():
     option('--skip-build-android', toggle_false('build_android'),
            help='skip building Swift stdlibs for Android')
 
-    option('--skip-build-benchmarks', toggle_false('build_benchmarks'),
-           help='skip building Swift Benchmark Suite')
-
-    option('--build-external-benchmarks', toggle_true,
-           help='skip building Swift Benchmark Suite')
-
     # -------------------------------------------------------------------------
     in_group('Skip testing specified targets')
 
@@ -870,6 +853,36 @@ def create_argument_parser():
                 'the "{}" directory.'.format(
                     android.adb.commands.DEVICE_TEMP_DIR))
 
+    # -------------------------------------------------------------------------
+    in_group('Benchmark Suite')
+
+    option(['-B', '--benchmark'],
+           store_true(['build_benchmarks', 'test_benchmarks']),
+           help='build and test the Swift benchmark suite')
+
+    set_defaults(build_benchmarks=False)
+    with mutually_exclusive_group():
+        option('--build-benchmarks', toggle_true,
+               help='build the Swift benchmark suite')
+
+        # NOTE: Deprecated with the addition of --build-benchmarks
+        option('--skip-build-benchmarks', toggle_false('build_benchmarks'),
+               help='skip building the Swift benchmark suite (Deprecated, '
+                    'please use --build-benchmarks instead)')
+
+    option('--test-benchmarks', toggle_true,
+           help='test the Swift benchmark suite')
+
+    option('--build-external-benchmarks', toggle_true,
+           help='build the external Swift benchmark suite')
+
+    option('--benchmark-num-o-iterations', store_int,
+           default=3,
+           help='number of iterations to run with -O')
+    option('--benchmark-num-onone-iterations', store_int,
+           default=3,
+           help='number of iterations to run with -Onone')
+
     # -------------------------------------------------------------------------
     in_group('Unsupported options')