Skip to content

Revert "[build-script] Use a builder to setup pass pipelines that are then scheduled when using --infer" #38928

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 19, 2021
Merged

Revert "[build-script] Use a builder to setup pass pipelines that are then scheduled when using --infer" #38928

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
264 changes: 146 additions & 118 deletions utils/swift_build_support/swift_build_support/build_script_invocation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,14 @@

import six

from swift_build_support.swift_build_support import build_graph
from swift_build_support.swift_build_support import products
from swift_build_support.swift_build_support import shell
from swift_build_support.swift_build_support import targets
from swift_build_support.swift_build_support import workspace
from swift_build_support.swift_build_support.cmake import CMake
from swift_build_support.swift_build_support.host_specific_configuration \
import HostSpecificConfiguration
from swift_build_support.swift_build_support.productpipeline_list_builder \
import ProductPipelineListBuilder
from swift_build_support.swift_build_support.targets \
import StdlibDeploymentTarget
from swift_build_support.swift_build_support.utils \
Expand Down Expand Up @@ -134,12 +133,12 @@ def convert_to_impl_arguments(self):

# Compute any product specific cmake arguments.
#
# NOTE: The sum(list(...)) is b/c compute_product_pipelines returns a
# NOTE: The sum(list(...)) is b/c compute_product_classes returns a
# tuple of lists of which the first is the build-script-impl products
# and the second is the non-build-script-impl-products. It guarantees
# that when we concatenate these two lists together we get a valid
# dependency graph.
for product_class in sum(list(self.compute_product_pipelines()[0]), []):
for product_class in sum(list(self.compute_product_classes()), []):
if not product_class.is_build_script_impl_product():
continue

Expand Down Expand Up @@ -492,20 +491,15 @@ def compute_host_specific_variables(self):

return options

def compute_product_pipelines(self):
"""compute_product_pipelines() -> [[Product]]
def compute_product_classes(self):
"""compute_product_classes() -> (list, list, list)

A list of lists of products.

Compute lists of product pipelines that we should run. It is guaranteed
that all product pipeline lists consist of solely build-script-impl
products or build-script products. So one can always check the first
element to know if a pipeline returned from the builder is an impl
product or not.
Compute the list first of all pre-build-script-impl products, then all
build-script-impl products and then all non-build-script-impl products.
It is assumed that concatenating the three lists together will result in a
valid dependency graph for the compilation.
"""
builder = ProductPipelineListBuilder(self.args)

builder.begin_pipeline()
before_impl_product_classes = []
# If --skip-early-swift-driver is passed in, swift will be built
# as usual, but relying on its own C++-based (Legacy) driver.
# Otherwise, we build an "early" swift-driver using the host
Expand All @@ -514,79 +508,120 @@ def compute_product_pipelines(self):
# in the host toolchain. If the host toolchain is not equpipped with
# a Swift compiler, a warning is emitted. In the future, it may become
# mandatory that the host toolchain come with its own `swiftc`.
builder.add_product(products.EarlySwiftDriver,
is_enabled=self.args.build_early_swift_driver)
if self.args.build_early_swift_driver:
before_impl_product_classes.append(products.EarlySwiftDriver)

builder.add_product(products.CMark,
is_enabled=self.args.build_cmark)
if self.args.build_cmark:
before_impl_product_classes.append(products.CMark)

# Begin a build-script-impl pipeline for handling the compiler toolchain
# and a subset of the tools that we build. We build these in this manner
# to preserve current build-script-impl run behavior as we transition
# the build-script code base. The main difference is that these are all
# build, tested, and installed all at once instead of performing build,
# test, install like a normal build-script product.
builder.begin_impl_pipeline(should_run_epilogue_operations=True)
# FIXME: This is a weird division (returning a list of class objects),
# but it matches the existing structure of the `build-script-impl`.
impl_product_classes = []

# If --skip-build-llvm is passed in, LLVM cannot be completely disabled, as
# Swift still needs a few LLVM targets like tblgen to be built for it to be
# configured. Instead, handle this in build-script-impl for now.
builder.add_impl_product(products.LLVM,
is_enabled=True)
builder.add_impl_product(products.LibCXX,
is_enabled=self.args.build_libcxx)
builder.add_impl_product(products.LibICU,
is_enabled=self.args.build_libicu)
builder.add_impl_product(products.Swift,
is_enabled=self.args.build_swift)
builder.add_impl_product(products.LLDB,
is_enabled=self.args.build_lldb)
builder.add_impl_product(products.LibDispatch,
is_enabled=self.args.build_libdispatch)
builder.add_impl_product(products.Foundation,
is_enabled=self.args.build_foundation)
builder.add_impl_product(products.XCTest,
is_enabled=self.args.build_xctest)
builder.add_impl_product(products.LLBuild,
is_enabled=self.args.build_llbuild)

# Begin the post build-script-impl build phase.
builder.begin_pipeline()

builder.add_product(products.SwiftPM,
is_enabled=self.args.build_swiftpm)
builder.add_product(products.SwiftSyntax,
is_enabled=self.args.build_swiftsyntax)
builder.add_product(products.SKStressTester,
is_enabled=self.args.build_skstresstester)
builder.add_product(products.SwiftFormat,
is_enabled=self.args.build_swiftformat)
builder.add_product(products.SwiftEvolve,
is_enabled=self.args.build_swiftevolve)
builder.add_product(products.IndexStoreDB,
is_enabled=self.args.build_indexstoredb)
builder.add_product(products.PlaygroundSupport,
is_enabled=self.args.build_playgroundsupport)
builder.add_product(products.SourceKitLSP,
is_enabled=self.args.build_sourcekitlsp)
builder.add_product(products.Benchmarks,
is_enabled=self.args.build_toolchainbenchmarks)
builder.add_product(products.SwiftInspect,
is_enabled=self.args.build_swift_inspect)
builder.add_product(products.TSanLibDispatch,
is_enabled=self.args.tsan_libdispatch_test)
impl_product_classes.append(products.LLVM)
if self.args.build_libcxx:
impl_product_classes.append(products.LibCXX)
if self.args.build_libicu:
impl_product_classes.append(products.LibICU)
if self.args.build_swift:
impl_product_classes.append(products.Swift)
if self.args.build_lldb:
impl_product_classes.append(products.LLDB)
if self.args.build_libdispatch:
impl_product_classes.append(products.LibDispatch)
if self.args.build_foundation:
impl_product_classes.append(products.Foundation)
if self.args.build_xctest:
impl_product_classes.append(products.XCTest)
if self.args.build_llbuild:
impl_product_classes.append(products.LLBuild)
# Sanity check that all of our impl classes are actually
# build_script_impl products.
for prod in impl_product_classes:
assert(prod.is_build_script_impl_product())

product_classes = []
if self.args.build_swiftpm:
product_classes.append(products.SwiftPM)
if self.args.build_swiftsyntax:
product_classes.append(products.SwiftSyntax)
if self.args.build_skstresstester:
product_classes.append(products.SKStressTester)
if self.args.build_swiftformat:
product_classes.append(products.SwiftFormat)
if self.args.build_swiftevolve:
product_classes.append(products.SwiftEvolve)
if self.args.build_indexstoredb:
product_classes.append(products.IndexStoreDB)
if self.args.build_playgroundsupport:
product_classes.append(products.PlaygroundSupport)
if self.args.build_sourcekitlsp:
product_classes.append(products.SourceKitLSP)
if self.args.build_toolchainbenchmarks:
product_classes.append(products.Benchmarks)
if self.args.build_swift_inspect:
product_classes.append(products.SwiftInspect)
if self.args.tsan_libdispatch_test:
product_classes.append(products.TSanLibDispatch)

# Keep SwiftDriver at last.
# swift-driver's integration with the build scripts is not fully
# supported. Using swift-driver to build these products may hit
# failures.
builder.add_product(products.SwiftDriver,
is_enabled=self.args.build_swift_driver
or self.args.install_swift_driver)

# Now that we have constructed our pass pipelines using our builder, get
# the final schedule and finalize the builder.
return builder.finalize(shouldInfer=self.args.infer_dependencies)
if self.args.build_swift_driver or self.args.install_swift_driver:
product_classes.append(products.SwiftDriver)
# Sanity check that all of our non-impl classes are actually
# not build_script_impl products.
for prod in product_classes:
assert(not prod.is_build_script_impl_product())

# Now that we have our two lists of product_classes, if we are asked to
# infer dependencies, infer the dependencies now and then re-split the
# list.
if self.args.infer_dependencies:
combined_classes = before_impl_product_classes +\
impl_product_classes +\
product_classes
if self.args.verbose_build:
print("-- Build Graph Inference --")
print("Initial Product List:")
for p in combined_classes:
print(" {}".format(p.product_name()))

# Now that we have produced the schedule, resplit. We require our
# dependencies to respect our build-script-impl property. This means
# that no build-script-impl products can have dependencies on
# non-build-script-impl products. Otherwise, it would be unsafe to
# re-order build-script-impl products in front of non
# build-script-impl products.
before_impl_product_classes = []
impl_product_classes = []
product_classes = []
is_darwin = platform.system() == 'Darwin'
final_schedule =\
build_graph.produce_scheduled_build(combined_classes)[0]
for p in final_schedule:
if is_darwin and p.is_nondarwin_only_build_product():
continue
if p.is_build_script_impl_product():
impl_product_classes.append(p)
elif p.is_before_build_script_impl_product():
before_impl_product_classes.append(p)
else:
product_classes.append(p)

if self.args.verbose_build:
print("Final Build Order:")
for p in before_impl_product_classes:
print(" {}".format(p.product_name()))
for p in impl_product_classes:
print(" {}".format(p.product_name()))
for p in product_classes:
print(" {}".format(p.product_name()))
return (before_impl_product_classes, impl_product_classes, product_classes)

def execute(self):
"""Execute the invocation with the configured arguments."""
Expand All @@ -609,40 +644,26 @@ def execute(self):
all_hosts = [StdlibDeploymentTarget.get_target_for_name(name)
for name in all_host_names]

# Compute the list of lists of product classes to operate on.
# Compute the list of product classes to operate on.
#
# FIXME: This should really be per-host, but the current structure
# matches that of `build-script-impl`.
(product_pipelines, last_impl_index) = self.compute_product_pipelines()

# Execute each "product pipeline".
for index in range(len(product_pipelines)):
pipeline = product_pipelines[index]
# Skip empty pipelines.
if len(pipeline) == 0:
continue
is_impl = pipeline[0].is_build_script_impl_product()
if is_impl:
perform_epilogue_opts = last_impl_index == index
self._execute_impl(pipeline, all_hosts, perform_epilogue_opts)
else:
assert(index != last_impl_index)
self._execute(pipeline, all_host_names)

# And then perform the rest of the non-core epilogue actions.

# Extract symbols...
for host_target in all_hosts:
self._execute_extract_symbols_action(host_target)
(before_impl_product_classes, impl_product_classes, product_classes) =\
self.compute_product_classes()

# Package...
for host_target in all_hosts:
self._execute_package_action(host_target)
# Execute each "pass".

# Lipo...
self._execute_merged_host_lipo_action()
# Pre-build-script-impl products...
# Note: currently only supports building for the host.
for host_target in all_host_names:
for product_class in before_impl_product_classes:
if product_class.is_build_script_impl_product():
continue
if not product_class.is_before_build_script_impl_product():
continue
# Execute clean, build, test, install
self.execute_product_build_steps(product_class, host_target)

def _execute_impl(self, pipeline, all_hosts, should_run_epilogue_operations):
# Build...
for host_target in all_hosts:
# FIXME: We should only compute these once.
Expand All @@ -660,35 +681,42 @@ def _execute_impl(self, pipeline, all_hosts, should_run_epilogue_operations):
print("Running Swift benchmarks for: {}".format(
" ".join(config.swift_benchmark_run_targets)))

for product_class in pipeline:
for product_class in impl_product_classes:
self._execute_build_action(host_target, product_class)

# Test...
for host_target in all_hosts:
for product_class in pipeline:
for product_class in impl_product_classes:
self._execute_test_action(host_target, product_class)

# Install...
for host_target in all_hosts:
for product_class in pipeline:
for product_class in impl_product_classes:
self._execute_install_action(host_target, product_class)

# And then we may be asked to perform several post-processing operations
# on what we have built. If we are not supposed to do so, bail now.
if not should_run_epilogue_operations:
return

# Core Lipo...
self._execute_merged_host_lipo_core_action()

def _execute(self, pipeline, all_host_names):
# Pre-build-script-impl products...
# Non-build-script-impl products...
# Note: currently only supports building for the host.
for host_target in all_host_names:
for product_class in pipeline:
for host_target in [self.args.host_target]:
for product_class in product_classes:
if product_class.is_build_script_impl_product():
continue
# Execute clean, build, test, install
self.execute_product_build_steps(product_class, host_target)

# Extract symbols...
for host_target in all_hosts:
self._execute_extract_symbols_action(host_target)

# Package...
for host_target in all_hosts:
self._execute_package_action(host_target)

# Lipo...
self._execute_merged_host_lipo_action()

def _execute_build_action(self, host_target, product_class):
action_name = "{}-{}-build".format(host_target.name,
product_class.product_name())
Expand Down
Loading