Skip to content

[Linux] Add compression libraries to the build. #66340

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 87 additions & 2 deletions utils/build-script-impl
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,9 @@ components=(
libdispatch
libicu
libxml2
liblzma
zlib
zstd
curl
llbuild
lldb
Expand Down Expand Up @@ -1205,12 +1207,18 @@ LIBDISPATCH_SOURCE_DIR="${WORKSPACE}/swift-corelibs-libdispatch"
LIBDISPATCH_STATIC_SOURCE_DIR="${WORKSPACE}/swift-corelibs-libdispatch"
LIBICU_SOURCE_DIR="${WORKSPACE}/icu"
LIBCXX_SOURCE_DIR="${WORKSPACE}/llvm-project/runtimes"
LIBLZMA_SOURCE_DIR="${WORKSPACE}/liblzma"
ZLIB_SOURCE_DIR="${WORKSPACE}/zlib"
ZSTD_SOURCE_DIR="${WORKSPACE}/zstd/build/cmake"
SWIFT_PATH_TO_STRING_PROCESSING_SOURCE="${WORKSPACE}/swift-experimental-string-processing"
SWIFTSYNTAX_SOURCE_DIR="${WORKSPACE}/swift-syntax"
SWIFT_SYNTAX_SOURCE_DIR="${WORKSPACE}/swift-syntax"

[[ "${SKIP_BUILD_LIBCXX}" ]] || PRODUCTS+=(libcxx)
[[ "${SKIP_BUILD_LIBICU}" ]] || PRODUCTS+=(libicu)
[[ "${SKIP_BUILD_LIBLZMA}" ]] || PRODUCTS+=(liblzma)
[[ "${SKIP_BUILD_ZLIB}" ]] || PRODUCTS+=(zlib)
[[ "${SKIP_BUILD_ZSTD}" ]] || PRODUCTS+=(zstd)
[[ "${SKIP_BUILD_SWIFT}" ]] || PRODUCTS+=(swift)
[[ "${SKIP_BUILD_LLDB}" ]] || PRODUCTS+=(lldb)
[[ "${SKIP_BUILD_LIBDISPATCH}" ]] || PRODUCTS+=(libdispatch)
Expand Down Expand Up @@ -1300,6 +1308,15 @@ function build_directory_bin() {
;;
libicu)
;;
liblzma)
echo "${root}/${LIBLZMA_BUILD_TYPE}/bin"
;;
zlib)
echo "${root}/${ZLIB_BUILD_TYPE}/bin"
;;
zstd)
echo "${root}/${ZSTD_BUILD_TYPE}/bin"
;;
*)
echo "error: unknown product: ${product}"
exit 1
Expand Down Expand Up @@ -1416,6 +1433,15 @@ function cmake_config_opt() {
;;
libicu)
;;
liblzma)
echo "--config ${LIBLZMA_BUILD_TYPE}"
;;
zlib)
echo "--config ${ZLIB_BUILD_TYPE}"
;;
zstd)
echo "--config ${ZSTD_BUILD_TYPE}"
;;
*)
echo "error: unknown product: ${product}"
exit 1
Expand Down Expand Up @@ -1518,7 +1544,6 @@ function copy_embedded_compiler_rt_builtins_from_darwin_host_toolchain() {
#
# Start with native deployment targets because the resulting tools are used during cross-compilation.


for host in "${ALL_HOSTS[@]}"; do
# Skip this pass when the only action to execute can't match.
if ! [[ $(should_execute_host_actions_for_phase ${host} build) ]]; then
Expand All @@ -1532,14 +1557,20 @@ for host in "${ALL_HOSTS[@]}"; do
# Don't echo anything if only executing an individual action.
if [[ "${ONLY_EXECUTE}" = "all" ]]; then
echo "Building the standard library for: ${SWIFT_STDLIB_TARGETS[@]}"

if [[ "${SWIFT_TEST_TARGETS[@]}" ]] && ! [[ "${SKIP_TEST_SWIFT}" ]]; then
echo "1"
echo "Running Swift tests for: ${SWIFT_TEST_TARGETS[@]}"
fi

if ! [[ "${SKIP_TEST_BENCHMARKS}" ]] &&
[[ "${SWIFT_RUN_BENCHMARK_TARGETS[@]}" ]]; then
[[ "${SWIFT_RUN_BENCHMARK_TARGETS[@]}" ]]; then
echo "3"
echo "Running Swift benchmarks for: ${SWIFT_RUN_BENCHMARK_TARGETS[@]}"
fi

if [ $(true_false "${BUILD_SWIFT_LIBEXEC}") == "TRUE" ]; then
echo "5"
echo "Building the auxiliary executables for: ${SWIFT_LIBEXEC_TARGETS[@]}"
fi
fi
Expand Down Expand Up @@ -2625,6 +2656,48 @@ for host in "${ALL_HOSTS[@]}"; do
# libicu builds itself and doesn't use cmake
continue
;;
liblzma)
if [[ "${host}" != "linux-"* ]]; then
break
fi

LIBLZMA_BUILD_DIR=$(build_directory ${host} ${product})

cmake_options=(
-DCMAKE_BUILD_TYPE:STRING="${LIBLZMA_BUILD_TYPE}"
-DCMAKE_C_COMPILER:PATH="${CLANG_BIN}/clang"
-DCMAKE_CXX_COMPILER:PATHJ="${CLANG_BIN}/clang++"
-DCMAKE_INSTALL_PREFIX:PATH="$(get_host_install_prefix ${host})"
)
;;
zlib)
if [[ "${host}" != "linux-"* ]]; then
break
fi

ZLIB_BUILD_DIR=$(build_directory ${host} ${product})

cmake_options=(
-DCMAKE_BUILD_TYPE:STRING="${ZLIB_BUILD_TYPE}"
-DCMAKE_C_COMPILER:PATH="${CLANG_BIN}/clang"
-DCMAKE_CXX_COMPILER:PATHJ="${CLANG_BIN}/clang++"
-DCMAKE_INSTALL_PREFIX:PATH="$(get_host_install_prefix ${host})"
)
;;
zstd)
if [[ "${host}" != "linux-"* ]]; then
break
fi

ZSTD_BUILD_DIR=$(build_directory ${host} ${product})

cmake_options=(
-DCMAKE_BUILD_TYPE:STRING="${ZSTD_BUILD_TYPE}"
-DCMAKE_C_COMPILER:PATH="${CLANG_BIN}/clang"
-DCMAKE_CXX_COMPILER:PATHJ="${CLANG_BIN}/clang++"
-DCMAKE_INSTALL_PREFIX:PATH="$(get_host_install_prefix ${host})"
)
;;
*)
echo "error: unknown product: ${product}"
exit 1
Expand Down Expand Up @@ -2952,6 +3025,12 @@ for host in "${ALL_HOSTS[@]}"; do
echo "--- Finished tests for ${product} ---"
continue
;;
liblzma)
;;
zlib)
;;
zstd)
;;
*)
echo "error: unknown product: ${product}"
exit 1
Expand Down Expand Up @@ -3149,6 +3228,12 @@ for host in "${ALL_HOSTS[@]}"; do
call cp -a "${ICU_TMP_INSTALL_DIR}/share/icuswift" "${ICU_INSTALL_DIR}share"
continue
;;
liblzma)
;;
zlib)
;;
zstd)
;;
*)
echo "error: unknown product: ${product}"
exit 1
Expand Down
23 changes: 22 additions & 1 deletion utils/build_swift/build_swift/driver_arguments.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,12 @@ def _apply_default_arguments(args):
if args.zlib_build_variant is None:
args.zlib_build_variant = args.build_variant

if args.zstd_build_variant is None:
args.zstd_build_variant = args.build_variant

if args.liblzma_build_variant is None:
args.liblzma_build_variant = args.build_variant

if args.curl_build_variant is None:
args.curl_build_variant = args.build_variant

Expand Down Expand Up @@ -771,9 +777,12 @@ def create_argument_parser():

option('--static-zlib', toggle_true('build_zlib'), default=False,
help='build static zlib')

option('--static-curl', toggle_true('build_curl'), default=False,
help='build static curl libraries')
option('--static-liblzma', toggle_true('build_liblzma'), default=False,
help='build static liblzma')
option('--static-zstd', toggle_true('build_zstd'), default=False,
help='build static zstd')

option('--playgroundsupport', toggle_true('build_playgroundsupport'),
help='build PlaygroundSupport')
Expand Down Expand Up @@ -903,6 +912,14 @@ def create_argument_parser():
const='Debug',
help='build the Debug variant of libcurl')

option('--debug-zstd', store('zstd_build_variant'),
const='Debug',
help='build the Debug variant of zstd')

option('--debug-liblzma', store('liblzma_build_variant'),
const='Debug',
help='build the Debug variant of liblzma')

# -------------------------------------------------------------------------
# Assertions group

Expand Down Expand Up @@ -1374,6 +1391,10 @@ def create_argument_parser():
help='skip building zlib')
option('--skip-build-curl', toggle_false('build_curl'),
help='skip building curl')
option('--skip-build-liblzma', toggle_false('build_lzma'),
help='skip building liblzma')
option('--skip-build-zstd', toggle_false('build_zstd'),
help='skip building zstd')

# We need to list --skip-test-swift explicitly because otherwise argparse
# will auto-expand arguments like --skip-test-swift to the only known
Expand Down
6 changes: 6 additions & 0 deletions utils/build_swift/tests/expected_options.py
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,8 @@ class BuildScriptImplOption(_BaseOption):
SetOption('--debug-libicu', dest='libicu_build_variant', value='Debug'),
SetOption('--debug-libxml2', dest='libxml2_build_variant', value='Debug'),
SetOption('--debug-zlib', dest='zlib_build_variant', value='Debug'),
SetOption('--debug-zstd', dest='zstd_build_variant', value='Debug'),
SetOption('--debug-liblzma', dest='liblzma_build_variant', value='Debug'),
SetOption('--debug-curl', dest='curl_build_variant', value='Debug'),
SetOption('--debug-lldb', dest='lldb_build_variant', value='Debug'),
SetOption('--lldb-build-with-xcode', dest='lldb_build_with_xcode',
Expand Down Expand Up @@ -601,6 +603,8 @@ class BuildScriptImplOption(_BaseOption):
EnableOption('--libicu', dest='build_libicu'),
EnableOption('--static-libxml2', dest='build_libxml2'),
EnableOption('--static-zlib', dest='build_zlib'),
EnableOption('--static-zstd', dest='build_zstd'),
EnableOption('--static-liblzma', dest='build_liblzma'),
EnableOption('--static-curl', dest='build_curl'),
EnableOption('--indexstore-db', dest='build_indexstoredb'),
EnableOption('--test-indexstore-db-sanitize-all',
Expand Down Expand Up @@ -717,6 +721,8 @@ class BuildScriptImplOption(_BaseOption):
DisableOption('--skip-build-libxml2', dest='build_libxml2'),
DisableOption('--skip-build-zlib', dest='build_zlib'),
DisableOption('--skip-build-curl', dest='build_curl'),
DisableOption('--skip-build-zstd', dest='build_zstd'),
DisableOption('--skip-build-liblzma', dest='build_liblzma'),
DisableOption('--skip-build-compiler-rt', dest='build_compiler_rt'),

ChoicesOption('--compiler-vendor',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,17 @@ def convert_to_impl_arguments(self):
if args.build_early_swiftsyntax:
impl_args += ["--swift-earlyswiftsyntax"]

# Add the paths of the compression libraries
zlib_src = os.path.join(self.workspace.source_root, "zlib")
zstd_src = os.path.join(self.workspace.source_root, "zstd")
liblzma_src = os.path.join(self.workspace.source_root, "liblzma")

args.extra_cmake_options += [
'-DSWIFT_PATH_TO_ZLIB_SOURCE:PATH={}'.format(zlib_src),
'-DSWIFT_PATH_TO_ZSTD_SOURCE:PATH={}'.format(zstd_src),
'-DSWIFT_PATH_TO_LIBLZMA_SOURCE:PATH={}'.format(liblzma_src),
]

# Then add subproject install flags that either skip building them /or/
# if we are going to build them and install_all is set, we also install
# them.
Expand All @@ -274,7 +285,9 @@ def convert_to_impl_arguments(self):
(args.build_libicu, "libicu"),
(args.build_libxml2, 'libxml2'),
(args.build_zlib, 'zlib'),
(args.build_curl, 'curl')
(args.build_curl, 'curl'),
(args.build_zstd, 'zstd'),
(args.build_liblzma, 'liblzma'),
]
for (should_build, string_name) in conditional_subproject_configs:
if not should_build and not self.args.infer_dependencies:
Expand Down Expand Up @@ -600,6 +613,12 @@ def compute_product_pipelines(self):
builder.add_product(products.curl.LibCurl,
is_enabled=self.args.build_curl)

builder.add_product(products.Zstd,
is_enabled=self.args.build_zstd)

builder.add_product(products.Liblzma,
is_enabled=self.args.build_liblzma)

# Begin a build-script-impl pipeline for handling the compiler toolchain
# and a subset of the tools that we build. We build these in this manner
# to preserve current build-script-impl run behavior as we transition
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@
from .tsan_libdispatch import TSanLibDispatch
from .xctest import XCTest
from .zlib import Zlib
from .zstd import Zstd
from .liblzma import Liblzma

__all__ = [
'BackDeployConcurrency',
Expand Down Expand Up @@ -75,5 +77,7 @@
'Benchmarks',
'TSanLibDispatch',
'SwiftDocC',
'SwiftDocCRender'
'SwiftDocCRender',
'Zstd',
'Liblzma',
]
Loading