Skip to content

Commit 7261cd6

Browse files
committed
Reduce toolchain size by preserving TensorFlow library symlinks.
Bazel produces multiple TensorFlow library artifacts: ``` $ ls -alh tensorflow/bazel-bin/tensorflow 18B libtensorflow.so -> libtensorflow.so.1 23B libtensorflow.so.1 -> libtensorflow.so.1.14.0 277M libtensorflow.so.1.14.0 ``` Previously, TensorFlow libraries were copied via `cp -p`. This did not preserve symlinks, leading to library duplication: ``` $ ls -alh <TOOLCHAIN_BEFORE>.xctoolchain/usr/lib/swift/macosx 18B libtensorflow.so -> libtensorflow.so.1 277M libtensorflow.so.1 277M libtensorflow.so.1.14.0 # duplicate library ``` Now, `cp -a` is used, which preserves symlinks: ``` $ ls -alh <TOOLCHAIN_AFTER>.xctoolchain/usr/lib/swift/macosx 18B libtensorflow.so -> libtensorflow.so.1 277M libtensorflow.so.1 ``` This combined with removing libtensorflow_framework.so dependency (swiftlang#27029) led to a macOS toolchain size reduction from 3.99 GB to 3.62 GB on the tensorflow-0.5 branch.
1 parent cf557ab commit 7261cd6

File tree

2 files changed

+19
-25
lines changed

2 files changed

+19
-25
lines changed

stdlib/public/CTensorFlow/CMakeLists.txt

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -98,17 +98,6 @@ foreach(sdk ${TARGET_SDKS})
9898
COMPONENT stdlib
9999
)
100100
endforeach()
101-
102-
# Install TensorFlow libraries in each SDK subdirectory.
103-
# FIXME: Currently, this installs the same version of TensorFlow libraries
104-
# everywhere.
105-
# To properly support multiple targets, TF library/include variables should be
106-
# SDK/architecture-specific, just like the ones for ICU.
107-
swift_install_in_component(
108-
FILES "${TF_LIBRARIES}"
109-
DESTINATION "lib/swift/${SWIFT_SDK_${sdk}_LIB_SUBDIR}"
110-
COMPONENT stdlib
111-
)
112101
endforeach()
113102
add_custom_target(ctensorflow_modulemap DEPENDS ${ctensorflow_modulemap_target_list})
114103

utils/build-script-impl

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2583,15 +2583,18 @@ for host in "${ALL_HOSTS[@]}"; do
25832583
TF_LIB_DIR="${SWIFT_LIB_PATH}/swift/${SWIFT_HOST_VARIANT}"
25842584
call mkdir -p "${TF_LIB_DIR}"
25852585

2586-
# Bazel-built libraries do not have write permission, which is
2587-
# problematic for overwriting/stripping symbols. Thus, write
2588-
# permission is added here.
25892586
for lib_name in tensorflow; do
2590-
lib=".*lib${lib_name}.so[0-9.]*"
2591-
dylib=".*lib${lib_name}[0-9.]*.dylib"
2592-
find "${TF_LIB_DIR}" \( -regex "${lib}" -o -regex "${dylib}" \) -exec rm -f {} \;
2593-
find "${TENSORFLOW_HOST_LIB_DIR}" \( -regex "${lib}" -o -regex "${dylib}" \) -exec chmod +w {} +
2594-
find "${TENSORFLOW_HOST_LIB_DIR}" \( -regex "${lib}" -o -regex "${dylib}" \) -exec cp -p {} "${TF_LIB_DIR}" \;
2587+
lib_full_name="lib${lib_name}.so"
2588+
# Wipe existing TensorFlow libraries in the destination
2589+
# directory. This ensures that old versions of libraries
2590+
# are not copied, taking up unnecessary space.
2591+
call rm -rf "${TF_LIB_DIR}/${lib_full_name}"*
2592+
# Copy TensorFlow libraries, preserving symlinks.
2593+
call find "${TENSORFLOW_HOST_LIB_DIR}" \( -regex ".*${lib_full_name}[0-9.]*" \) -exec cp -a -v {} "${TF_LIB_DIR}" \;
2594+
# Bazel-built libraries do not have write permission, which is
2595+
# problematic for overwriting/stripping symbols. Thus, write
2596+
# permission is added here.
2597+
call chmod +w "${TF_LIB_DIR}/${lib_full_name}"*
25952598
done
25962599

25972600
cmake_options=(
@@ -4008,15 +4011,17 @@ for host in "${ALL_HOSTS[@]}"; do
40084011
exit 1
40094012
fi
40104013
echo "--- Installing ${product} ---"
4011-
TF_BUILD_DIR="$(build_directory ${host} ${product})"
4012-
TF_LIBDIR="$(build_directory ${host} swift)/lib/swift/${SWIFT_HOST_VARIANT}"
4014+
TF_LIB_DIR="$(build_directory ${host} swift)/lib/swift/${SWIFT_HOST_VARIANT}"
40134015
TF_DEST_DIR="$(get_host_install_destdir ${host})$(get_host_install_prefix ${host})lib/swift/${SWIFT_HOST_VARIANT}"
40144016
mkdir -p "${TF_DEST_DIR}"
40154017
for lib_name in tensorflow; do
4016-
lib=".*lib${lib_name}.so[0-9.]*"
4017-
dylib=".*lib${lib_name}[0-9.]*.dylib"
4018-
find "${TF_LIBDIR}" \( -regex "${lib}" -o -regex "${dylib}" \) -exec echo "{} => ${TF_DEST_DIR}" \;
4019-
find "${TF_LIBDIR}" \( -regex "${lib}" -o -regex "${dylib}" \) -exec cp -a {} "${TF_DEST_DIR}" \;
4018+
lib_full_name="lib${lib_name}.so"
4019+
# Wipe existing TensorFlow libraries in the destination
4020+
# directory. This ensures that old versions of libraries
4021+
# are not copied, taking up unnecessary space.
4022+
call rm -rf "${TF_DEST_DIR}/${lib_full_name}"*
4023+
# Copy TensorFlow libraries, preserving symlinks.
4024+
call find "${TF_LIB_DIR}" \( -regex ".*${lib_full_name}[0-9.]*" \) -exec cp -a -v {} "${TF_DEST_DIR}" \;
40204025
done
40214026
continue
40224027
;;

0 commit comments

Comments
 (0)