build: Improvements to bundled libraries support. (#6435)

guspan-tanadi · Apr 28, 2023 · ea64926 · ea64926
1 parent 30bf654
commit ea64926
Show file tree

Hide file tree

Showing 21 changed files with 255 additions and 114 deletions.
diff --git a/.ci/macos/deps.sh b/.ci/macos/deps.sh
@@ -1,20 +1,13 @@
 #!/bin/sh -ex
 
 brew install ccache ninja || true
-pip3 install macpack
 
-export FFMPEG_VER=5.1
 export QT_VER=5.15.8
 
 mkdir tmp
 cd tmp/
 
-# install FFMPEG
-wget https://github.com/SachinVin/ext-macos-bin/raw/main/ffmpeg/ffmpeg-${FFMPEG_VER}.7z
-7z x ffmpeg-${FFMPEG_VER}.7z
-cp -rv $(pwd)/ffmpeg-${FFMPEG_VER}/* /
-
 # install Qt
-wget https://github.com/SachinVin/ext-macos-bin/raw/main/qt/qt-${QT_VER}.7z
+wget https://github.com/citra-emu/ext-macos-bin/raw/main/qt/qt-${QT_VER}.7z
 7z x qt-${QT_VER}.7z
 sudo cp -rv $(pwd)/qt-${QT_VER}/* /usr/local/
diff --git a/.ci/macos/upload.sh b/.ci/macos/upload.sh
@@ -9,25 +9,8 @@ COMPRESSION_FLAGS="-czvf"
 mkdir "$REV_NAME"
 
 cp build/bin/Release/citra "$REV_NAME"
+cp -r build/bin/Release/libs "$REV_NAME"
 cp -r build/bin/Release/citra-qt.app "$REV_NAME"
 cp build/bin/Release/citra-room "$REV_NAME"
 
-# move libs into folder for deployment
-macpack "${REV_NAME}/citra-qt.app/Contents/MacOS/citra-qt" -d "../Frameworks"
-# move qt frameworks into app bundle for deployment
-macdeployqt "${REV_NAME}/citra-qt.app" -executable="${REV_NAME}/citra-qt.app/Contents/MacOS/citra-qt"
-
-# move libs into folder for deployment
-macpack "${REV_NAME}/citra" -d "libs"
-
-# workaround for libc++
-install_name_tool -change @loader_path/../Frameworks/libc++.1.0.dylib /usr/lib/libc++.1.dylib "${REV_NAME}/citra-qt.app/Contents/MacOS/citra-qt"
-install_name_tool -change @loader_path/libs/libc++.1.0.dylib /usr/lib/libc++.1.dylib "${REV_NAME}/citra"
-
-# Make the launching script executable
-chmod +x ${REV_NAME}/citra-qt.app/Contents/MacOS/citra-qt
-
-# Verify loader instructions
-find "$REV_NAME" -type f -exec otool -L {} \;
-
 . .ci/common/post-upload.sh
diff --git a/.ci/windows-msvc/build.sh b/.ci/windows-msvc/build.sh
@@ -6,7 +6,6 @@ cmake .. \
     -G Ninja \
     -DCMAKE_TOOLCHAIN_FILE="$(pwd)/../CMakeModules/MSVCCache.cmake" \
     -DCITRA_USE_CCACHE=ON \
-    -DCITRA_USE_BUNDLED_QT=1 \
     -DENABLE_QT_TRANSLATION=OFF \
     -DCITRA_ENABLE_COMPATIBILITY_REPORTING=${ENABLE_COMPATIBILITY_REPORTING:-"OFF"} \
     -DENABLE_COMPATIBILITY_LIST_DOWNLOAD=ON \

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -43,7 +43,7 @@ if (ENABLE_FFMPEG_AUDIO_DECODER OR ENABLE_FFMPEG_VIDEO_DUMPER)
     set(ENABLE_FFMPEG ON)
 endif()
 
-CMAKE_DEPENDENT_OPTION(CITRA_USE_BUNDLED_FFMPEG "Download bundled FFmpeg binaries" ON "ENABLE_FFMPEG;MSVC" OFF)
+CMAKE_DEPENDENT_OPTION(CITRA_USE_BUNDLED_FFMPEG "Download bundled FFmpeg binaries" ON "ENABLE_FFMPEG;MSVC OR APPLE" OFF)
 
 option(USE_DISCORD_PRESENCE "Enables Discord Rich Presence" OFF)
 
@@ -57,6 +57,8 @@ option(USE_SYSTEM_BOOST "Use the system Boost libs (instead of the bundled ones)
 
 CMAKE_DEPENDENT_OPTION(ENABLE_FDK "Use FDK AAC decoder" OFF "NOT ENABLE_FFMPEG_AUDIO_DECODER;NOT ENABLE_MF" OFF)
 
+CMAKE_DEPENDENT_OPTION(CITRA_BUNDLE_LIBRARIES "Bundle dependent libraries with the output executables" ON "APPLE" OFF)
+
 if (CITRA_USE_PRECOMPILED_HEADERS)
     if (MSVC AND CCACHE)
         # buildcache does not properly cache PCH files, leading to compilation errors.
@@ -197,31 +199,25 @@ find_package(Threads REQUIRED)
 
 if (ENABLE_QT)
     if (CITRA_USE_BUNDLED_QT)
-        if (MSVC_VERSION GREATER_EQUAL 1920 AND "x86_64" IN_LIST ARCHITECTURE)
-            set(QT_VER qt-5.15.7-msvc2019_64)
-        else()
-            message(FATAL_ERROR "No bundled Qt binaries for your toolchain. Disable CITRA_USE_BUNDLED_QT and provide your own.")
-        endif()
-
-        if (DEFINED QT_VER)
-            download_bundled_external("qt/" ${QT_VER} QT_PREFIX)
-        endif()
-
-        set(QT_PREFIX_HINT HINTS "${QT_PREFIX}")
-    else()
-        # Passing an empty HINTS seems to cause default system paths to get ignored in CMake 2.8 so
-        # make sure to not pass anything if we don't have one.
-        set(QT_PREFIX_HINT)
+        download_qt_external(5.15.2 QT_PREFIX)
+        list(APPEND CMAKE_PREFIX_PATH ${QT_PREFIX})
     endif()
 
-    find_package(Qt5 REQUIRED COMPONENTS Widgets Multimedia Concurrent ${QT_PREFIX_HINT})
+    find_package(Qt5 REQUIRED COMPONENTS Widgets Multimedia Concurrent)
 
     if (UNIX AND NOT APPLE)
-        find_package(Qt5 REQUIRED COMPONENTS DBus ${QT_PREFIX_HINT})
+        find_package(Qt5 REQUIRED COMPONENTS DBus)
     endif()
 
     if (ENABLE_QT_TRANSLATION)
-        find_package(Qt5 REQUIRED COMPONENTS LinguistTools ${QT_PREFIX_HINT})
+        find_package(Qt5 REQUIRED COMPONENTS LinguistTools)
+    endif()
+
+    if (NOT CITRA_USE_BUNDLED_QT)
+        # Make sure the Qt bin directory is in the prefix path for later consumers.
+        get_target_property(qmake_executable Qt5::qmake IMPORTED_LOCATION)
+        get_filename_component(qt_bin_dir "${qmake_executable}" DIRECTORY)
+        list(APPEND CMAKE_PREFIX_PATH ${qt_bin_dir})
     endif()
 endif()
 
@@ -242,6 +238,8 @@ if (ENABLE_FFMPEG)
     if (CITRA_USE_BUNDLED_FFMPEG)
         if ((MSVC_VERSION GREATER_EQUAL 1920 AND MSVC_VERSION LESS 1940) AND "x86_64" IN_LIST ARCHITECTURE)
             set(FFmpeg_VER "ffmpeg-4.1-win64")
+        elseif (APPLE)
+            set(FFmpeg_VER "ffmpeg-6.0")
         else()
             message(FATAL_ERROR "No bundled FFmpeg binaries for your toolchain. Disable CITRA_USE_BUNDLED_FFMPEG and provide your own.")
         endif()

diff --git a/CMakeModules/BuildInstaller.cmake b/CMakeModules/BuildInstaller.cmake
@@ -1,4 +1,4 @@
-# To use this as a script, make sure you pass in the variables SRC_DIR BUILD_DIR and TARGET_FILE
+# To use this as a script, make sure you pass in the variables BASE_DIR, SRC_DIR, BUILD_DIR, and TARGET_FILE
 
 if(WIN32)
     set(PLATFORM "windows")
@@ -10,29 +10,15 @@ else()
     message(FATAL_ERROR "Cannot build installer for this unsupported platform")
 endif()
 
-set(DIST_DIR "${BUILD_DIR}/dist")
-set(ARCHIVE "${PLATFORM}.7z")
+list(APPEND CMAKE_MODULE_PATH "${BASE_DIR}/CMakeModules")
+include(DownloadExternals)
+download_qt_external(tools_ifw QT_PREFIX)
 
-file(MAKE_DIRECTORY ${BUILD_DIR})
-file(MAKE_DIRECTORY ${DIST_DIR})
-file(DOWNLOAD https://github.com/citra-emu/ext-windows-bin/raw/master/qtifw/${ARCHIVE}
-    "${BUILD_DIR}/${ARCHIVE}" SHOW_PROGRESS)
-execute_process(COMMAND ${CMAKE_COMMAND} -E tar xf "${BUILD_DIR}/${ARCHIVE}"
-    WORKING_DIRECTORY "${BUILD_DIR}/")
+file(GLOB_RECURSE INSTALLER_BASE "${QT_PREFIX}/**/installerbase*")
+file(GLOB_RECURSE BINARY_CREATOR "${QT_PREFIX}/**/binarycreator*")
 
-set(TARGET_NAME "citra-setup-${PLATFORM}")
 set(CONFIG_FILE "${SRC_DIR}/config/config_${PLATFORM}.xml")
-set(INSTALLER_BASE "${BUILD_DIR}/installerbase_${PLATFORM}")
-set(BINARY_CREATOR "${BUILD_DIR}/binarycreator_${PLATFORM}")
 set(PACKAGES_DIR "${BUILD_DIR}/packages")
 file(MAKE_DIRECTORY ${PACKAGES_DIR})
 
-if (UNIX OR APPLE)
-    execute_process(COMMAND chmod 744 ${BINARY_CREATOR})
-endif()
-
 execute_process(COMMAND ${BINARY_CREATOR} -t ${INSTALLER_BASE} -n -c ${CONFIG_FILE} -p ${PACKAGES_DIR} ${TARGET_FILE})
-
-if (APPLE)
-    execute_process(COMMAND chmod 744 ${TARGET_FILE}.app/Contents/MacOS/${TARGET_NAME})
-endif()
diff --git a/CMakeModules/BundleLibraries.cmake b/CMakeModules/BundleLibraries.cmake
@@ -0,0 +1,62 @@
+# Bundles libraries with an output executable.
+# Parameters:
+# - TYPE: "qt" or "standalone". The type of app to bundle.
+#   - "qt" uses windeployqt/macdeployqt to bundle Qt and other libraries.
+#   - "standalone" copies dependent libraries to a "libs" folder alongside the executable file.
+# - EXECUTABLE_PATH: Path to the executable binary.
+
+# TODO: This does not really work fully for Windows yet, some libraries are missing from the output.
+# TODO: Leaving a basic framework of Windows support here to be iterated on in the future.
+if (WIN32)
+    message(FATAL_ERROR "Advanced library bundling is not yet supported for Windows.")
+endif()
+
+if ("${TYPE}" STREQUAL "qt")
+    get_filename_component(executable_dir ${EXECUTABLE_PATH} DIRECTORY)
+
+    # Bundle dependencies using appropriate Qt tool.
+    if (WIN32)
+        find_program(WINDEPLOYQT_EXECUTABLE windeployqt)
+        execute_process(COMMAND ${WINDEPLOYQT_EXECUTABLE} ${EXECUTABLE_PATH}
+            --no-compiler-runtime --no-system-d3d-compiler --no-opengl-sw --no-translations
+            --plugindir "${executable_dir}/plugins")
+    elseif (APPLE)
+        get_filename_component(contents_dir ${executable_dir} DIRECTORY)
+        get_filename_component(bundle_dir ${contents_dir} DIRECTORY)
+
+        find_program(MACDEPLOYQT_EXECUTABLE macdeployqt)
+        execute_process(COMMAND ${MACDEPLOYQT_EXECUTABLE} ${bundle_dir} -executable=${EXECUTABLE_PATH} -always-overwrite)
+    else()
+        message(FATAL_ERROR "Unsupported OS for Qt-based library bundling.")
+    endif()
+else()
+    # Resolve dependent library files.
+    file(GET_RUNTIME_DEPENDENCIES
+        EXECUTABLES ${EXECUTABLE_PATH}
+        RESOLVED_DEPENDENCIES_VAR resolved_deps
+        UNRESOLVED_DEPENDENCIES_VAR unresolved_deps
+        POST_EXCLUDE_REGEXES ".*system32/.*\\.dll")
+
+    # Determine libraries directory.
+    get_filename_component(executable_dir ${EXECUTABLE_PATH} DIRECTORY)
+    if (WIN32)
+        # Same directory since we don't have rpath.
+        set(lib_dir ${executable_dir})
+    else()
+        set(lib_dir ${executable_dir}/libs)
+    endif()
+
+    # Copy library files to bundled output.
+    file(MAKE_DIRECTORY ${lib_dir})
+    foreach (lib_file ${resolved_deps})
+        # Use native copy to turn symlinks into normal files.
+        execute_process(COMMAND cp -L ${lib_file} ${lib_dir})
+    endforeach()
+
+    # Add libs directory to executable rpath where applicable.
+    if (APPLE)
+        execute_process(COMMAND install_name_tool -add_rpath "@loader_path/libs" ${EXECUTABLE_PATH})
+    elseif (UNIX)
+        execute_process(COMMAND patchelf --set-rpath '$ORIGIN/../libs' ${EXECUTABLE_PATH})
+    endif()
+endif()
diff --git a/CMakeModules/DownloadExternals.cmake b/CMakeModules/DownloadExternals.cmake
@@ -4,15 +4,95 @@
 #   remote_path: path to the file to download, relative to the remote repository root
 #   prefix_var: name of a variable which will be set with the path to the extracted contents
 function(download_bundled_external remote_path lib_name prefix_var)
-set(prefix "${CMAKE_BINARY_DIR}/externals/${lib_name}")
-if (NOT EXISTS "${prefix}")
-    message(STATUS "Downloading binaries for ${lib_name}...")
-    file(DOWNLOAD
-        https://github.com/citra-emu/ext-windows-bin/raw/master/${remote_path}${lib_name}.7z
-        "${CMAKE_BINARY_DIR}/externals/${lib_name}.7z" SHOW_PROGRESS)
-    execute_process(COMMAND ${CMAKE_COMMAND} -E tar xf "${CMAKE_BINARY_DIR}/externals/${lib_name}.7z"
-        WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/externals")
-endif()
-message(STATUS "Using bundled binaries at ${prefix}")
-set(${prefix_var} "${prefix}" PARENT_SCOPE)
-endfunction()
+    get_external_prefix(${lib_name} prefix)
+    if (NOT EXISTS "${prefix}")
+        message(STATUS "Downloading binaries for ${lib_name}...")
+
+        if (WIN32)
+            set(repo_base "ext-windows-bin/raw/master")
+        elseif (APPLE)
+            set(repo_base "ext-macos-bin/raw/main")
+        else()
+            message(FATAL_ERROR "Bundled libraries are unsupported for this OS.")
+        endif()
+
+        file(DOWNLOAD
+            https://github.com/citra-emu/${repo_base}/${remote_path}${lib_name}.7z
+            "${CMAKE_BINARY_DIR}/externals/${lib_name}.7z" SHOW_PROGRESS)
+        execute_process(COMMAND ${CMAKE_COMMAND} -E tar xf "${CMAKE_BINARY_DIR}/externals/${lib_name}.7z"
+            WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/externals")
+    endif()
+
+    # For packages that include the /usr/local prefix, include it in the prefix path.
+    if (EXISTS "${prefix}/usr/local")
+        set(prefix "${prefix}/usr/local")
+    endif()
+
+    message(STATUS "Using bundled binaries at ${prefix}")
+    set(${prefix_var} "${prefix}" PARENT_SCOPE)
+endfunction()
+
+# This function downloads Qt using aqt.
+# Params:
+#   target: Qt dependency to install. Specify a version number to download Qt, or "tools_(name)" for a specific build tool.
+#   prefix_var: Name of a variable which will be set with the path to the extracted contents.
+function(download_qt_external target prefix_var)
+    # Determine installation parameters for OS, architecture, and compiler
+    if (WIN32)
+        set(host "windows")
+        if (MINGW)
+            set(arch_path "mingw81")
+        elseif ((MSVC_VERSION GREATER_EQUAL 1920 AND MSVC_VERSION LESS 1940) AND "x86_64" IN_LIST ARCHITECTURE)
+            if ("arm64" IN_LIST ARCHITECTURE)
+                set(arch_path "msvc2019_arm64")
+            elseif ("x86_64" IN_LIST ARCHITECTURE)
+                set(arch_path "msvc2019_64")
+            else()
+                message(FATAL_ERROR "Unsupported bundled Qt architecture. Disable CITRA_USE_BUNDLED_QT and provide your own.")
+            endif()
+        else()
+            message(FATAL_ERROR "Unsupported bundled Qt toolchain. Disable CITRA_USE_BUNDLED_QT and provide your own.")
+        endif()
+        set(arch "win64_${arch_path}")
+    elseif (APPLE)
+        set(host "mac")
+        set(arch "clang_64")
+        set(arch_path "macos")
+    else()
+        set(host "linux")
+        set(arch "gcc_64")
+        set(arch_path "linux")
+    endif()
+
+    get_external_prefix(qt base_path)
+    if (target MATCHES "tools_.*")
+        set(prefix "${base_path}")
+        set(install_args install-tool --outputdir ${base_path} ${host} desktop ${target})
+    else()
+        set(prefix "${base_path}/${target}/${arch_path}")
+        set(install_args install-qt --outputdir ${base_path} ${host} desktop ${target} ${arch})
+    endif()
+
+    if (NOT EXISTS "${prefix}")
+        message(STATUS "Downloading binaries for Qt...")
+        if (WIN32)
+            set(aqt_path "${CMAKE_BINARY_DIR}/externals/aqt.exe")
+            file(DOWNLOAD
+                https://github.com/miurahr/aqtinstall/releases/download/v3.1.4/aqt.exe
+                ${aqt_path} SHOW_PROGRESS)
+            execute_process(COMMAND ${aqt_path} ${install_args})
+        else()
+            # aqt does not offer binary releases for other platforms, so download and run from pip.
+            set(aqt_install_path "${CMAKE_BINARY_DIR}/externals/aqt")
+            execute_process(COMMAND python3 -m pip install --target=${aqt_install_path} aqtinstall)
+            execute_process(COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${aqt_install_path} python3 -m aqt ${install_args})
+        endif()
+    endif()
+
+    message(STATUS "Using downloaded Qt binaries at ${prefix}")
+    set(${prefix_var} "${prefix}" PARENT_SCOPE)
+endfunction()
+
+function(get_external_prefix lib_name prefix_var)
+    set(${prefix_var} "${CMAKE_BINARY_DIR}/externals/${lib_name}" PARENT_SCOPE)
+endfunction()
diff --git a/dist/installer/CMakeLists.txt b/dist/installer/CMakeLists.txt
@@ -5,18 +5,24 @@ elseif(APPLE)
     set(PLATFORM "mac")
 elseif(UNIX)
     set(PLATFORM "linux")
+else()
+    message(FATAL_ERROR "Cannot build installer for this unsupported platform")
 endif()
 
 set(BUILD_DIR "${CMAKE_BINARY_DIR}/installer")
 set(DIST_DIR "${BUILD_DIR}/dist")
 set(TARGET_FILE "${DIST_DIR}/citra-setup-${PLATFORM}")
-file(MAKE_DIRECTORY ${BUILD_DIR})
 
 # Adds a custom target that will run the BuildInstaller.cmake file
 # CMake can't just run a cmake function as a custom command, so this is a way around it.
 # Calls the cmake command and runs a cmake file in "scripting" mode passing in variables with -D
 add_custom_command(OUTPUT "${TARGET_FILE}"
-    COMMAND ${CMAKE_COMMAND} -DSRC_DIR=${CMAKE_CURRENT_SOURCE_DIR} -D BUILD_DIR=${BUILD_DIR} -D TARGET_FILE=${TARGET_FILE} -P ${CMAKE_SOURCE_DIR}/CMakeModules/BuildInstaller.cmake
+    COMMAND ${CMAKE_COMMAND}
+        -DBASE_DIR=${CMAKE_SOURCE_DIR}
+        -DSRC_DIR=${CMAKE_CURRENT_SOURCE_DIR}
+        -DBUILD_DIR=${BUILD_DIR}
+        -DTARGET_FILE=${TARGET_FILE}
+        -P ${CMAKE_SOURCE_DIR}/CMakeModules/BuildInstaller.cmake
     WORKING_DIRECTORY ${BUILD_DIR}
 )
 

diff --git a/externals/cmake-modules/FindFFmpeg.cmake b/externals/cmake-modules/FindFFmpeg.cmake
@@ -152,6 +152,11 @@ foreach(c ${_FFmpeg_ALL_COMPONENTS})
       IMPORTED_LOCATION ${FFmpeg_LIBRARY_${c}}
       INTERFACE_INCLUDE_DIRECTORIES ${FFmpeg_INCLUDE_${c}}
     )
+    if(APPLE)
+      set_target_properties(FFmpeg::${c} PROPERTIES
+        MACOSX_RPATH 1
+      )
+    endif()
     if(_FFmpeg_DEPS_${c})
       set(deps)
       foreach(dep ${_FFmpeg_DEPS_${c}})

diff --git a/src/android/app/src/main/jni/CMakeLists.txt b/src/android/app/src/main/jni/CMakeLists.txt
@@ -33,7 +33,7 @@ add_library(citra-android SHARED
     ndk_motion.h
 )
 
-target_link_libraries(citra-android PRIVATE audio_core common core input_common network)
+target_link_libraries(citra-android PRIVATE audio_core citra_common citra_core input_common network)
 target_link_libraries(citra-android PRIVATE android camera2ndk EGL glad inih jnigraphics log mediandk yuv)
 
 set(CPACK_PACKAGE_EXECUTABLES ${CPACK_PACKAGE_EXECUTABLES} citra-android)
diff --git a/src/audio_core/CMakeLists.txt b/src/audio_core/CMakeLists.txt
@@ -36,7 +36,7 @@ add_library(audio_core STATIC
 
 create_target_directory_groups(audio_core)
 
-target_link_libraries(audio_core PUBLIC common)
+target_link_libraries(audio_core PUBLIC citra_common)
 target_link_libraries(audio_core PRIVATE SoundTouch teakra)
 set_target_properties(audio_core PROPERTIES INTERPROCEDURAL_OPTIMIZATION ${ENABLE_LTO})
 add_definitions(-DSOUNDTOUCH_INTEGER_SAMPLES)