From 6898451f5e4c84e79a6f48ea6c46959960743970 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Tue, 19 Mar 2024 20:13:17 -0700 Subject: [PATCH] Setup upb installation with cmake. This installs a upb's static library, headers, and three protoc plugin binaries (upb, upbdefs, upb_minitable). These will now be enabled by default, but can be disabled by setting protobuf_BUILD_LIBUPB=OFF. To qualify this, we hook into our existing install test infrastructure which attempts to build and run our tests without any of the installation artifacts available from the source tree. Public headers are deleted, and builds of exported libraries/binaries are disabled. PiperOrigin-RevId: 617376961 --- .github/workflows/test_cpp.yml | 24 +++++++++++-------- CMakeLists.txt | 5 ++-- cmake/install.cmake | 43 +++++++++++++++++++++++++--------- cmake/libprotobuf-lite.cmake | 5 +++- cmake/libprotobuf.cmake | 5 +++- cmake/libupb.cmake | 5 +++- cmake/tests.cmake | 14 ++++++----- cmake/upb_generators.cmake | 7 +++--- pkg/BUILD.bazel | 1 + 9 files changed, 74 insertions(+), 35 deletions(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 117ccc8e1024..b18bc344af73 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -108,7 +108,9 @@ jobs: command: > -c "set -ex; sccache -z; - cmake . -DWITH_PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=14 -Dprotobuf_WITH_ZLIB=OFF ${{ env.SCCACHE_CMAKE_FLAGS }}; + cmake . -DWITH_PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} + -Dprotobuf_BUILD_LIBUPB=OFF -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=14 + -Dprotobuf_WITH_ZLIB=OFF ${{ env.SCCACHE_CMAKE_FLAGS }}; cmake --build . --parallel 20; ctest --parallel 20; sccache -s" @@ -151,7 +153,7 @@ jobs: command: >- /test.sh ${{ matrix.flags}} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_BUILD_TESTS=ON -Dprotobuf_USE_EXTERNAL_GTEST=ON - -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_BUILD_LIBUPB=ON + -Dprotobuf_ABSL_PROVIDER=package linux-cmake-install: name: Linux CMake Install @@ -175,7 +177,9 @@ jobs: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-63dd26c0c7a808d92673a3e52e848189d4ab0f17 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- - /install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_BUILD_SHARED_LIBS=ON \&\& + /install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }} + -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package + -Dprotobuf_BUILD_SHARED_LIBS=ON \&\& /test.sh ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON @@ -250,7 +254,7 @@ jobs: -c 'set -ex; cd /workspace; sccache -z; - cmake . -Dprotobuf_BUILD_LIBUPB=ON ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }}; + cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }}; cmake --build . --parallel 20; ctest --verbose --parallel 20; sccache -s' @@ -278,7 +282,7 @@ jobs: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- /test.sh ${{ env.SCCACHE_CMAKE_FLAGS }} - -Dprotobuf_BUILD_CONFORMANCE=ON -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_LIBUPB=ON + -Dprotobuf_BUILD_CONFORMANCE=ON -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14 linux-cmake-32-bit: name: Linux CMake 32-bit @@ -306,7 +310,7 @@ jobs: /bin/bash -cex ' cd /workspace; sccache -z; - cmake . -DCMAKE_CXX_STANDARD=14 -Dprotobuf_BUILD_LIBUPB=ON ${{ env.SCCACHE_CMAKE_FLAGS }}; + cmake . -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }}; cmake --build . --parallel 20; ctest --verbose --parallel 20; sccache -s' @@ -370,7 +374,7 @@ jobs: flags: >- -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF - -Dprotobuf_BUILD_EXAMPLES=ON -Dprotobuf_BUILD_LIBUPB=ON + -Dprotobuf_BUILD_EXAMPLES=ON vsversion: '2022' cache-prefix: windows-2022-cmake - name: Windows CMake 2019 @@ -378,7 +382,7 @@ jobs: flags: >- -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF - -Dprotobuf_BUILD_EXAMPLES=ON -Dprotobuf_BUILD_LIBUPB=ON + -Dprotobuf_BUILD_EXAMPLES=ON vsversion: '2019' cache-prefix: windows-2019-cmake # windows-2019 has python3.7 installed, which is incompatible with the latest gcloud @@ -386,14 +390,14 @@ jobs: - name: Windows CMake 32-bit os: windows-2022 flags: >- - -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_LIBUPB=ON + -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF vsversion: '2022' windows-arch: 'win32' cache-prefix: windows-2022-win32-cmake - name: Windows CMake Shared os: windows-2022 flags: >- - -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_LIBUPB=ON + -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_SHARED_LIBS=ON vsversion: '2022' cache-prefix: windows-2022-cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index eaed1fe9cf1c..4169fa1d223e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,7 +31,7 @@ option(protobuf_BUILD_EXAMPLES "Build examples" OFF) option(protobuf_BUILD_PROTOBUF_BINARIES "Build protobuf libraries and protoc compiler" ON) option(protobuf_BUILD_PROTOC_BINARIES "Build libprotoc and protoc compiler" ON) option(protobuf_BUILD_LIBPROTOC "Build libprotoc" OFF) -option(protobuf_BUILD_LIBUPB "Build libupb" OFF) +option(protobuf_BUILD_LIBUPB "Build libupb" ON) option(protobuf_DISABLE_RTTI "Remove runtime type information in the binaries" OFF) option(protobuf_TEST_XML_OUTDIR "Output directory for XML logs from tests." "") option(protobuf_ALLOW_CCACHE "Adjust build flags to allow for ccache support." OFF) @@ -316,10 +316,10 @@ if (protobuf_BUILD_PROTOBUF_BINARIES) endif () if (protobuf_BUILD_LIBUPB) include(${protobuf_SOURCE_DIR}/cmake/libupb.cmake) - include(${protobuf_SOURCE_DIR}/cmake/upb_generators.cmake) if (NOT DEFINED protobuf_LIB_UPB) set(protobuf_LIB_UPB libupb) endif () + include(${protobuf_SOURCE_DIR}/cmake/upb_generators.cmake) endif () if (protobuf_BUILD_PROTOC_BINARIES) include(${protobuf_SOURCE_DIR}/cmake/protoc.cmake) @@ -334,6 +334,7 @@ else () set(protobuf_LIB_PROTOC protobuf::libprotoc) set(protobuf_LIB_PROTOBUF protobuf::libprotobuf) set(protobuf_LIB_PROTOBUF_LITE protobuf::libprotobuf-lite) + set(protobuf_LIB_UPB protobuf::libupb) message(STATUS "CMake installation of Protobuf found.") endif () endif () diff --git a/cmake/install.cmake b/cmake/install.cmake index c822838d0379..676d70236320 100644 --- a/cmake/install.cmake +++ b/cmake/install.cmake @@ -29,12 +29,11 @@ set(_protobuf_libraries libprotobuf-lite libprotobuf) if (protobuf_BUILD_LIBPROTOC) list(APPEND _protobuf_libraries libprotoc) endif (protobuf_BUILD_LIBPROTOC) +if (protobuf_BUILD_LIBUPB) + list(APPEND _protobuf_libraries libupb) +endif () foreach(_library ${_protobuf_libraries}) - set_property(TARGET ${_library} - PROPERTY INTERFACE_INCLUDE_DIRECTORIES - $ - $) if (UNIX AND NOT APPLE) set_property(TARGET ${_library} PROPERTY INSTALL_RPATH "$ORIGIN") @@ -49,16 +48,27 @@ foreach(_library ${_protobuf_libraries}) endforeach() if (protobuf_BUILD_PROTOC_BINARIES) + set(_protobuf_binaries protoc) install(TARGETS protoc EXPORT protobuf-targets RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT protoc BUNDLE DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT protoc) - if (UNIX AND NOT APPLE) - set_property(TARGET protoc - PROPERTY INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}") - elseif (APPLE) - set_property(TARGET protoc - PROPERTY INSTALL_RPATH "@loader_path/../lib") - endif() + if (protobuf_BUILD_LIBUPB) + foreach (generator upb upbdefs upb_minitable) + list(APPEND _protobuf_binaries protoc-gen-${generator}) + install(TARGETS protoc-gen-${generator} EXPORT protobuf-targets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT upb-generators + BUNDLE DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT upb-generators) + endforeach () + endif () + foreach (binary IN LISTS _protobuf_binaries) + if (UNIX AND NOT APPLE) + set_property(TARGET ${binary} + PROPERTY INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}") + elseif (APPLE) + set_property(TARGET ${binary} + PROPERTY INSTALL_RPATH "@loader_path/../lib") + endif () + endforeach () endif (protobuf_BUILD_PROTOC_BINARIES) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/protobuf.pc ${CMAKE_CURRENT_BINARY_DIR}/protobuf-lite.pc DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") @@ -73,6 +83,17 @@ set(protobuf_HEADERS ${plugin_proto_proto_srcs} ${java_features_proto_proto_srcs} ) +if (protobuf_BUILD_LIBUPB) + list(APPEND protobuf_HEADERS ${libupb_hdrs}) + # Manually install the bootstrap headers + install( + FILES + ${protobuf_SOURCE_DIR}/upb/reflection/cmake/google/protobuf/descriptor.upb.h + ${protobuf_SOURCE_DIR}/upb/reflection/cmake/google/protobuf/descriptor.upb_minitable.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/google/protobuf + COMPONENT protobuf-headers + ) +endif () foreach(_header ${protobuf_HEADERS}) string(FIND ${_header} "${protobuf_SOURCE_DIR}/src" _find_src) string(FIND ${_header} "${protobuf_SOURCE_DIR}" _find_nosrc) diff --git a/cmake/libprotobuf-lite.cmake b/cmake/libprotobuf-lite.cmake index f343458cf5a1..dfccbc807777 100644 --- a/cmake/libprotobuf-lite.cmake +++ b/cmake/libprotobuf-lite.cmake @@ -23,7 +23,10 @@ endif() if(${CMAKE_SYSTEM_NAME} STREQUAL "Android") target_link_libraries(libprotobuf-lite PRIVATE log) endif() -target_include_directories(libprotobuf-lite PUBLIC ${protobuf_SOURCE_DIR}/src) +target_include_directories(libprotobuf-lite PUBLIC + $ + $ +) target_link_libraries(libprotobuf-lite PUBLIC ${protobuf_ABSL_USED_TARGETS}) protobuf_configure_target(libprotobuf-lite) if(protobuf_BUILD_SHARED_LIBS) diff --git a/cmake/libprotobuf.cmake b/cmake/libprotobuf.cmake index 422754a1ab5f..11c09b1bc0e1 100644 --- a/cmake/libprotobuf.cmake +++ b/cmake/libprotobuf.cmake @@ -26,7 +26,10 @@ endif() if(${CMAKE_SYSTEM_NAME} STREQUAL "Android") target_link_libraries(libprotobuf PRIVATE log) endif() -target_include_directories(libprotobuf PUBLIC ${protobuf_SOURCE_DIR}/src) +target_include_directories(libprotobuf PUBLIC + $ + $ +) target_link_libraries(libprotobuf PUBLIC ${protobuf_ABSL_USED_TARGETS}) protobuf_configure_target(libprotobuf) if(protobuf_BUILD_SHARED_LIBS) diff --git a/cmake/libupb.cmake b/cmake/libupb.cmake index a9558463dcb6..7476e56eb788 100644 --- a/cmake/libupb.cmake +++ b/cmake/libupb.cmake @@ -20,7 +20,10 @@ add_library(libupb STATIC ${protobuf_version_rc_file} ) -target_include_directories(libupb PUBLIC ${bootstrap_cmake_dir}) +target_include_directories(libupb PUBLIC + $ + $ +) protobuf_configure_target(libupb) diff --git a/cmake/tests.cmake b/cmake/tests.cmake index 351aeb0d1f8d..45e775b765df 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -206,8 +206,8 @@ if (protobuf_BUILD_LIBUPB) OUT_VAR pb_generated_files IMPORT_DIRS ${protobuf_SOURCE_DIR}/src IMPORT_DIRS ${protobuf_SOURCE_DIR} - PLUGIN protoc-gen-${generator}=$ - DEPENDENCIES $ + PLUGIN protoc-gen-${generator}=$ + DEPENDENCIES $ ) set(upb_test_proto_genfiles ${upb_test_proto_genfiles} ${pb_generated_files}) endforeach() @@ -219,8 +219,8 @@ if (protobuf_BUILD_LIBUPB) ${upb_test_util_files}) target_link_libraries(upb-test - libprotobuf - libupb + ${protobuf_LIB_PROTOBUF} + ${protobuf_LIB_UPB} ${protobuf_ABSL_USED_TARGETS} ${protobuf_ABSL_USED_TEST_TARGETS} GTest::gmock_main) @@ -239,7 +239,9 @@ add_custom_target(restore-installed-headers) file(GLOB_RECURSE _local_hdrs "${PROJECT_SOURCE_DIR}/src/*.h" - "${PROJECT_SOURCE_DIR}/src/*.inc") + "${PROJECT_SOURCE_DIR}/src/*.inc" + "${PROJECT_SOURCE_DIR}/upb/*.h" +) # Exclude the bootstrapping that are directly used by tests. set(_exclude_hdrs @@ -250,7 +252,7 @@ set(_exclude_hdrs # Exclude test library headers. list(APPEND _exclude_hdrs ${test_util_hdrs} ${lite_test_util_hdrs} ${common_test_hdrs} - ${compiler_test_utils_hdrs}) + ${compiler_test_utils_hdrs} ${upb_test_util_files}) foreach(_hdr ${_exclude_hdrs}) list(REMOVE_ITEM _local_hdrs ${_hdr}) endforeach() diff --git a/cmake/upb_generators.cmake b/cmake/upb_generators.cmake index 2944dc5dc8a9..b2e66b130e34 100644 --- a/cmake/upb_generators.cmake +++ b/cmake/upb_generators.cmake @@ -14,11 +14,12 @@ foreach(generator upb upbdefs upb_minitable) ${bootstrap_sources} ${protobuf_version_rc_file} ) - target_include_directories(protoc-gen-${generator} PUBLIC ${bootstrap_cmake_dir}) + target_include_directories(protoc-gen-${generator} PRIVATE ${bootstrap_cmake_dir}) target_link_libraries(protoc-gen-${generator} - libprotobuf - libupb + ${protobuf_LIB_PROTOBUF} + ${protobuf_LIB_UPB} ${protobuf_ABSL_USED_TARGETS} ) set_target_properties(protoc-gen-${generator} PROPERTIES VERSION ${protobuf_VERSION}) + add_executable(protobuf::protoc-gen-${generator} ALIAS protoc-gen-${generator}) endforeach() diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel index 85df654a03e5..b87b4e974551 100644 --- a/pkg/BUILD.bazel +++ b/pkg/BUILD.bazel @@ -208,6 +208,7 @@ cc_dist_library( name = "upb", tags = ["manual"], deps = [ + "//upb:generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", "//upb/json", "//upb/message:compare", "//upb/message:copy",