Merge pull request #15195 from ethereum/some_dependencies_as_git_subm…

…odules

Use submodules for some dependencies.

.circleci/config.yml

@@ -866,7 +866,7 @@ jobs:
           name: Check spelling
           command: |
             ~/.local/bin/codespell \
-              --skip "*.enc,.git,Dockerfile*,LICENSE,codespell_whitelist.txt,codespell_ignored_lines.txt" \
+              --skip "*.enc,.git,Dockerfile*,LICENSE,codespell_whitelist.txt,codespell_ignored_lines.txt,deps" \
               --ignore-words scripts/codespell_whitelist.txt \
               --exclude-file scripts/codespell_ignored_lines.txt
       - matrix_notify_failure_unless_pr
@@ -1273,7 +1273,7 @@ jobs:
       - run:
           name: Install runtime dependencies
           command: |
-            pacman --noconfirm -Syu --noprogressbar --needed z3
+            pacman --noconfirm -Syu --noprogressbar --needed z3 git
       - soltest
 
   t_ubu_clang_soltest: &t_ubu_clang_soltest

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "deps/nlohmann-json"]
+	path = deps/nlohmann-json
+	url = https://github.com/nlohmann/json.git
+[submodule "deps/range-v3"]
+	path = deps/range-v3
+	url = https://github.com/ericniebler/range-v3.git
+[submodule "deps/fmtlib"]
+	path = deps/fmtlib
+	url = https://github.com/fmtlib/fmt.git

Changelog.md

@@ -36,6 +36,8 @@ Bugfixes:
  * Yul Optimizer: Name simplification could lead to forbidden identifiers with a leading and/or trailing dot, e.g., ``x._`` would get simplified into ``x.``.
  * Yul Parser: Fix segfault when parsing very long location comments.
 
+Build System:
+ * Change build system to use git submodules for some dependencies (nlohmann-json, fmtlib & range-v3).
 
 ### 0.8.26 (2024-05-21)
 

cmake/fmtlib.cmake

@@ -1,20 +1,9 @@
-include(FetchContent)
+include(${CMAKE_SOURCE_DIR}/cmake/submodules.cmake)
+initialize_submodule(fmtlib)
 
-FetchContent_Declare(
-	fmtlib
-	PREFIX "${PROJECT_BINARY_DIR}/deps"
-	DOWNLOAD_DIR "${PROJECT_SOURCE_DIR}/deps/downloads"
-	DOWNLOAD_NAME fmt-9.1.0.tar.gz
-	URL https://github.com/fmtlib/fmt/archive/9.1.0.tar.gz
-	URL_HASH SHA256=5dea48d1fcddc3ec571ce2058e13910a0d4a6bab4cc09a809d8b1dd1c88ae6f2
+set(FMT_SYSTEM_HEADERS ON)
+add_subdirectory(
+	${CMAKE_SOURCE_DIR}/deps/fmtlib
+	EXCLUDE_FROM_ALL
 )
 
-if (CMAKE_VERSION VERSION_LESS "3.14.0")
-	FetchContent_GetProperties(fmtlib)
-	if (NOT fmtlib_POPULATED)
-		FetchContent_Populate(fmtlib)
-		add_subdirectory(${fmtlib_SOURCE_DIR} ${fmtlib_BINARY_DIR})
-	endif()
-else()
-	FetchContent_MakeAvailable(fmtlib)
-endif()

cmake/nlohmann-json.cmake

@@ -1,21 +1,9 @@
-include(ExternalProject)
+include(${CMAKE_SOURCE_DIR}/cmake/submodules.cmake)
+initialize_submodule(nlohmann-json)
 
-ExternalProject_Add(nlohmann-json-project
-        DOWNLOAD_DIR "${CMAKE_SOURCE_DIR}/deps/nlohmann/nlohmann"
-        DOWNLOAD_NAME json.hpp
-        DOWNLOAD_NO_EXTRACT 1
-        URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.hpp
-        URL_HASH SHA256=9bea4c8066ef4a1c206b2be5a36302f8926f7fdc6087af5d20b417d0cf103ea6
-        CMAKE_COMMAND true
-        BUILD_COMMAND true
-        INSTALL_COMMAND true
+set(JSON_Install OFF CACHE INTERNAL "")
+add_subdirectory(
+	${CMAKE_SOURCE_DIR}/deps/nlohmann-json
+	EXCLUDE_FROM_ALL
 )
 
-# Create nlohmann-json imported library
-add_library(nlohmann-json INTERFACE IMPORTED)
-file(MAKE_DIRECTORY ${CMAKE_SOURCE_DIR}/deps/nlohmann)  # Must exist.
-set_target_properties(nlohmann-json PROPERTIES
-        INTERFACE_COMPILE_OPTIONS "\$<\$<CXX_COMPILER_ID:MSVC>:/permissive->"
-        INTERFACE_SYSTEM_INCLUDE_DIRECTORIES  ${CMAKE_SOURCE_DIR}/deps/nlohmann
-        INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_SOURCE_DIR}/deps/nlohmann)
-add_dependencies(nlohmann-json nlohmann-json-project)

cmake/range-v3.cmake

@@ -1,38 +1,11 @@
-include(ExternalProject)
+include(${CMAKE_SOURCE_DIR}/cmake/submodules.cmake)
+initialize_submodule(range-v3)
 
-if (${CMAKE_SYSTEM_NAME} STREQUAL "Emscripten")
-    set(RANGE_V3_CMAKE_COMMAND emcmake cmake)
-else()
-    set(RANGE_V3_CMAKE_COMMAND ${CMAKE_COMMAND})
-endif()
-
-set(prefix "${PROJECT_BINARY_DIR}/deps")
-set(RANGE_V3_INCLUDE_DIR "${prefix}/include")
-
-ExternalProject_Add(range-v3-project
-    PREFIX "${prefix}"
-    DOWNLOAD_DIR "${PROJECT_SOURCE_DIR}/deps/downloads"
-    DOWNLOAD_NAME range-v3-0.12.0.tar.gz
-    URL https://github.com/ericniebler/range-v3/archive/0.12.0.tar.gz
-    URL_HASH SHA256=015adb2300a98edfceaf0725beec3337f542af4915cec4d0b89fa0886f4ba9cb
-    CMAKE_COMMAND ${RANGE_V3_CMAKE_COMMAND}
-    CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
-               -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-               -DBUILD_TESTING=OFF
-               -DRANGES_CXX_STD=${CMAKE_CXX_STANDARD}
-               -DRANGE_V3_DOCS=OFF
-               -DRANGE_V3_EXAMPLES=OFF
-               -DRANGE_V3_TESTS=OFF
-               -DRANGES_BUILD_CALENDAR_EXAMPLE=OFF
-               -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-    BUILD_BYPRODUCTS "${RANGE_V3_INCLUDE_DIR}/range/v3/all.hpp"
-)
-
-# Create range-v3 imported library
 add_library(range-v3 INTERFACE IMPORTED)
-file(MAKE_DIRECTORY ${RANGE_V3_INCLUDE_DIR})  # Must exist.
 set_target_properties(range-v3 PROPERTIES
-    INTERFACE_COMPILE_OPTIONS "\$<\$<CXX_COMPILER_ID:MSVC>:/permissive->"
-    INTERFACE_SYSTEM_INCLUDE_DIRECTORIES ${RANGE_V3_INCLUDE_DIR}
-    INTERFACE_INCLUDE_DIRECTORIES ${RANGE_V3_INCLUDE_DIR})
+	INTERFACE_COMPILE_OPTIONS "\$<\$<CXX_COMPILER_ID:MSVC>:/permissive->"
+	INTERFACE_SYSTEM_INCLUDE_DIRECTORIES ${CMAKE_SOURCE_DIR}/deps/range-v3/include
+	INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_SOURCE_DIR}/deps/range-v3/include
+)
 add_dependencies(range-v3 range-v3-project)
+

cmake/submodules.cmake

@@ -0,0 +1,23 @@
+macro(initialize_submodule SUBMODULE_PATH)
+	if(NOT USE_SYSTEM_LIBRARIES)
+		file(GLOB submodule_contents "${CMAKE_SOURCE_DIR}/deps/${SUBMODULE_PATH}/*")
+
+		if(submodule_contents)
+			message(STATUS "git submodule '${SUBMODULE_PATH}' seem to be already initialized: nothing to do.")
+		else()
+			message(STATUS "git submodule '${SUBMODULE_PATH}' seem not to be initialized: implicitly executing 'git submodule update --init '${CMAKE_SOURCE_DIR}/deps/${SUBMODULE_PATH}'.")
+			find_package(Git)
+			if(NOT Git_FOUND)
+				message(FATAL_ERROR "Failed to initialize submodules: 'git' command not found.")
+			endif()
+			execute_process(
+					COMMAND git submodule update --init ${CMAKE_SOURCE_DIR}/deps/${SUBMODULE_PATH}
+					WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+					RESULT_VARIABLE result
+			)
+			if(NOT result EQUAL 0)
+				message(FATAL_ERROR "Failed to initialize submodules: 'git submodule update --init' failed.")
+			endif()
+		endif()
+	endif()
+endmacro()

libsolutil/CMakeLists.txt

@@ -44,7 +44,7 @@ set(sources
 )
 
 add_library(solutil ${sources})
-target_link_libraries(solutil PUBLIC Boost::boost Boost::filesystem Boost::system range-v3 fmt::fmt-header-only nlohmann-json)
+target_link_libraries(solutil PUBLIC Boost::boost Boost::filesystem Boost::system range-v3 fmt::fmt-header-only nlohmann_json::nlohmann_json)
 target_include_directories(solutil PUBLIC "${PROJECT_SOURCE_DIR}")
 add_dependencies(solutil solidity_BuildInfo.h)
 

scripts/check_style.sh

@@ -6,6 +6,7 @@ ERROR_LOG="$(mktemp -t check_style_XXXXXX.log)"
 
 EXCLUDE_FILES=(
     # The line below is left unquoted to allow the shell globbing path expansion
+    deps/*
     test/cmdlineTests/*/{err,output}
     "libsolutil/picosha2.h"
     "test/cmdlineTests/strict_asm_only_cr/input.yul"

scripts/chk_shellscripts/chk_shellscripts.sh

@@ -13,5 +13,5 @@ if [ ! -f "${SHELLCHECK}" ]; then
     exit 1
 fi
 
-mapfile -t FILES < <(find . -type f -name "*.sh")
+mapfile -t FILES < <(find . -type f -name "*.sh" -not -path "./deps/*")
 "${SHELLCHECK}" "${FILES[@]}"

scripts/create_source_tarball.sh

@@ -23,6 +23,8 @@ REPO_ROOT="$(dirname "$0")"/..
     mkdir "$SOLDIR"
     # Store the current source
     git checkout-index -a --prefix="$SOLDIR"
+    # shellcheck disable=SC2016
+    SOLDIR="$SOLDIR" git submodule foreach 'git checkout-index -a --prefix="$SOLDIR/$sm_path/"'
     # Store the commit hash
     echo "$commithash" > "$SOLDIR/commit_hash.txt"
     if [ -e prerelease.txt ] && [ ! -s prerelease.txt ]

scripts/install_evmone.ps1

@@ -5,5 +5,4 @@ $progressPreference = "silentlyContinue"
 
 Invoke-WebRequest -URI "https://github.com/ethereum/evmone/releases/download/v0.11.0/evmone-0.11.0-windows-amd64.zip" -OutFile "evmone.zip"
 tar -xf evmone.zip "bin/evmone.dll"
-mkdir deps
-mv bin/evmone.dll deps
+mv bin/evmone.dll deps/

scripts/release_ppa.sh

@@ -123,13 +123,6 @@ ppafilesurl=https://launchpad.net/~ethereum/+archive/ubuntu/${pparepo}/+files
 git clone --depth 2 --recursive https://github.com/ethereum/solidity.git -b "$branch"
 mv solidity solc
 
-# Fetch dependencies
-mkdir -p ./solc/deps/downloads/ 2>/dev/null || true
-mkdir -p ./solc/deps/nlohmann/nlohmann/ 2>/dev/null || true
-wget -O ./solc/deps/nlohmann/nlohmann/json.hpp https://github.com/nlohmann/json/releases/download/v3.11.3/json.hpp
-wget -O ./solc/deps/downloads/range-v3-0.12.0.tar.gz https://github.com/ericniebler/range-v3/archive/0.12.0.tar.gz
-wget -O ./solc/deps/downloads/fmt-9.1.0.tar.gz https://github.com/fmtlib/fmt/archive/9.1.0.tar.gz
-
 # Determine version
 cd solc
 version=$("$(dirname "$0")/get_version.sh")