Add new checksum refreshing infrastructure (#38465)

Co-authored-by: Jameson Nash <vtjnash@gmail.com>

Author: staticfloat

contrib/refresh_bb_tarballs.sh

@@ -0,0 +1,83 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+# Invoke this with no arguments to refresh all tarballs, or with a project name to refresh only that project.
+#
+# Example:
+#   make -f contrib/refresh_checksums.mk gmp
+
+SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
+JULIAHOME := $(abspath $(SRCDIR)/..)
+
+# Default target that will have everything else added to it as a dependency
+all:
+
+# Get this list via:
+#    using BinaryBuilder
+#    print("TRIPLETS=\"$(join(sort(triplet.(BinaryBuilder.supported_platforms(;experimental=true))), " "))\"")
+TRIPLETS=aarch64-apple-darwin aarch64-linux-gnu aarch64-linux-musl armv6l-linux-gnueabihf armv6l-linux-musleabihf armv7l-linux-gnueabihf armv7l-linux-musleabihf i686-linux-gnu i686-linux-musl i686-w64-mingw32 powerpc64le-linux-gnu x86_64-apple-darwin x86_64-linux-gnu x86_64-linux-musl x86_64-unknown-freebsd x86_64-w64-mingw32
+CLANG_TRIPLETS=$(filter %-darwin %-freebsd,$(TRIPLETS))
+NON_CLANG_TRIPLETS=$(filter-out %-darwin %-freebsd,$(TRIPLETS))
+
+# These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded:
+BB_PROJECTS=mbedtls libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind dsfmt objconv p7zip zlib suitesparse openlibm
+BB_GCC_EXPANDED_PROJECTS=openblas
+BB_CXX_EXPANDED_PROJECTS=gmp llvm
+# These are non-BB source-only deps
+NON_BB_PROJECTS=patchelf mozillacert lapack
+
+# Convert `openblas` to `OPENBLAS`
+define upper
+$(shell echo $(1) | tr 'a-z' 'A-Z')
+endef
+
+# If $(2) == `src`, this will generate a `USE_BINARYBUILDER_FOO=0` make flag
+# It will also generate a `FOO_BB_TRIPLET=$(2)` make flag.
+define make_flags
+USE_BINARYBUILDER_$(call upper,$(1))=$(if $(filter src,$(2)),0,1) $(call upper,$(1))_BB_TRIPLET=$(if $(filter src,$(2)),,$(2)) DEPS_GIT=0
+endef
+
+# checksum_bb_dep takes in (name, triplet), and generates a `checksum-$(1)-$(2)` target.
+# note that `"src"` is a special triplet value.
+define checksum_dep
+checksum-$(1)-$(2):
+	@-$(MAKE) -C "$(JULIAHOME)/deps" $(call make_flags,$(1),$(2)) checksum-$(1)
+
+# Add this guy to his project target (e.g. `make -f contrib/refresh_checksums.mk openblas`)
+$(1): checksum-$(1)-$(2)
+
+# Add this guy to the `all` default target
+all: checksum-$(1)-$(2)
+endef
+
+# Generate targets for source hashes for all our projects
+$(foreach project,$(BB_PROJECTS) $(BB_GCC_EXPANDED_PROJECTS) $(BB_CXX_EXPANDED_PROJECTS) $(NON_BB_PROJECTS),$(eval $(call checksum_dep,$(project),src)))
+
+# Generate targets for triplet-specific hashes for all our BB projects
+$(foreach project,$(BB_PROJECTS),$(foreach triplet,$(TRIPLETS),$(eval $(call checksum_dep,$(project),$(triplet)))))
+$(foreach project,$(BB_GCC_EXPANDED_PROJECTS),$(foreach triplet,$(TRIPLETS),$(foreach libgfortran_version,libgfortran3 libgfortran4 libgfortran5,$(eval $(call checksum_dep,$(project),$(triplet)-$(libgfortran_version))))))
+
+# Because MacOS and FreeBSD use clang, they don't actually use cxxstring_abi expansion:
+$(foreach project,$(BB_CXX_EXPANDED_PROJECTS),$(foreach triplet,$(NON_CLANG_TRIPLETS),$(foreach cxxstring_abi,cxx11 cxx03,$(eval $(call checksum_dep,$(project),$(triplet)-$(cxxstring_abi))))))
+$(foreach project,$(BB_CXX_EXPANDED_PROJECTS),$(foreach triplet,$(CLANG_TRIPLETS),$(eval $(call checksum_dep,$(project),$(triplet)))))
+
+# External stdlibs
+checksum-stdlibs:
+	@-$(MAKE) -C "$(JULIAHOME)/stdlib" checksumall
+all: checksum-stdlibs
+
+# doc unicode data
+checksum-doc-unicodedata:
+	@-$(MAKE) -C "$(JULIAHOME)/doc" checksum-unicodedata
+all: checksum-doc-unicodedata
+
+# Special LLVM source hashes for optional targets
+checksum-llvm-special-src:
+	@-$(MAKE) -C "${JULIAHOME}/deps" USE_BINARYBUILDER_LLVM=0 DEPS_GIT=0 BUILD_LLDB=1 BUILD_LLVM_CLANG=1 BUILD_CUSTOM_LIBCXX=1 checksum-llvm
+all: checksum-llvm-special-src
+
+# This file is completely phony
+.PHONY: checksum-*
+
+# Debugging helper
+print-%:
+	@echo '$*=$(subst ','\'',$($*))'

contrib/refresh_checksums.mk

@@ -1,4 +1,5 @@
 ## OpenBLAS ##
+ifneq ($(USE_BINARYBUILDER_OPENBLAS), 1)
 # LAPACK is built into OpenBLAS by default
 OPENBLAS_GIT_URL := git://github.com/xianyi/OpenBLAS.git
 OPENBLAS_TAR_URL = https://api.github.com/repos/xianyi/OpenBLAS/tarball/$1
@@ -92,8 +93,6 @@ endif
 # Do not overwrite the "-j" flag
 OPENBLAS_BUILD_OPTS += MAKE_NB_JOBS=0
 
-ifneq ($(USE_BINARYBUILDER_OPENBLAS), 1)
-
 $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/openblas-winexit.patch-applied: $(BUILDDIR)/$(OPENBLAS_SRC_DIR)/source-extracted
 	cd $(BUILDDIR)/$(OPENBLAS_SRC_DIR) && \
 		patch -p1 -f < $(SRCDIR)/patches/openblas-winexit.patch
@@ -164,6 +163,9 @@ $(BUILDDIR)/lapack-$(LAPACK_VER)/source-extracted: $(SRCCACHE)/lapack-$(LAPACK_V
 	cp $(dir $@)INSTALL/make.inc.gfortran $(dir $@)make.inc
 	echo 1 > $@
 
+checksum-lapack: $(SRCCACHE)/lapack-$(LAPACK_VER).tgz
+	$(JLCHECKSUM) $<
+
 ifeq ($(USE_SYSTEM_BLAS), 0)
 $(BUILDDIR)/lapack-$(LAPACK_VER)/build-compiled0: | $(build_prefix)/manifest/openblas
 else ifeq ($(OS),Darwin)

deps/blas.mk

@@ -26,6 +26,9 @@ $(SRCCACHE)/curl-$(CURL_VER)/source-extracted: $(SRCCACHE)/curl-$(CURL_VER).tar.
 	touch -c $(SRCCACHE)/curl-$(CURL_VER)/configure # old target
 	echo 1 > $@
 
+checksum-curl: $(SRCCACHE)/curl-$(CURL_VER).tar.bz2
+	$(JLCHECKSUM) $<
+
 $(BUILDDIR)/curl-$(CURL_VER)/build-configured: $(SRCCACHE)/curl-$(CURL_VER)/source-extracted
 	mkdir -p $(dir $@)
 	cd $(dir $@) && \

deps/curl.mk

@@ -26,6 +26,9 @@ $(BUILDDIR)/dsfmt-$(DSFMT_VER)/source-extracted: $(SRCCACHE)/dsfmt-$(DSFMT_VER).
 	cd $(dir $@) && patch < $(SRCDIR)/patches/dSFMT.c.patch
 	echo 1 > $@
 
+checksum-dsfmt: $(SRCCACHE)/dsfmt-$(DSFMT_VER).tar.gz
+	$(JLCHECKSUM) $<
+
 $(BUILDDIR)/dsfmt-$(DSFMT_VER)/build-compiled: $(BUILDDIR)/dsfmt-$(DSFMT_VER)/source-extracted
 	cd $(dir $<) && \
 	$(CC) $(CPPFLAGS) $(DSFMT_CFLAGS) $(LDFLAGS) dSFMT.c -o libdSFMT.$(SHLIB_EXT)

deps/dsfmt.mk

@@ -19,6 +19,9 @@ $(SRCCACHE)/gmp-$(GMP_VER)/source-extracted: $(SRCCACHE)/gmp-$(GMP_VER).tar.bz2
 	touch -c $(SRCCACHE)/gmp-$(GMP_VER)/configure # old target
 	echo 1 > $@
 
+checksum-gmp: $(SRCCACHE)/gmp-$(GMP_VER).tar.bz2
+	$(JLCHECKSUM) $<
+
 $(SRCCACHE)/gmp-$(GMP_VER)/build-patched: $(SRCCACHE)/gmp-$(GMP_VER)/source-extracted
 	cd $(dir $@) && patch -p1 < $(SRCDIR)/patches/gmp-exception.patch
 	cd $(dir $@) && patch -p1 < $(SRCDIR)/patches/gmp_alloc_overflow_func.patch

deps/gmp.mk

@@ -1,4 +1,5 @@
 ## libgit2
+ifneq ($(USE_BINARYBUILDER_LIBGIT2),1)
 
 LIBGIT2_GIT_URL := git://github.com/libgit2/libgit2.git
 LIBGIT2_TAR_URL = https://api.github.com/repos/libgit2/libgit2/tarball/$1
@@ -12,8 +13,6 @@ ifeq ($(USE_SYSTEM_MBEDTLS), 0)
 $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured: | $(build_prefix)/manifest/mbedtls
 endif
 
-ifneq ($(USE_BINARYBUILDER_LIBGIT2),1)
-
 LIBGIT2_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DTHREADSAFE=ON -DUSE_BUNDLED_ZLIB=ON
 ifeq ($(OS),WINNT)
 LIBGIT2_OPTS += -DWIN32=ON -DMINGW=ON
@@ -114,5 +113,8 @@ $(build_datarootdir)/julia/cert.pem: $(SRCCACHE)/cacert-$(MOZILLA_CACERT_VERSION
 	mkdir -p $(build_datarootdir)/julia
 	cp $< $@
 
+checksum-mozillacert: $(SRCCACHE)/cacert-$(MOZILLA_CACERT_VERSION).pem
+	$(JLCHECKSUM) $<
+
 # When "get"'ing libgit2, download the .pem
 get-libgit2: $(SRCCACHE)/cacert-$(MOZILLA_CACERT_VERSION).pem

deps/libgit2.mk

@@ -1,5 +1,5 @@
 ## libssh2
-
+ifneq ($(USE_BINARYBUILDER_LIBSSH2), 1)
 LIBSSH2_GIT_URL := git://github.com/libssh2/libssh2.git
 LIBSSH2_TAR_URL = https://api.github.com/repos/libssh2/libssh2/tarball/$1
 $(eval $(call git-external,libssh2,LIBSSH2,CMakeLists.txt,,$(SRCCACHE)))
@@ -8,7 +8,6 @@ ifeq ($(USE_SYSTEM_MBEDTLS), 0)
 $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-configured: | $(build_prefix)/manifest/mbedtls
 endif
 
-ifneq ($(USE_BINARYBUILDER_LIBSSH2), 1)
 LIBSSH2_OPTS := $(CMAKE_COMMON) -DBUILD_SHARED_LIBS=ON -DBUILD_EXAMPLES=OFF \
 		-DCMAKE_BUILD_TYPE=Release
 

deps/libssh2.mk

@@ -1,10 +1,9 @@
 ## LIBUV ##
+ifneq ($(USE_BINARYBUILDER_LIBUV),1)
 LIBUV_GIT_URL:=git://github.com/JuliaLang/libuv.git
 LIBUV_TAR_URL=https://api.github.com/repos/JuliaLang/libuv/tarball/$1
 $(eval $(call git-external,libuv,LIBUV,configure,,$(SRCCACHE)))
 
-ifneq ($(USE_BINARYBUILDER_LIBUV),1)
-
 UV_CFLAGS := -O2
 ifeq ($(USEMSVC), 1)
 UV_CFLAGS += -DBUILDING_UV_SHARED

deps/libuv.mk

@@ -340,6 +340,7 @@ distclean-libcxx:
 distclean-libcxxabi:
 	-rm -rf $(LLVM_LIBCXXABI_TAR) $(LLVM_SRC_DIR)/projects/libcxxabi $(LLVM_BUILD_DIR)/libcxxabi-build
 
+
 # We want to ensure that the libcxx linking flags don't get passed to the libcxx build, since it will
 # error on a fresh build
 LLVM_CMAKE += -DCMAKE_EXE_LINKER_FLAGS="$(LLVM_LDFLAGS) $(LLVM_LIBCXX_LDFLAGS)" \
@@ -355,6 +356,23 @@ LIBCXX_DEPENDENCY := $(build_libdir)/libc++abi.so.1.0 $(build_libdir)/libc++.so.
 get-llvm: get-libcxx get-libcxxabi
 endif
 
+checksum-llvm: $(LLVM_TAR) $(LLVM_CLANG_TAR) $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR)
+ifneq ($(LLVM_CLANG_TAR),)
+	$(JLCHECKSUM) $(LLVM_CLANG_TAR)
+endif
+ifneq ($(LLVM_COMPILER_RT_TAR),)
+	$(JLCHECKSUM) $(LLVM_COMPILER_RT_TAR)
+endif
+ifneq ($(LLVM_LIBCXX_TAR),)
+	$(JLCHECKSUM) $(LLVM_LIBCXX_TAR)
+endif
+ifneq ($(LLVM_VER),svn)
+	$(JLCHECKSUM) $(LLVM_TAR)
+endif
+ifneq ($(LLVM_LLDB_TAR),)
+	$(JLCHECKSUM) $(LLVM_LLDB_TAR)
+endif
+
 $(LLVM_SRC_DIR)/source-extracted: | $(LLVM_TAR) $(LLVM_CLANG_TAR) $(LLVM_COMPILER_RT_TAR) $(LLVM_LIBCXX_TAR) $(LLVM_LLDB_TAR)
 ifneq ($(LLVM_CLANG_TAR),)
 	$(JLCHECKSUM) $(LLVM_CLANG_TAR)