Take a step toward complete BB domination (#33125)

This provides a number of benefits:

* Significantly reduces the amount of work in win-extras, which I've
always thought was an annoying wart in our build system.

* Provides a consistent 7z executable on all platforms, which can be
picked up by Pkg/BinaryProvider for consistent
unzipping/untarring/ungzipping on all platforms. Note that on Windows,
7z is a repackaged binary from sourceforge and is the "full" 7z, whereas
on all other platforms it is a p7zip build of the standalone 7za
executable with all relevant patches applied. There are some format
capability differences, but for the important things it should be fine.

(cherry picked from commit b6ddd87)

Author: staticfloat

LICENSE.md

@@ -52,6 +52,8 @@ USE_SYSTEM_LIBSSH2:=0
 USE_SYSTEM_CURL:=0
 USE_SYSTEM_LIBGIT2:=0
 USE_SYSTEM_PATCHELF:=0
+USE_SYSTEM_ZLIB:=0
+USE_SYSTEM_P7ZIP:=0
 
 # Link to the LLVM shared library
 USE_LLVM_SHLIB := 1
@@ -1005,7 +1007,7 @@ USE_BINARYBUILDER ?= 0
 endif
 
 # This is the set of projects that BinaryBuilder dependencies are hooked up for.
-BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM MBEDTLS LIBSSH2 CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV
+BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM MBEDTLS LIBSSH2 CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP
 define SET_BB_DEFAULT
 # First, check to see if BB is disabled on a global setting
 ifeq ($$(USE_BINARYBUILDER),0)

Make.inc

@@ -275,7 +275,7 @@ ifeq ($(BUNDLE_DEBUG_LIBS),1)
 else
 	@$(MAKE) $(QUIET_MAKE) release
 endif
-	@for subdir in $(bindir) $(datarootdir)/julia/stdlib/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir); do \
+	@for subdir in $(bindir) $(datarootdir)/julia/stdlib/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir) $(libexecdir); do \
 		mkdir -p $(DESTDIR)$$subdir; \
 	done
 
@@ -293,6 +293,9 @@ endif
 	-$(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/
 else
 
+	# Install `7z` into libexec/
+	$(INSTALL_M) $(build_bindir)/7z $(DESTDIR)$(libexecdir)/
+
 # Copy over .dSYM directories directly for Darwin
 ifneq ($(DARWIN_FRAMEWORK),1)
 ifeq ($(OS),Darwin)
@@ -454,8 +457,6 @@ ifeq ($(OS), Darwin)
 endif
 
 ifeq ($(OS), WINNT)
-	[ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \
-		cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin )
 	cd $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe
 
 	# create file listing for uninstall. note: must have Windows path separators and line endings.
@@ -465,7 +466,7 @@ ifeq ($(OS), WINNT)
 	cd $(BUILDROOT) && $(call spawn,$(JULIAHOME)/dist-extras/nsis/makensis.exe) -NOCD -DVersion=$(JULIA_VERSION) -DArch=$(ARCH) -DCommit=$(JULIA_COMMIT) -DJULIAHOME="$(call cygpath_w,$(JULIAHOME))" $(call cygpath_w,$(JULIAHOME)/contrib/windows/build-installer.nsi) | iconv -f latin1
 
 	# compress nsis installer and combine with 7zip self-extracting header
-	cd $(BUILDROOT) && $(JULIAHOME)/dist-extras/7z a -mx=9 "julia-install-$(JULIA_COMMIT)-$(ARCH).7z" julia-installer.exe
+	cd $(BUILDROOT) && $(JULIAHOME)/usr/bin/7z a -mx=9 "julia-install-$(JULIA_COMMIT)-$(ARCH).7z" julia-installer.exe
 	cd $(BUILDROOT) && cat $(JULIAHOME)/contrib/windows/7zS.sfx $(JULIAHOME)/contrib/windows/7zSFX-config.txt "julia-install-$(JULIA_COMMIT)-$(ARCH).7z" > "$(JULIA_BINARYDIST_FILENAME).exe"
 	chmod a+x "$(BUILDROOT)/$(JULIA_BINARYDIST_FILENAME).exe"
 	-rm -f $(BUILDROOT)/julia-install-$(JULIA_COMMIT)-$(ARCH).7z
@@ -585,37 +586,12 @@ test-%: check-whitespace $(JULIA_BUILD_MODE)
 # download target for some hardcoded windows dependencies
 .PHONY: win-extras wine_path
 win-extras:
-	[ -d $(JULIAHOME)/dist-extras ] || mkdir $(JULIAHOME)/dist-extras
-ifneq ($(BUILD_OS),WINNT)
-ifeq (,$(findstring CYGWIN,$(BUILD_OS)))
-	cp /usr/lib/p7zip/7z /usr/lib/p7zip/7z.so $(JULIAHOME)/dist-extras
-endif
-endif
-ifneq (,$(filter $(ARCH), i386 i486 i586 i686))
-	cd $(JULIAHOME)/dist-extras && \
-	$(JLDOWNLOAD) https://sourceforge.net/projects/sevenzip/files/7-Zip/19.00/7z1900.exe && \
-	$(JLCHECKSUM) 7z1900.exe && \
-	7z x -y 7z1900.exe 7z.exe 7z.dll && \
-	../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win32/openSUSE_Leap_42.2 \
-		"mingw32-libexpat1 mingw32-zlib1" && \
-	cp usr/i686-w64-mingw32/sys-root/mingw/bin/*.dll .
-else ifeq ($(ARCH),x86_64)
-	cd $(JULIAHOME)/dist-extras && \
-	$(JLDOWNLOAD) https://downloads.sourceforge.net/project/sevenzip/7-Zip/19.00/7z1900-x64.exe && \
-	$(JLCHECKSUM) 7z1900-x64.exe && \
-	7z x -y 7z1900-x64.exe 7z.exe 7z.dll && \
-	../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win64/openSUSE_Leap_42.2 \
-		"mingw64-libexpat1 mingw64-zlib1" && \
-	cp usr/x86_64-w64-mingw32/sys-root/mingw/bin/*.dll .
-else
-	$(error no win-extras target for ARCH=$(ARCH))
-endif
+	@$(MAKE) -C $(BUILDROOT)/deps install-p7zip
+	mkdir -p $(JULIAHOME)/dist-extras
 	cd $(JULIAHOME)/dist-extras && \
 	$(JLDOWNLOAD) https://sourceforge.net/projects/nsis/files/NSIS%203/3.04/nsis-3.04-setup.exe && \
 	$(JLCHECKSUM) nsis-3.04-setup.exe && \
-	chmod a+x 7z.exe && \
-	chmod a+x 7z.dll && \
-	$(call spawn,./7z.exe) x -y -onsis nsis-3.04-setup.exe && \
+	$(call spawn,$(JULIAHOME)/usr/bin/7z.exe) x -y -onsis nsis-3.04-setup.exe && \
 	chmod a+x ./nsis/makensis.exe
 
 # various statistics about the build that may interest the user

Makefile

@@ -12,7 +12,7 @@
 TRIPLETS="i686-linux-gnu x86_64-linux-gnu aarch64-linux-gnu arm-linux-gnueabihf powerpc64le-linux-gnu i686-linux-musl x86_64-linux-musl aarch64-linux-musl arm-linux-musleabihf x86_64-apple-darwin14 x86_64-unknown-freebsd11.1 i686-w64-mingw32 x86_64-w64-mingw32"
 
 # These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded:
-BB_PROJECTS="gmp mbedtls libssh2 mpfr curl libgit2 pcre libuv unwind osxunwind dsfmt objconv"
+BB_PROJECTS="gmp mbedtls libssh2 mpfr curl libgit2 pcre libuv unwind osxunwind dsfmt objconv p7zip zlib"
 BB_GCC_EXPANDED_PROJECTS="llvm openblas suitesparse openlibm"
 
 # If we've been given a project name, filter down to that one:

contrib/refresh_bb_tarballs.sh

@@ -139,6 +139,15 @@ ifeq ($(USE_SYSTEM_UTF8PROC), 0)
 DEP_LIBS += utf8proc
 endif
 
+ifeq ($(USE_SYSTEM_LIBZ), 0)
+DEP_LIBS += libz
+endif
+
+ifeq ($(USE_SYSTEM_P7ZIP), 0)
+DEP_LIBS += p7zip
+endif
+
+
 # Only compile standalone LAPACK if we are not using OpenBLAS.
 # OpenBLAS otherwise compiles LAPACK as part of its build.
 # This is useful where one wants to use the vendor BLAS, but
@@ -191,3 +200,5 @@ include $(SRCDIR)/libssh2.mk
 include $(SRCDIR)/curl.mk
 include $(SRCDIR)/libgit2.mk
 include $(SRCDIR)/libwhich.mk
+include $(SRCDIR)/zlib.mk
+include $(SRCDIR)/p7zip.mk

deps/Makefile

@@ -32,6 +32,10 @@ LIBUV_VER = 1.29.1
 LIBUV_BB_REL = 0
 OBJCONV_VER = 2.49.0
 OBJCONV_BB_REL = 0
+ZLIB_VER = 1.2.11
+ZLIB_BB_REL = 3
+P7ZIP_VER = 16.2.0
+P7ZIP_BB_REL = 1
 
 # Specify the version of the Mozilla CA Certificate Store to obtain.
 # The versions of cacert.pem are identified by the date (YYYY-MM-DD) of their changes.

deps/Versions.make

@@ -0,0 +1 @@
+489c5ebbca174fdbfae86846e270e112

deps/checksums/Zlib.v1.2.11-3.aarch64-linux-gnu.tar.gz/md5

@@ -0,0 +1 @@
+da1bac181754711c50909492127f855c9acc2928951ce95310a6d32c03bb731d82fe2f0f97ae482c0da319b09d74b91aa143bd90b992489d386beee492d30e6f

deps/checksums/Zlib.v1.2.11-3.aarch64-linux-gnu.tar.gz/sha512

@@ -0,0 +1 @@
+e132449b6a56b91aa6db9e9b1c386517

deps/checksums/Zlib.v1.2.11-3.aarch64-linux-musl.tar.gz/md5

@@ -0,0 +1 @@
+7faf95166293b65f0c348417520b590b89cffcce6f2be19684398c9d313393a8405aeabbde0855e770a06a858c8684f298ec0123b60c75ff187dbf735be871b1