build: gitian, linting and macOS-specific QoL improvements (part 3)

contrib/macdeploy/README.md

@@ -60,7 +60,7 @@ Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple
 
 Apple uses `clang` extensively for development and has upstreamed the necessary
 functionality so that a vanilla clang can take advantage. It supports the use of `-F`,
-`-target`, `-mmacosx-version-min`, and `--sysroot`, which are all necessary when
+`-target`, `-mmacosx-version-min`, and `-isysroot`, which are all necessary when
 building for macOS.
 
 Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the

depends/README.md

@@ -34,7 +34,7 @@ Install the required dependencies: Ubuntu & Debian
 
 For macOS cross compilation:
 
-    sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils imagemagick libcap-dev libz-dev libbz2-dev python3-setuptools
+    sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils imagemagick libcap-dev libz-dev libbz2-dev python3-setuptools libtinfo5
 
 For Win64 cross compilation:
 

depends/builders/darwin.mk

@@ -1,5 +1,5 @@
-build_darwin_CC:=$(shell xcrun -f clang) --sysroot $(shell xcrun --show-sdk-path)
-build_darwin_CXX:=$(shell xcrun -f clang++) --sysroot $(shell xcrun --show-sdk-path)
+build_darwin_CC:=$(shell xcrun -f clang) -isysroot$(shell xcrun --show-sdk-path)
+build_darwin_CXX:=$(shell xcrun -f clang++) -isysroot$(shell xcrun --show-sdk-path)
 build_darwin_AR:=$(shell xcrun -f ar)
 build_darwin_RANLIB:=$(shell xcrun -f ranlib)
 build_darwin_STRIP:=$(shell xcrun -f strip)
@@ -10,8 +10,8 @@ build_darwin_SHA256SUM=shasum -a 256
 build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) -o
 
 #darwin host on darwin builder. overrides darwin host preferences.
-darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(shell xcrun --show-sdk-path)
-darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ -fvisibility=hidden --sysroot $(shell xcrun --show-sdk-path)
+darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) -isysroot$(shell xcrun --show-sdk-path)
+darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ -fvisibility=hidden -isysroot$(shell xcrun --show-sdk-path)
 darwin_AR:=$(shell xcrun -f ar)
 darwin_RANLIB:=$(shell xcrun -f ranlib)
 darwin_STRIP:=$(shell xcrun -f strip)

depends/hosts/darwin.mk

@@ -12,14 +12,13 @@ ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 # FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned clang
 # from llvm.org
 
-# The native_cctools package is what provides clang when FORCE_USE_SYSTEM_CLANG
-# is empty
+# Clang is a dependency of native_cctools when FORCE_USE_SYSTEM_CLANG is empty
 darwin_native_toolchain=native_cctools
 
 clang_prog=$(build_prefix)/bin/clang
 clangxx_prog=$(clang_prog)++
 
-clang_resource_dir=$(build_prefix)/lib/clang/$(native_cctools_clang_version)
+clang_resource_dir=$(build_prefix)/lib/clang/$(native_clang_version)
 else
 # FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's
 # system
@@ -101,15 +100,15 @@ darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
               -u LIBRARY_PATH \
             $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \
               -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \
-              --sysroot=$(OSX_SDK) \
+              -isysroot$(OSX_SDK) \
               -Xclang -internal-externc-isystem$(clang_resource_dir)/include \
               -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include
 darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
                -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \
                -u LIBRARY_PATH \
              $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \
                -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \
-               --sysroot=$(OSX_SDK) \
+               -isysroot$(OSX_SDK) \
                -stdlib=libc++ -nostdinc++ \
                -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1 \
                -Xclang -internal-externc-isystem$(clang_resource_dir)/include \

depends/packages/bdb.mk

@@ -4,6 +4,7 @@ $(package)_download_path=http://download.oracle.com/berkeley-db
 $(package)_file_name=db-$($(package)_version).NC.tar.gz
 $(package)_sha256_hash=12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef
 $(package)_build_subdir=build_unix
+$(package)_patches=clang_cxx_11.patch
 
 define $(package)_set_vars
 $(package)_config_opts=--disable-shared --enable-cxx --disable-replication
@@ -16,8 +17,7 @@ $(package)_cppflags_mingw32=-DUNICODE -D_UNICODE
 endef
 
 define $(package)_preprocess_cmds
-  sed -i.old 's/__atomic_compare_exchange/__atomic_compare_exchange_db/' dbinc/atomic.h && \
-  sed -i.old 's/atomic_init/atomic_init_db/' dbinc/atomic.h mp/mp_region.c mp/mp_mvcc.c mp/mp_fget.c mutex/mut_method.c mutex/mut_tas.c && \
+  patch -p1 < $($(package)_patch_dir)/clang_cxx_11.patch && \
   cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub dist
 endef
 

depends/packages/boost.mk

@@ -27,7 +27,9 @@ $(package)_cxxflags=-std=c++11 -fvisibility=hidden
 $(package)_cxxflags_linux=-fPIC
 endef
 
+# Fix unused variable in boost_process, can be removed after upgrading to 1.72
 define $(package)_preprocess_cmds
+  sed -i.old "s/int ret_sig = 0;//" boost/process/detail/posix/wait_group.hpp && \
   echo "using $($(package)_toolset_$(host_os)) : : $($(package)_cxx) : <cflags>\"$($(package)_cflags)\" <cxxflags>\"$($(package)_cxxflags)\" <compileflags>\"$($(package)_cppflags)\" <linkflags>\"$($(package)_ldflags)\" <archiver>\"$($(package)_ar)\" <striper>\"$(host_STRIP)\"  <ranlib>\"$(host_RANLIB)\" <rc>\"$(host_WINDRES)\" : ;" > user-config.jam
 endef
 

depends/packages/fontconfig.mk

@@ -4,22 +4,22 @@ $(package)_download_path=http://www.freedesktop.org/software/fontconfig/release/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=b449a3e10c47e1d1c7a6ec6e2016cca73d3bd68fbbd4f0ae5cc6b573f7d6c7f3
 $(package)_dependencies=freetype expat
+$(package)_patches=remove_char_width_usage.patch gperf_header_regen.patch
 
 define $(package)_set_vars
   $(package)_config_opts=--disable-docs --disable-static --disable-libxml2 --disable-iconv
 endef
 
+define $(package)_preprocess_cmds
+  patch -p1 < $($(package)_patch_dir)/remove_char_width_usage.patch && \
+  patch -p1 < $($(package)_patch_dir)/gperf_header_regen.patch
+endef
+
 define $(package)_config_cmds
   $($(package)_autoconf)
 endef
 
-# 2.12.1 uses CHAR_WIDTH which is reserved and clashes with some glibc versions, but newer versions of fontconfig
-# have broken makefiles which needlessly attempt to re-generate headers with gperf.
-# Instead, change all uses of CHAR_WIDTH, and disable the rule that forces header re-generation.
-# This can be removed once the upstream build is fixed.
 define $(package)_build_cmds
-  sed -i 's/CHAR_WIDTH/CHARWIDTH/g' fontconfig/fontconfig.h src/fcobjshash.gperf src/fcobjs.h src/fcobjshash.h && \
-  sed -i 's/fcobjshash.h: fcobjshash.gperf/fcobjshash.h:/' src/Makefile && \
   $(MAKE)
 endef
 

depends/packages/miniupnpc.mk

@@ -3,6 +3,7 @@ $(package)_version=2.0.20180203
 $(package)_download_path=https://miniupnp.tuxfamily.org/files
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=90dda8c7563ca6cd4a83e23b3c66dbbea89603a1675bfdb852897c2c9cc220b7
+$(package)_patches=dont_use_wingen.patch
 
 define $(package)_set_vars
 $(package)_build_opts=CC="$($(package)_cc)"
@@ -14,7 +15,7 @@ endef
 define $(package)_preprocess_cmds
   mkdir dll && \
   sed -e 's|MINIUPNPC_VERSION_STRING \"version\"|MINIUPNPC_VERSION_STRING \"$($(package)_version)\"|' -e 's|OS/version|$(host)|' miniupnpcstrings.h.in > miniupnpcstrings.h && \
-  sed -i.old "s|miniupnpcstrings.h: miniupnpcstrings.h.in wingenminiupnpcstrings|miniupnpcstrings.h: miniupnpcstrings.h.in|" Makefile.mingw
+  patch -p1 < $($(package)_patch_dir)/dont_use_wingen.patch
 endef
 
 define $(package)_build_cmds

depends/packages/native_cctools.mk

@@ -1,83 +1,24 @@
 package=native_cctools
-$(package)_version=4da2f3b485bcf4cef526f30c0b8c0bcda99cdbb4
+$(package)_version=55562e4073dea0fbfd0b20e0bf69ffe6390c7f97
 $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive
 $(package)_file_name=$($(package)_version).tar.gz
-$(package)_sha256_hash=a2d491c0981cef72fee2b833598f20f42a6c44a7614a61c439bda93d56446fec
+$(package)_sha256_hash=e51995a843533a3dac155dd0c71362dd471597a2d23f13dff194c6285362f875
 $(package)_build_subdir=cctools
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-$(package)_clang_version=8.0.0
-$(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version)
-$(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-$(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-$(package)_clang_sha256_hash=9ef854b71949f825362a119bf2597f744836cb571131ae6b721cd102ffea8cd0
-endif
-
-$(package)_libtapi_version=3efb201881e7a76a21e0554906cf306432539cef
-$(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive
-$(package)_libtapi_download_file=$($(package)_libtapi_version).tar.gz
-$(package)_libtapi_file_name=$($(package)_libtapi_version).tar.gz
-$(package)_libtapi_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3
-
-$(package)_extra_sources=$($(package)_libtapi_file_name)
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-$(package)_extra_sources += $($(package)_clang_file_name)
-endif
-
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-define $(package)_fetch_cmds
-$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash))
-endef
-else
-define $(package)_fetch_cmds
-$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash))
-endef
-endif
-
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-define $(package)_extract_cmds
-  mkdir -p $($(package)_extract_dir) && \
-  echo "$($(package)_sha256_hash)  $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  echo "$($(package)_clang_sha256_hash)  $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  echo "$($(package)_libtapi_sha256_hash)  $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  mkdir -p toolchain/bin toolchain/lib/clang/$($(package)_clang_version)/include && \
-  mkdir -p libtapi && \
-  tar --no-same-owner --strip-components=1 -C libtapi -xf $($(package)_source_dir)/$($(package)_libtapi_file_name) && \
-  tar --no-same-owner --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \
-  rm -f toolchain/lib/libc++abi.so* && \
-  tar --no-same-owner --strip-components=1 -xf $($(package)_source)
-endef
-else
-define $(package)_extract_cmds
-  mkdir -p $($(package)_extract_dir) && \
-  echo "$($(package)_sha256_hash)  $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  echo "$($(package)_libtapi_sha256_hash)  $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  mkdir -p libtapi && \
-  tar --no-same-owner --strip-components=1 -C libtapi -xf $($(package)_source_dir)/$($(package)_libtapi_file_name) && \
-  tar --no-same-owner --strip-components=1 -xf $($(package)_source)
-endef
-endif
+$(package)_patches=ld64_disable_threading.patch
+$(package)_dependencies=native_libtapi
 
 define $(package)_set_vars
-  $(package)_config_opts=--target=$(host) --disable-lto-support --with-libtapi=$($(package)_extract_dir)
+  $(package)_config_opts=--target=$(host)
   $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib
   ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-  $(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang
-  $(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++
-  else
-  $(package)_cc=clang
-  $(package)_cxx=clang++
+  $(package)_config_opts+=--enable-lto-support --with-llvm-config=$(build_prefix)/bin/llvm-config
   endif
+  $(package)_cc=$(clang_prog)
+  $(package)_cxx=$(clangxx_prog)
 endef
 
 define $(package)_preprocess_cmds
-  CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/build.sh && \
-  CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/install.sh && \
-  sed -i.old "/define HAVE_PTHREADS/d" $($(package)_build_subdir)/ld64/src/ld/InputFiles.h
+  patch -p1 < $($(package)_patch_dir)/ld64_disable_threading.patch
 endef
 
 define $(package)_config_cmds
@@ -88,26 +29,10 @@ define $(package)_build_cmds
   $(MAKE)
 endef
 
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 define $(package)_stage_cmds
-  $(MAKE) DESTDIR=$($(package)_staging_dir) install && \
-  mkdir -p $($(package)_staging_prefix_dir)/lib/ && \
-  cd $($(package)_extract_dir) && \
-  cp lib/libtapi.so.6 $($(package)_staging_prefix_dir)/lib/ && \
-  cd $($(package)_extract_dir)/toolchain && \
-  mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include && \
-  mkdir -p $($(package)_staging_prefix_dir)/bin $($(package)_staging_prefix_dir)/include && \
-  cp bin/clang $($(package)_staging_prefix_dir)/bin/ &&\
-  cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\
-  cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \
-  cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \
-  cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install
 endef
-else
-define $(package)_stage_cmds
-  $(MAKE) DESTDIR=$($(package)_staging_dir) install && \
-  mkdir -p $($(package)_staging_prefix_dir)/lib/ && \
-  cd $($(package)_extract_dir) && \
-  cp lib/libtapi.so.6 $($(package)_staging_prefix_dir)/lib/
+
+define $(package)_postprocess_cmds
+  rm -rf share
 endef
-endif

depends/packages/native_clang.mk

@@ -0,0 +1,26 @@
+package=native_clang
+$(package)_version=10.0.1
+$(package)_download_path=https://github.com/llvm/llvm-project/releases/download/llvmorg-$($(package)_version)
+$(package)_download_file=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz
+$(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz
+$(package)_sha256_hash=48b83ef827ac2c213d5b64f5ad7ed082c8bcb712b46644e0dc5045c6f462c231
+
+define $(package)_preprocess_cmds
+  rm -f $($(package)_extract_dir)/lib/libc++abi.so*
+endef
+
+define $(package)_stage_cmds
+  mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include && \
+  mkdir -p $($(package)_staging_prefix_dir)/bin && \
+  mkdir -p $($(package)_staging_prefix_dir)/include && \
+  cp bin/clang $($(package)_staging_prefix_dir)/bin/ && \
+  cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ && \
+  cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \
+  cp bin/llvm-config $($(package)_staging_prefix_dir)/bin/ && \
+  cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \
+  cp -rf lib/clang/$($(package)_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include/
+endef
+
+define $(package)_postprocess_cmds
+  rmdir include
+endef

depends/packages/native_libtapi.mk

@@ -0,0 +1,20 @@
+package=native_libtapi
+$(package)_version=3efb201881e7a76a21e0554906cf306432539cef
+$(package)_download_path=https://github.com/tpoechtrager/apple-libtapi/archive
+$(package)_download_file=$($(package)_version).tar.gz
+$(package)_file_name=$($(package)_version).tar.gz
+$(package)_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3
+
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+$(package)_dependencies=native_clang
+endif
+
+define $(package)_build_cmds
+  CC=$(clang_prog) CXX=$(clangxx_prog) INSTALLPREFIX=$($(package)_staging_prefix_dir) ./build.sh
+endef
+
+define $(package)_stage_cmds
+  ./install.sh && \
+  mkdir -p $($(package)_staging_prefix_dir)/include/llvm-c && \
+  cp src/llvm/include/llvm-c/lto.h $($(package)_staging_prefix_dir)/include/llvm-c
+endef

depends/packages/packages.mk

@@ -17,5 +17,10 @@ darwin_native_packages = native_biplist native_ds_store native_mac_alias
 $(host_arch)_$(host_os)_native_packages += native_b2
 
 ifneq ($(build_os),darwin)
-darwin_native_packages += native_cctools native_cdrkit native_libdmg-hfsplus
+darwin_native_packages += native_cctools native_cdrkit native_libtapi native_libdmg-hfsplus
+
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+darwin_native_packages+= native_clang
+endif
+
 endif