Patches for Mac (including ARM NEON 128 bits)

epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Makefile

@@ -8,9 +8,11 @@ SHELL = /bin/bash
 
 # Detect O/S kernel (Linux, Darwin...)
 UNAME_S := $(shell uname -s)
+###$(info UNAME_S='$(UNAME_S)')
 
 # Detect architecture (x86_64, ppc64le...)
 UNAME_P := $(shell uname -p)
+###$(info UNAME_P='$(UNAME_P)')
 
 #-------------------------------------------------------------------------------
 
@@ -42,6 +44,9 @@ CXXFLAGS+= -ffast-math # see issue #117
 ###CXXFLAGS+= -Ofast # performance is not different from --fast-math
 ###CXXFLAGS+= -g # FOR DEBUGGING ONLY
 
+# Optionally add debug flags to display the full list of flags (eg on Darwin)
+###CXXFLAGS+= -v
+
 # Note: AR, CXX and FC are implicitly defined if not set externally
 # See https://www.gnu.org/software/make/manual/html_node/Implicit-Variables.html
 
@@ -154,6 +159,12 @@ ifeq ($(AVX),)
   ifeq ($(UNAME_P),ppc64le)
     ###override AVX = none
     override AVX = sse4
+  else ifeq ($(UNAME_P),arm)
+    ###override AVX = none
+    override AVX = sse4
+  else ifeq ($(wildcard /proc/cpuinfo),)
+    override AVX = none
+    $(warning Using AVX='$(AVX)' because host SIMD features cannot be read from /proc/cpuinfo)
   else ifeq ($(shell grep -m1 -c avx512vl /proc/cpuinfo)$(shell $(CXX) --version | grep ^clang),1)
     override AVX = 512y
     ###$(info Using AVX='$(AVX)' as no user input exists)
@@ -217,6 +228,12 @@ ifeq ($(UNAME_P),ppc64le)
   else ifneq ($(AVX),none)
     $(error Unknown AVX='$(AVX)': only 'none' and 'sse4' are supported on PowerPC for the moment)
   endif
+else ifeq ($(UNAME_P),arm)
+  ifeq ($(AVX),sse4)
+    override AVXFLAGS = -D__SSE4_2__ # ARM NEON with 128 width (Q/quadword registers)
+  else ifneq ($(AVX),none)
+    $(error Unknown AVX='$(AVX)': only 'none' and 'sse4' are supported on ARM for the moment)
+  endif
 else
   ifeq ($(AVX),sse4)
     override AVXFLAGS = -march=nehalem # SSE4.2 with 128 width (xmm registers)
@@ -287,27 +304,31 @@ override TAG = $(AVX)_$(FPTYPE)_inl$(HELINL)_hrd$(HRDCOD)_$(RNDGEN)
 ifeq ($(USEBUILDDIR),1)
   override BUILDDIR = build.$(DIRTAG)
   override LIBDIR = ../../lib/$(BUILDDIR)
-  override LIBDIRRPATH = '$(RPATHPREFIX)../$(LIBDIR)'
+  override LIBDIRRPATH = '$$ORIGIN/../$(LIBDIR)'
   $(info Building in BUILDDIR=$(BUILDDIR) for tag=$(TAG) (USEBUILDDIR is set = 1))
 else
   override BUILDDIR = .
   override LIBDIR = ../../lib
-  override LIBDIRRPATH = '$(RPATHPREFIX)$(LIBDIR)'
+  override LIBDIRRPATH = '$$ORIGIN/$(LIBDIR)'
   $(info Building in BUILDDIR=$(BUILDDIR) for tag=$(TAG) (USEBUILDDIR is not set))
 endif
 override INCDIR = ../../include
 ###$(info Building in BUILDDIR=$(BUILDDIR) for tag=$(TAG))
 
-# Set rpath to LIBDIR to make it unnecessary to use LD_LIBRARY_PATH or DYLD_LIBRARY_PATH
-# Use relative paths with respect to the executables ($ORIGIN on Linux and @executable_path on Darwin)
+# On Linux, set rpath to LIBDIR to make it unnecessary to use LD_LIBRARY_PATH
+# Use relative paths with respect to the executables ($ORIGIN on Linux)
+# On Darwin, building libraries with absolute paths in LIBDIR makes this unnecessary
 ifeq ($(UNAME_S),Darwin)
-  override RPATHPREFIX = @executable_path/
-  override RUNTIME = DYLD_LIBRARY_PATH="$(LIBDIR):$$DYLD_LIBRARY_PATH" # TEMPORARY for Mac! #375
+  override CXXLIBFLAGSRPATH =
+  override CULIBFLAGSRPATH =
 else
-  override RPATHPREFIX = $$ORIGIN/
-  override RUNTIME =
+  override CXXLIBFLAGSRPATH = -Wl,-rpath,$(LIBDIRRPATH)
+  override CULIBFLAGSRPATH = -Xlinker -rpath,$(LIBDIRRPATH)
 endif
 
+# Setting LD_LIBRARY_PATH or DYLD_LIBRARY_PATH in the RUNTIME is no longer necessary (neither on Linux nor on Mac)
+override RUNTIME =
+
 #===============================================================================
 #=== Makefile TARGETS and build rules below
 #===============================================================================
@@ -413,7 +434,7 @@ $(INCDIR)/%%.inc : ../%%.inc
 #-------------------------------------------------------------------------------
 
 # Target (and build rules): C++ and CUDA standalone executables
-$(cxx_main): LIBFLAGS += -Wl,-rpath,$(LIBDIRRPATH) # avoid the need for LD_LIBRARY_PATH to execute ./runTest.exe
+$(cxx_main): LIBFLAGS += $(CXXLIBFLAGSRPATH) # avoid the need for LD_LIBRARY_PATH
 $(cxx_main): $(BUILDDIR)/check_sa.o $(LIBDIR)/lib$(MG5AMC_CXXLIB).so
 	$(CXX) -o $@ $(BUILDDIR)/check_sa.o $(OMPFLAGS) -ldl -pthread $(LIBFLAGS) $(CULIBFLAGS) -L$(LIBDIR) -l$(MG5AMC_CXXLIB)
 
@@ -422,7 +443,7 @@ ifneq ($(shell $(CXX) --version | grep ^Intel),)
 $(cu_main): LIBFLAGS += -lintlc # compile with icpx and link with nvcc (undefined reference to `_intel_fast_memcpy')
 $(cu_main): LIBFLAGS += -lsvml # compile with icpx and link with nvcc (undefined reference to `__svml_cos4_l9')
 endif
-$(cu_main): LIBFLAGS += -Xlinker -rpath,$(LIBDIRRPATH) # avoid the need for LD_LIBRARY_PATH to execute ./runTest.exe
+$(cu_main): LIBFLAGS += $(CULIBFLAGSRPATH) # avoid the need for LD_LIBRARY_PATH
 $(cu_main): $(BUILDDIR)/gcheck_sa.o $(LIBDIR)/lib$(MG5AMC_CULIB).so
 	$(NVCC) -o $@ $(BUILDDIR)/gcheck_sa.o $(CUARCHFLAGS) $(LIBFLAGS) $(CULIBFLAGS) -L$(LIBDIR) -l$(MG5AMC_CULIB)
 endif
@@ -441,7 +462,7 @@ $(BUILDDIR)/fcheck_sa.o : $(INCDIR)/fbridge.inc
 ifeq ($(UNAME_S),Darwin)
 $(fcxx_main): LIBFLAGS += -L$(shell dirname $(shell $(FC) --print-file-name libgfortran.dylib)) # add path to libgfortran on Mac #375
 endif
-$(fcxx_main): LIBFLAGS += -Wl,-rpath,$(LIBDIRRPATH) # avoid the need for LD_LIBRARY_PATH to execute ./runTest.exe
+$(fcxx_main): LIBFLAGS += $(CXXLIBFLAGSRPATH) # avoid the need for LD_LIBRARY_PATH
 $(fcxx_main): $(BUILDDIR)/fcheck_sa.o $(BUILDDIR)/fsampler.o $(LIBDIR)/lib$(MG5AMC_CXXLIB).so
 	$(CXX) -o $@ $(BUILDDIR)/fcheck_sa.o $(BUILDDIR)/fsampler.o $(LIBFLAGS) $(CULIBFLAGS) -lgfortran -L$(LIBDIR) -l$(MG5AMC_CXXLIB)
 
@@ -453,7 +474,7 @@ endif
 ifeq ($(UNAME_S),Darwin)
 $(fcu_main): LIBFLAGS += -L$(shell dirname $(shell $(FC) --print-file-name libgfortran.dylib)) # add path to libgfortran on Mac #375
 endif
-$(fcu_main): LIBFLAGS += -Xlinker -rpath,$(LIBDIRRPATH) # avoid the need for LD_LIBRARY_PATH to execute ./runTest.exe
+$(fcu_main): LIBFLAGS += $(CULIBFLAGSRPATH) # avoid the need for LD_LIBRARY_PATH
 $(fcu_main): $(BUILDDIR)/fcheck_sa.o $(BUILDDIR)/fsampler_cu.o $(LIBDIR)/lib$(MG5AMC_CULIB).so
 	$(NVCC) -o $@ $(BUILDDIR)/fcheck_sa.o $(BUILDDIR)/fsampler_cu.o $(LIBFLAGS) $(CULIBFLAGS) -lgfortran -L$(LIBDIR) -l$(MG5AMC_CULIB)
 endif
@@ -511,11 +532,11 @@ $(testmain): LIBFLAGS += -L$(patsubst %%bin/clang++,%%lib,$(shell which $(CXX) |
 endif
 
 ifeq ($(NVCC),) # link only runTest.o
-$(testmain): LIBFLAGS += -Wl,-rpath,$(LIBDIRRPATH) # avoid the need for LD_LIBRARY_PATH to execute ./runTest.exe
+$(testmain): LIBFLAGS += $(CXXLIBFLAGSRPATH) # avoid the need for LD_LIBRARY_PATH
 $(testmain): $(LIBDIR)/lib$(MG5AMC_COMMONLIB).so $(cxx_objects) $(GTESTLIBS)
 	$(CXX) -o $@ $(cxx_objects) -ldl -pthread $(LIBFLAGS) $(CULIBFLAGS)
 else # link both runTest.o and runTest_cu.o
-$(testmain): LIBFLAGS += -Xlinker -rpath,$(LIBDIRRPATH) # avoid the need for LD_LIBRARY_PATH to execute ./runTest.exe
+$(testmain): LIBFLAGS += $(CULIBFLAGSRPATH) # avoid the need for LD_LIBRARY_PATH
 $(testmain): $(LIBDIR)/lib$(MG5AMC_COMMONLIB).so $(cxx_objects) $(cu_objects) $(GTESTLIBS)
 	$(NVCC) -o $@ $(cxx_objects) $(cu_objects) -ldl $(LIBFLAGS) $(CULIBFLAGS) -lcuda -lgomp
 endif
@@ -548,7 +569,13 @@ avx512z:
 	@echo
 	make USEBUILDDIR=1 AVX=512z
 
+ifeq ($(UNAME_P),ppc64le)
+avxall: $(INCDIR)/fbridge.inc avxnone avxsse4
+else ifeq ($(UNAME_P),arm)
+avxall: $(INCDIR)/fbridge.inc avxnone avxsse4
+else
 avxall: $(INCDIR)/fbridge.inc avxnone avxsse4 avxavx2 avx512y avx512z
+endif
 
 #-------------------------------------------------------------------------------
 
@@ -582,10 +609,17 @@ distclean: cleanall
 info:
 	@echo ""
 	@uname -spn # e.g. Linux nodename.cern.ch x86_64
+ifeq ($(UNAME_S),Darwin)
+	@sysctl -a | grep -i brand
+	@sysctl -a | grep machdep.cpu | grep features || true
+	@sysctl -a | grep hw.physicalcpu:
+	@sysctl -a | grep hw.logicalcpu:
+else
 	@cat /proc/cpuinfo | grep "model name" | sort -u
 	@cat /proc/cpuinfo | grep "flags" | sort -u
 	@cat /proc/cpuinfo | grep "cpu cores" | sort -u
 	@cat /proc/cpuinfo | grep "physical id" | sort -u
+endif
 	@echo ""
 ifneq ($(shell which nvidia-smi 2>/dev/null),)
 	nvidia-smi -L
@@ -615,9 +649,6 @@ endif
 
 #-------------------------------------------------------------------------------
 
-# Note: using rpath makes it unnecessary to prepend DYLD_LIBRARY_PATH="$(LIBDIR):$(DYLD_LIBRARY_PATH)" LD_LIBRARY_PATH="$(LIBDIR):$(LD_LIBRARY_PATH)"
-# [Temporary patch for Mac #375: prepend RUNTIME setting DYLD_LIBRARY_PATH]
-
 # Target: check (run the C++ test executable)
 # [NB THIS IS WHAT IS USED IN THE GITHUB CI!]
 ifneq ($(NVCC),)

epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/Makefile_src

@@ -80,6 +80,12 @@ ifeq ($(UNAME_P),ppc64le)
   else ifneq ($(AVX),none)
     $(error Unknown AVX='$(AVX)': only 'none' and 'sse4' are supported on PowerPC for the moment)
   endif
+else ifeq ($(UNAME_P),arm)
+  ifeq ($(AVX),sse4)
+    override AVXFLAGS = -D__SSE4_2__ # ARM NEON with 128 width (Q/quadword registers)
+  else ifneq ($(AVX),none)
+    $(error Unknown AVX='$(AVX)': only 'none' and 'sse4' are supported on ARM for the moment)
+  endif
 else
   ifeq ($(AVX),sse4)
     override AVXFLAGS = -march=nehalem # SSE4.2 with 128 width (xmm registers)
@@ -143,17 +149,30 @@ override DIRTAG = $(AVX)_$(FPTYPE)_inl$(HELINL)_hrd$(HRDCOD)
 override TAG = $(AVX)_$(FPTYPE)_inl$(HELINL)_hrd$(HRDCOD)_$(RNDGEN)
 
 # Build directory: current directory by default, or build.$(DIRTAG) if USEBUILDDIR==1
+###$(info Current directory is $(shell pwd))
 ifeq ($(USEBUILDDIR),1)
   override BUILDDIR = build.$(DIRTAG)
-  override LIBDIR   = ../lib/$(BUILDDIR)
+  override LIBDIRREL = ../lib/$(BUILDDIR)
   ###$(info Building in BUILDDIR=$(BUILDDIR) for tag=$(TAG) (USEBUILDDIR=1 is set))
 else
   override BUILDDIR = .
-  override LIBDIR   = ../lib
+  override LIBDIRREL = ../lib
   ###$(info Building in BUILDDIR=$(BUILDDIR) for tag=$(TAG) (USEBUILDDIR is not set))
 endif
 ######$(info Building in BUILDDIR=$(BUILDDIR) for tag=$(TAG))
 
+# Workaround for Mac #375 (I did not manage to fix rpath with @executable_path): use absolute paths for LIBDIR
+# (NB: this is quite ugly because it creates the directory if it does not exist - to avoid removing src by mistake)
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Darwin)
+override LIBDIR = $(shell mkdir -p $(LIBDIRREL); cd $(LIBDIRREL); pwd)
+ifeq ($(wildcard $(LIBDIR)),)
+$(error Directory LIBDIR="$(LIBDIR)" should have been created by now)
+endif
+else
+override LIBDIR = $(LIBDIRREL)
+endif
+
 #===============================================================================
 #=== Makefile TARGETS and build rules below
 #===============================================================================

epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/MatrixElementKernels.cc

@@ -67,30 +67,44 @@ namespace mg5amcCpu
   bool MatrixElementKernelHost::hostSupportsSIMD( const bool verbose )
   {
 #if defined __AVX512VL__
+    bool known = true;
     bool ok = __builtin_cpu_supports( "avx512vl" );
     const std::string tag = "skylake-avx512 (AVX512VL)";
 #elif defined __AVX2__
+    bool known = true;
     bool ok = __builtin_cpu_supports( "avx2" );
     const std::string tag = "haswell (AVX2)";
 #elif defined __SSE4_2__
 #ifdef __PPC__
     // See https://gcc.gnu.org/onlinedocs/gcc/Basic-PowerPC-Built-in-Functions-Available-on-all-Configurations.html
+    bool known = true;
     bool ok = __builtin_cpu_supports( "vsx" );
     const std::string tag = "powerpc vsx (128bit as in SSE4.2)";
+#elif defined __ARM_NEON__ // consider using __BUILTIN_CPU_SUPPORTS__
+    bool known = false; // __builtin_cpu_supports is not supported
+    // See https://gcc.gnu.org/onlinedocs/gcc/Basic-PowerPC-Built-in-Functions-Available-on-all-Configurations.html
+    // See https://stackoverflow.com/q/62783908
+    // See https://community.arm.com/arm-community-blogs/b/operating-systems-blog/posts/runtime-detection-of-cpu-features-on-an-armv8-a-cpu
+    bool ok = true; // this is just an assumption!
+    const std::string tag = "arm neon (128bit as in SSE4.2)";
 #else
+    bool known = true;
     bool ok = __builtin_cpu_supports( "sse4.2" );
     const std::string tag = "nehalem (SSE4.2)";
 #endif
 #else
+    bool known = true;
     bool ok = true;
     const std::string tag = "none";
 #endif
     if( verbose )
     {
       if( tag == "none" )
         std::cout << "INFO: The application does not require the host to support any AVX feature" << std::endl;
-      else if( ok )
+      else if( ok && known )
         std::cout << "INFO: The application is built for " << tag << " and the host supports it" << std::endl;
+      else if( ok )
+        std::cout << "WARNING: The application is built for " << tag << " but it is unknown if the host supports it" << std::endl;
       else
         std::cout << "ERROR! The application is built for " << tag << " but the host does not support it" << std::endl;
     }

epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/check_sa.cc

@@ -763,6 +763,8 @@ main( int argc, char** argv )
 #elif defined __SSE4_2__
 #ifdef __PPC__
   wrkflwtxt += "/ppcv";
+#elif defined __ARM_NEON__
+  wrkflwtxt += "/neon";
 #else
   wrkflwtxt += "/sse4";
 #endif
@@ -792,13 +794,15 @@ main( int argc, char** argv )
   if( perf )
   {
 #ifndef __CUDACC__
+#ifdef _OPENMP
     // Get the output of "nproc --all" (https://stackoverflow.com/a/478960)
     std::string nprocall;
     std::unique_ptr<FILE, decltype( &pclose )> nprocpipe( popen( "nproc --all", "r" ), pclose );
     if( !nprocpipe ) throw std::runtime_error( "`nproc --all` failed?" );
     std::array<char, 128> nprocbuf;
     while( fgets( nprocbuf.data(), nprocbuf.size(), nprocpipe.get() ) != nullptr ) nprocall += nprocbuf.data();
 #endif
+#endif
 #ifdef MGONGPU_CPPSIMD
 #ifdef MGONGPU_HAS_CPPCXTYPEV_BRK
     const std::string cxtref = " [cxtype_ref=YES]";
@@ -869,6 +873,8 @@ main( int argc, char** argv )
               << "Internal loops fptype_sv    = VECTOR[" << neppV
 #ifdef __PPC__
               << "] ('sse4': PPC VSX, 128bit)" << cxtref << std::endl
+#elif defined __ARM_NEON__
+              << "] ('sse4': ARM NEON, 128bit)" << cxtref << std::endl
 #else
               << "] ('sse4': SSE4.2, 128bit)" << cxtref << std::endl
 #endif

epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/mgOnGpuConfig.h

@@ -143,7 +143,7 @@ using mgOnGpu::fptype;
 #else
 #define MGONGPU_CPPSIMD 8
 #endif
-#elif defined __SSE4_2__ // C++ "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT)
+#elif defined __SSE4_2__ // C++ "sse4" SSE4.2 (128-bit ie 16-byte): 2 (DOUBLE) or 4 (FLOAT) [Power9 and ARM default]
 #ifdef MGONGPU_FPTYPE_DOUBLE
 #define MGONGPU_CPPSIMD 2
 #else

epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/process_function_definitions.inc

@@ -153,6 +153,9 @@ namespace mg5amcCpu
     // CLANG version (either as CXX or as host compiler inside NVCC or inside ICX)
 #if defined __clang__
 #if defined __clang_major__ && defined __clang_minor__ && defined __clang_patchlevel__
+#ifdef __APPLE__
+    out << "Apple clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__;
+#else
     out << "clang " << __clang_major__ << "." << __clang_minor__ << "." << __clang_patchlevel__;
     // GCC toolchain version inside CLANG
     std::string tchainout;
@@ -167,6 +170,7 @@ namespace mg5amcCpu
 #else
     out << " (gcc " << tchainout << ")";
 #endif
+#endif
 #else
     out << "clang UNKNOWKN";
 #endif

epochX/cudacpp/ee_mumu.auto/CODEGEN_cudacpp_ee_mumu_log.txt

@@ -50,7 +50,7 @@ generate e+ e- > mu+ mu-
 No model currently active, so we import the Standard Model
 INFO: load particles 
 INFO: load vertices 
-DEBUG: model prefixing  takes 0.0069081783294677734 
+DEBUG: model prefixing  takes 0.00741124153137207 
 INFO: Restrict model sm with file models/sm/restrict_default.dat . 
 INFO: Change particles name to pass to MG5 convention 
 Defined multiparticle p = g u c d s u~ c~ d~ s~
@@ -101,7 +101,7 @@ ALOHA: aloha creates FFV1 routines
 ALOHA: aloha creates FFV2 routines
 ALOHA: aloha creates FFV4 routines
 ALOHA: aloha creates FFV2_4 routines
-ALOHA: aloha creates 4 routines in  0.354 s
+ALOHA: aloha creates 4 routines in  0.320 s
 <class 'aloha.create_aloha.AbstractRoutine'> FFV1
 DEBUG:  language =  <class 'PLUGIN.CUDACPP_SA_OUTPUT.model_handling.PLUGIN_ALOHAWriter'> [aloha_writers.py at line 2451] 
 <class 'aloha.create_aloha.AbstractRoutine'> FFV1
@@ -127,6 +127,6 @@ INFO: /data/avalassi/GPU2020/MG5aMC/3.1.1_lo_vectorization/CODEGEN_cudacpp_ee_mu
 DEBUG:  Entering PLUGIN_ProcessExporter.finalize [output.py at line 212] 
 quit
 
-real	0m1.095s
-user	0m0.915s
-sys	0m0.119s
+real	0m1.068s
+user	0m0.918s
+sys	0m0.121s