Merge pull request fmihpc#339 from ursg/master

Makefile reordering, marconi makefile and "DO LB" command.

MAKE/Makefile.marconi

@@ -0,0 +1,84 @@
+#MARCONI - KNL based cluster
+#
+#The following module commands should give you the correct environment for this makefile
+#module purge
+#module load env-knl
+#module load profile/advanced
+#module load intel
+#module load intelmpi
+#module load boost
+#module swap intelmpi mvapich2/2.2--intel--pe-xe-2017--binary
+
+
+#======== Vectorization ==========
+#Set vector backend type for vlasov solvers, sets precision and length. 
+#NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION)
+#Options: 
+# AVX:	    VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER
+# AVX512:   VEC8D_AGNER, VEC16F_AGNER
+# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK
+
+ifeq ($(DISTRIBUTION_FP_PRECISION),SPF)
+#Single-precision        
+	VECTORCLASS = VEC16F_AGNER
+else
+#Double-precision
+	VECTORCLASS = VEC8D_AGNER
+endif
+
+#======== PAPI ==========
+#Add PAPI_MEM define to use papi to report memory consumption?
+CXXFLAGS +=  -DPAPI_MEM
+
+
+#======== Allocator =========
+#Use TBB malloc
+LDFLAGS += -L$(TBBROOT)/lib/intel64/gcc4.7/ -ltbbmalloc_proxy -ltbbmalloc
+
+
+#======= Compiler and compilation flags =========
+# NOTES on compiler flags:
+# CXXFLAGS is for compiler flags, they are always used
+# MATHFLAGS are for special math etc. flags, these are only applied on solver functions
+# LDFLAGS flags for linker
+#GNU flags:
+CC_BRAND = intel
+CC_BRAND_VERSION = 17.0.1
+CXXFLAGS +=  -O2 -g -DMAX_VECTOR_SIZE=512 -xMIC-AVX512 -std=c++11 -qopenmp -ansi-alias
+testpackage: CXXFLAGS = -O2 -g -DMAX_VECTOR_SIZE=512 -xMIC-AVX512 -std=c++11 -qopenmp -ansi-alias
+MATHFLAGS = 
+LDFLAGS += -qopenmp -lifcore 
+CMP = mpicxx
+LNK = mpicxx
+
+
+#======== Libraries ===========
+
+HEADER_LIBRARY_PREFIX = /marconi_work/Pra14_3521/libraries
+LIBRARY_PREFIX = /marconi_work/Pra14_3521/libraries
+
+
+INC_BOOST = -I$(BOOST_INC)
+LIB_BOOST = -L$(BOOST_LIB) -lboost_program_options
+
+INC_ZOLTAN = -I$(LIBRARY_PREFIX)/zoltan/include
+LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/zoltan/lib -lzoltan
+
+INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv
+LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv
+
+
+LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof
+INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include
+
+#system
+INC_PAPI = -I$(LIBRARY_PREFIX)/papi/include
+LIB_PAPI = -L$(LIBRARY_PREFIX)/papi/lib -lpapi -lpfm
+
+#header libraries
+
+INC_EIGEN = -I$(HEADER_LIBRARY_PREFIX)/eigen
+INC_DCCRG = -I$(HEADER_LIBRARY_PREFIX)/dccrg/
+INC_VECTORCLASS = -I$(HEADER_LIBRARY_PREFIX)/vectorclass
+
+

MAKE/Makefile.sisu_gcc

@@ -1,6 +1,39 @@
 CMP = CC
 LNK = CC
 
+#======== Vectorization ==========
+#Set vector backend type for vlasov solvers, sets precision and length. 
+#NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION)
+#Options: 
+# AVX:	    VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER
+# AVX512:   VEC8D_AGNER, VEC16F_AGNER
+# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK
+
+ifeq ($(DISTRIBUTION_FP_PRECISION),SPF)
+#Single-precision        
+	VECTORCLASS = VEC8F_AGNER
+else
+#Double-precision
+	VECTORCLASS = VEC4D_AGNER
+endif
+
+#======== PAPI ==========
+#Add PAPI_MEM define to use papi to report memory consumption?
+CXXFLAGS +=  -DPAPI_MEM
+
+
+#======== Allocator =========
+#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc
+#Configure jemalloc with  --with-jemalloc-prefix=je_ when installing it
+CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
+
+
+#======= Compiler and compilation flags =========
+# NOTES on compiler flags:
+# CXXFLAGS is for compiler flags, they are always used
+# MATHFLAGS are for special math etc. flags, these are only applied on solver functions
+# LDFLAGS flags for linker
+
 #-DNO_WRITE_AT_ALL:  Define to disable write at all to 
 #                    avoid memleak (much slower IO)
 #-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition 
@@ -11,11 +44,6 @@ CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK
 
 FLAGS = 
 
-# NOTES on compiler flags:
-# CXXFLAGS is for compiler flags, they are always used
-# MATHFLAGS are for special math etc. flags, these are only applied on solver functions
-# LDFLAGS flags for linker
-
 #GNU flags:
 CC_BRAND = gcc
 CC_BRAND_VERSION = 5.1.0
@@ -29,6 +57,8 @@ LIB_MPI = -lgomp
 # BOOST_VERSION = current trilinos version
 # ZOLTAN_VERSION = current trilinos verson
 
+#======== Libraries ===========
+
 MPT_VERSION = 7.2.6
 JEMALLOC_VERSION = 4.0.4
 LIBRARY_PREFIX = /proj/vlasiato/libraries/

MAKE/Makefile.taito_gcc

@@ -1,6 +1,39 @@
 CMP = mpiCC
 LNK = mpiCC
 
+#======== Vectorization ==========
+#Set vector backend type for vlasov solvers, sets precision and length. 
+#NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION)
+#Options: 
+# AVX:	    VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER
+# AVX512:   VEC8D_AGNER, VEC16F_AGNER
+# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK
+
+ifeq ($(DISTRIBUTION_FP_PRECISION),SPF)
+#Single-precision        
+	VECTORCLASS = VEC8F_AGNER
+else
+#Double-precision
+	VECTORCLASS = VEC4D_AGNER
+endif
+
+#======== PAPI ==========
+#Add PAPI_MEM define to use papi to report memory consumption?
+CXXFLAGS +=  -DPAPI_MEM
+
+
+#======== Allocator =========
+#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc
+#Configure jemalloc with  --with-jemalloc-prefix=je_ when installing it
+CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
+
+
+#======= Compiler and compilation flags =========
+# NOTES on compiler flags:
+# CXXFLAGS is for compiler flags, they are always used
+# MATHFLAGS are for special math etc. flags, these are only applied on solver functions
+# LDFLAGS flags for linker
+
 #-DNO_WRITE_AT_ALL:  Define to disable write at all to 
 #                    avoid memleak (much slower IO)
 #-DMPICH_IGNORE_CXX_SEEK: Ignores some multiple definition 
@@ -11,11 +44,6 @@ CXXFLAGS = -DMPICH_IGNORE_CXX_SEEK
 
 FLAGS = 
 
-# NOTES on compiler flags:
-# CXXFLAGS is for compiler flags, they are always used
-# MATHFLAGS are for special math etc. flags, these are only applied on solver functions
-# LDFLAGS flags for linker
-
 #GNU flags:
 CC_BRAND = gcc
 CC_BRAND_VERSION = 4.9.3
@@ -28,6 +56,8 @@ LIB_MPI = -lgomp
 
 # BOOST_VERSION = current trilinos version
 # ZOLTAN_VERSION = current trilinos verson
+#
+#======== Libraries ===========
 
 MPT_VERSION = 1.10.2
 JEMALLOC_VERSION = 4.0.4

Makefile

@@ -1,22 +1,23 @@
 #set default architecture, can be overridden from the compile line
 ARCH = ${VLASIATOR_ARCH}
-include MAKE/Makefile.${ARCH}
-
 #set FP precision to SP (single) or DP (double)
 FP_PRECISION = DP
-
 #Set floating point precision for distribution function to SPF (single) or DPF (double)
-#NOTE this has to have the same precision as the vectorclass defined below (VECTORCLASS)
 DISTRIBUTION_FP_PRECISION = SPF
+#override flags if we are building testpackage:
+
+ifneq (,$(findstring testpackage,$(MAKECMDGOALS)))
+	MATHFLAGS =
+	FP_PRECISION = DP
+	DISTRIBUTION_FP_PRECISION = DPF
+endif
+
+
+include MAKE/Makefile.${ARCH}
+
+
+
 
-#Set vector backend type for vlasov solvers, sets precision and length. 
-#NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION).
-#Options: 
-# Agners vectorclass: 
-# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER
-# AVX512: VEC8D_AGNER, VEC16F_AGNER
-# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK
-VECTORCLASS = VEC8F_AGNER
 
 #set a default archive utility, can also be set in Makefile.arch
 AR ?= ar
@@ -27,19 +28,7 @@ FIELDSOLVER ?= londrillo_delzanna
 # COMPFLAGS += -DFS_1ST_ORDER_SPACE
 # COMPFLAGS += -DFS_1ST_ORDER_TIME
 
-#override flags if we are building testpackage:
-testpackage: MATHFLAGS =
-testpackage: FP_PRECISION = DP
-testpackage: DISTRIBUTION_FP_PRECISION = DPF
-testpackage: VECTORCLASS = VEC4D_AGNER
-
-#also use papi to report memory consumption?
-PAPI_FLAG ?= -DPAPI_MEM
-COMPFLAGS +=${PAPI_FLAG}
 
-#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc
-#Configure jemalloc with  --with-jemalloc-prefix=je_ when installing it
-COMPFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
 
 #is profiling on?
 COMPFLAGS += -DPROFILE
@@ -66,10 +55,6 @@ COMPFLAGS += -DACC_SEMILAG_PQM -DTRANS_SEMILAG_PPM
 #Define MESH=AMR if you want to use adaptive mesh refinement in velocity space
 #MESH = AMR
 
-#Add -DFS_1ST_ORDER_SPACE or -DFS_1ST_ORDER_TIME to make the field solver first-order in space or time
-# COMPFLAGS += -DFS_1ST_ORDER_SPACE
-# COMPFLAGS += -DFS_1ST_ORDER_TIME
-
 #//////////////////////////////////////////////////////
 # The rest of this file users shouldn't need to change
 #//////////////////////////////////////////////////////

common.h

@@ -371,6 +371,7 @@ const uint SIZE_VELBLOCK    = WID3; /*!< Number of cells in a velocity block. */
 struct globalflags {
    static int bailingOut; /*!< Global flag raised to true if a run bailout (write restart if requested/set and stop the simulation peacefully) is needed. */
    static bool writeRestart; /*!< Global flag raised to true if a restart writing is needed (without bailout). NOTE: used only by MASTER_RANK in vlasiator.cpp. */
+   static bool balanceLoad; /*!< Global flag raised to true if a load balancing is needed. NOTE: used only by MASTER_RANK in vlasiator.cpp. */
 };
 
 /*!

ioread.cpp

@@ -51,6 +51,7 @@ typedef Parameters P;
  * If a file STOP was written and is readable, then a bailout with restart writing is initiated.
  * If a file KILL was written and is readable, then a bailout without a restart is initiated.
  * If a file SAVE was written and is readable, then restart writing without a bailout is initiated.
+ * If a file DOLB was written and is readable, then a new load balancing is initiated.
  * To avoid bailing out upfront on a new run the files are renamed with the date to keep a trace.
  * The function should only be called by MASTER_RANK. This ensures that resetting P::bailout_write_restart works.
  */
@@ -89,6 +90,17 @@ void checkExternalCommands() {
       rename("SAVE", newName);
       return;
    }
+   if(stat("DOLB", &tempStat) == 0) {
+      cerr << "Received an external DOLB command. Balancing load." << endl;
+      globalflags::balanceLoad = true;
+      char newName[80];
+      // Get the current time.
+      const time_t rawTime = time(NULL);
+      const struct tm * timeInfo = localtime(&rawTime);
+      strftime(newName, 80, "DOLB_%F_%H-%M-%S", timeInfo);
+      rename("DOLB", newName);
+      return;
+   }
 }
 
 /*!

tools/vlsv_util.cpp

@@ -27,33 +27,36 @@
 
 #include <cstdlib>
 #include <iostream>
-#include <dirent.h>
+#include <glob.h>
 
 #include "vlsv_util.h"
 
 using namespace std;
 
 std::vector<std::string> toolutil::getFiles(const std::string& mask) {
    vector<string> fileList;
-   const string directory = ".";
-   const string suffix = ".vlsv";
-   DIR* dir = opendir(directory.c_str());
-   if (dir == NULL) {
-      cerr << "ERROR in reading directory contents in " << __FILE__ << ":" << __LINE__ << endl;
-      closedir(dir);
-      return fileList;
-   }
 
-   struct dirent* entry = readdir(dir);
-   while (entry != NULL) {
-      const string entryName = entry->d_name;
-      if (entryName.find(mask) == string::npos || entryName.find(suffix) == string::npos) {
-         entry = readdir(dir);
-         continue;
+   glob_t glob_matches;
+   int retval = glob(mask.c_str(), GLOB_TILDE_CHECK, NULL, &glob_matches);
+   if(retval != 0) {
+      switch(retval) {
+         case GLOB_NOSPACE:
+            cerr << "ERROR enumerating filenames: out of memory in " << __FILE__ << ":" << __LINE__ << endl;
+            break;
+         case GLOB_ABORTED:
+            cerr << "ERROR in reading directory contents in " << __FILE__ << ":" << __LINE__ << endl;
+            break;
+         case GLOB_NOMATCH:
+            cerr << "ERROR: no matching file found for pattern " << mask << endl;
+            break;
       }
-      fileList.push_back(entryName);
-      entry = readdir(dir);
+      globfree(&glob_matches);
+      return fileList;
    }
-   closedir(dir);
+   for(unsigned int i=0; i<glob_matches.gl_pathc; i++) {
+      fileList.push_back(glob_matches.gl_pathv[i]);
+   }
+
+   globfree(&glob_matches);
    return fileList;
 }

tools/vlsvdiff.cpp

@@ -305,6 +305,7 @@ bool convertSILO(const string fileName,
 
    if (vlsvReader.open(fileName) == false) {
       cerr << "Failed to open '" << fileName << "'" << endl;
+      cerr << "VLSV error " << vlsvReader.getErrorString() << endl;
       return false;
    }