Merge branch 'dev' into makefileCleanup2

MAKE/Makefile.kstppd

@@ -1,8 +1,13 @@
-#Kostis Papadakis
+#Kostis Laptop
+#Compiler & Linker
+CMP = mpic++
+LNK = mpic++
+
+
 #======== Vectorization ==========
 #Set vector backend type for vlasov solvers, sets precision and length.
 #NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION)
-#Options:
+#Options;
 # AVX:	    VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER
 # AVX512:   VEC8D_AGNER, VEC16F_AGNER
 # Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK
@@ -15,19 +20,33 @@ else
 	VECTORCLASS = VEC4D_AGNER
 endif
 
-CMP = mpic++
-LNK = mpic++
 
-FLAGS =
 
-#note: std was c++0x
-#note: no testpackage flags
-#CXXFLAGS = -I $(HOME)/include  -L $(HOME)/lib -g  -funroll-loops -std=c++17 -fopenmp -W -Wall -pedantic -Wno-unused -fabi-version=0 -mavx
-CXXFLAGS = -I $(HOME)/include  -L $(HOME)/lib -O3  -funroll-loops -std=c++17 -fopenmp -W -Wall -Wno-unused -fabi-version=0 -mavx
+#======== Flags =========
+
+FLAGS =
+CXXFLAGS = -O3   -funroll-loops -std=c++17 -fopenmp -W -Wall -Wno-unused -Wno-unused-parameter -Wno-missing-braces -fabi-version=0 -mavx
+testpackage: CXXFLAGS = -g -ggdb -O2 -fopenmp -funroll-loops -std=c++17 -fabi-version=0  -mno-avx -mno-fma -fno-unsafe-math-optimizations
+CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE -DPAPI_MEM
 MATHFLAGS = -ffast-math
+testpackage: MATHFLAGS =  -fno-unsafe-math-optimizations
 LDFLAGS = -L $(HOME)/lib
-LIB_MPI = -lgomp
+LIB_MPI = -lgomp -lpapi 
+
+
+
 
+#================PAPI==================
+#Add PAPI_MEM define to use papi to report memory consumption?
+CXXFLAGS +=  -DPAPI_MEM
+testpackage: CXXFLAGS +=  -DPAPI_MEM
+
+
+#======== Allocator =========
+#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc
+#Configure jemalloc with  --with-jemalloc-prefix=je_ when installing it
+CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
+testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
 
 #======== Allocator =========
 #Use TBB malloc
@@ -58,15 +77,14 @@ INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg
 
 INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid
 
-#INC_JEMALLOC = -I$(LIBRARY_PREFIX)/include/jemalloc
-#LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/lib -ljemalloc
+INC_PAPI = -I$(LIBRARY_PREFIX)/papi-5.4.3/src
+LIB_PAPI = -L$(LIBRARY_PREFIX)/papi-5.4.3/src -lpapi
 
-INC_JEMALLOC = -I/home/kostis/libraries/jemalloc
-LIB_JEMALLOC = -L/home/kostis/libraries/jemalloc/lib -ljemalloc
+INC_JEMALLOC = -I$(LIBRARY_PREFIX)/libraries/jemalloc
+LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/libraries/jemalloc/lib -ljemalloc
 
 LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof
 INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include
-INC_TOPO =
 
 INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen-eigen-07105f7124f9
 

iowrite.cpp

@@ -901,7 +901,7 @@ bool writeFsGridMetadata(FsGrid< fsgrids::technical, FS_STENCIL_WIDTH> & technic
 
 
   // writeDomainSizes
-  std::array<uint32_t,2> meshDomainSize({globalIds.size(), 0});
+  std::array<uint64_t,2> meshDomainSize({globalIds.size(), 0});
   vlsvWriter.writeArray("MESH_DOMAIN_SIZES", xmlAttributes, 1, 2, &meshDomainSize[0]);
 
   // how many MPI ranks we wrote from

item_storage.h

@@ -24,6 +24,7 @@
 #define ITEM_STORAGE_H
 
 #include <map>
+#include <string>
 
 #include "definitions.h"
 

mesh_data.h

@@ -29,6 +29,7 @@
 #define MESH_DATA_H
 
 #include <cstdlib>
+#include <string>
 
 namespace mesh {
 

sysboundary/sysboundary.cpp

@@ -693,7 +693,7 @@ void SysBoundary::applySysBoundaryVlasovConditions(
 
       timer=phiprof::initializeTimer("Wait for receives","MPI","Wait");
       phiprof::start(timer);
-      mpiGrid.wait_remote_neighbor_copy_update_receives(SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID);
+      mpiGrid.wait_remote_neighbor_copy_updates(SYSBOUNDARIES_EXTENDED_NEIGHBORHOOD_ID);
       phiprof::stop(timer);
 
       // Compute vlasov boundary on system boundary/process boundary cells
@@ -713,11 +713,6 @@ void SysBoundary::applySysBoundaryVlasovConditions(
       }
       phiprof::stop(timer);
 
-      timer=phiprof::initializeTimer("Wait for sends","MPI","Wait");
-      phiprof::start(timer);
-      mpiGrid.wait_remote_neighbor_copy_update_sends();
-      phiprof::stop(timer);
-
       // WARNING Blocks are changed but lists not updated now, if you need to use/communicate them before the next update is done, add an update here.
       // reset lists in smaller default neighborhood
       updateRemoteVelocityBlockLists(mpiGrid, popID);

tools/vlsvdiff.cpp

@@ -256,7 +256,7 @@ bool HandleFsGrid(const string& inputFileName,
    patch["vectorsize"]=to_string(vectorsize2);
 
    //Override MESH_DOMAIN_SIZES
-   std::array<uint32_t,2> meshDomainSize({globalIds.size(), 0});
+   std::array<uint64_t,2> meshDomainSize({globalIds.size(), 0});
    output.writeArray("MESH_DOMAIN_SIZES",patch ,1,vectorsize2, &meshDomainSize[0]);
 
 
@@ -421,18 +421,18 @@ bool convertMesh(vlsvinterface::Reader& vlsvReader,
       cerr << "ERROR, failed to get array info for '" << _varToExtract << "' at " << __FILE__ << " " << __LINE__ << endl;
       return false;
    }
+      char *variableBuffer = new char[variableVectorSize * variableDataSize];
+      float *variablePtrFloat = reinterpret_cast<float *>(variableBuffer);
+      double *variablePtrDouble = reinterpret_cast<double *>(variableBuffer);
+      uint *variablePtrUint = reinterpret_cast<uint *>(variableBuffer);
+      int *variablePtrInt = reinterpret_cast<int *>(variableBuffer);
 
 
    if (gridName==gridType::SpatialGrid){
 
       // Read the mesh array one node (of a spatial cell) at a time
       // and create a map which contains each cell's CellID and variable to be extracted
-      char *variableBuffer = new char[variableVectorSize * variableDataSize];
-      float *variablePtrFloat = reinterpret_cast<float *>(variableBuffer);
-      double *variablePtrDouble = reinterpret_cast<double *>(variableBuffer);
-      uint *variablePtrUint = reinterpret_cast<uint *>(variableBuffer);
-      int *variablePtrInt = reinterpret_cast<int *>(variableBuffer);
-      //Get local cell ids:
+         //Get local cell ids:
       vector<uint64_t> local_cells;
       if ( vlsvReader.getCellIds( local_cells, meshName) == false ) {
          cerr << "Failed to read cell ids at "  << __FILE__ << " " << __LINE__ << endl;
@@ -509,7 +509,7 @@ bool convertMesh(vlsvinterface::Reader& vlsvReader,
       std::array<int32_t,3> taskSize,taskStart;
       std::array<int32_t,3> taskEnd;
       int readOffset=0;
-      int readSize;
+      size_t readSize;
       int index,my_x,my_y,my_z;
       orderedData->clear();
 
@@ -536,11 +536,6 @@ bool convertMesh(vlsvinterface::Reader& vlsvReader,
          readSize= taskSize[0] * taskSize[1] * taskSize[2];
          std::vector<Real> readIn(variableVectorSize * variableDataSize*readSize);
 
-         if (vlsvReader.readArray("VARIABLE", variableAttributes, readOffset,readSize, (char*)readIn.data()) == false){
-               cerr << "ERROR, failed to read variable '" << _varToExtract << "' at " << __FILE__ << " " << __LINE__ << endl;
-               variableSuccess = false;
-               break;
-            }
 
          int counter2=0;
          uint64_t globalindex;
@@ -551,20 +546,39 @@ bool convertMesh(vlsvinterface::Reader& vlsvReader,
 
                   //Get global index
                   globalindex= x + y*xcells + z*xcells*ycells;
-                  Real extract;
-                  extract = (Real) readIn[counter+compToExtract];
 
+                  if (vlsvReader.readArray("VARIABLE", variableAttributes, readOffset+counter,1, variableBuffer) == false) {
+                     cerr << "ERROR, failed to read variable '" << _varToExtract << "' at " << __FILE__ << " " << __LINE__ << endl;
+                     variableSuccess = false; 
+                     abort();
+                     break;
+                  }
+
+                  // Get the variable value
+                  Real extract = NAN;
+
+                  switch (variableDataType) {
+                     case datatype::type::FLOAT:
+                        if(variableDataSize == sizeof(float)) extract = (Real)(variablePtrFloat[compToExtract]);
+                        if(variableDataSize == sizeof(double)) extract = (Real)(variablePtrDouble[compToExtract]);
+                        break;
+                     case datatype::type::UINT:
+                        extract = (Real)(variablePtrUint[compToExtract]);
+                        break;
+                     case datatype::type::INT:
+                        extract = (Real)(variablePtrInt[compToExtract]);
+                        break;
+                     case datatype::type::UNKNOWN:
+                        cerr << "ERROR, BAD DATATYPE AT " << __FILE__ << " " << __LINE__ << endl;
+                        break;
+                  }
                   orderedData->insert(pair<uint64_t, Real>(globalindex, extract));
-                  counter+=variableVectorSize;
+                  counter++;
 
                }
             }
          }
-
-
          readOffset+=readSize;
-
-
       }
    }else{
     cerr<<"meshName not recognized\t" << __FILE__ << " " << __LINE__ <<endl;
@@ -577,6 +591,7 @@ bool convertMesh(vlsvinterface::Reader& vlsvReader,
    if (variableSuccess == false) {
       cerr << "ERROR reading array VARIABLE " << varToExtract << endl;
    }
+   delete variableBuffer;
    return meshSuccess && variableSuccess;
 }
 
@@ -1531,11 +1546,15 @@ bool process2Files(const string fileName1,
          }
 
 
+         map<string,string>::const_iterator it = attributes.find("--meshname");
+         if (cloneMesh(fileName1,outputFile,it->second,orderedData1) == false) {
+            std::cerr<<"Failed"<<std::endl;
+            return false;
+         }
+      }
+
       singleStatistics(&orderedData1, &size, &mini, &maxi, &avg, &stdev); //CONTINUE
       // Clone mesh from input file to diff file
-      map<string,string>::const_iterator it = attributes.find("--meshname");
-         if (cloneMesh(fileName1,outputFile,it->second,orderedData1) == false) return false;
-      }
       outputStats(&size, &mini, &maxi, &avg, &stdev, verboseOutput, false);
 
       singleStatistics(&orderedData2, &size, &mini, &maxi, &avg, &stdev);