Skip to content

Commit

Permalink
Merge pull request fmihpc#541 from kstppd/dev
Browse files Browse the repository at this point in the history 
vlsvdiff fix
  • Loading branch information
@markusbattarbee
markusbattarbee authored Mar 2, 2021
2 parents 6e60690 + 4e090c7 commit 396d3fd
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 54 deletions.
62 changes: 30 additions & 32 deletions MAKE/Makefile.kstppd
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
#Kostis Papadakis
#Kostis Laptop
#Compiler & Linker
CMP = mpic++
LNK = mpic++


#======== Vectorization ==========
#Set vector backend type for vlasov solvers, sets precision and length.
#NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION)
#Options:
#Options;
# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER
# AVX512: VEC8D_AGNER, VEC16F_AGNER
# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK
Expand All @@ -16,37 +21,31 @@ else
endif


#======== Allocator =========
#Use TBB malloc
#======== Flags =========

CMP = mpic++
LNK = mpic++
FLAGS =
CXXFLAGS = -O3 -funroll-loops -std=c++17 -fopenmp -W -Wall -Wno-unused -Wno-unused-parameter -Wno-missing-braces -fabi-version=0 -mavx
testpackage: CXXFLAGS = -g -ggdb -O2 -fopenmp -funroll-loops -std=c++17 -fabi-version=0 -mno-avx -mno-fma -fno-unsafe-math-optimizations
CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE -DPAPI_MEM
MATHFLAGS = -ffast-math
testpackage: MATHFLAGS = -fno-unsafe-math-optimizations
LDFLAGS = -L $(HOME)/lib
LIB_MPI = -lgomp -lpapi

PAPI_FLAG =

#======== Vectorization ==========
#Set vector backend type for vlasov solvers, sets precision and length.
#NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION)
#Options:
# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER
# AVX512: VEC8D_AGNER, VEC16F_AGNER
# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK

ifeq ($(DISTRIBUTION_FP_PRECISION),SPF)
#Single-precision
VECTORCLASS = VEC8F_AGNER
else
#Double-precision
VECTORCLASS = VEC4D_AGNER
endif

FLAGS =
#CXXFLAGS = -I $(HOME)/include -L $(HOME)/lib -g -funroll-loops -std=c++0x -fopenmp -W -Wall -pedantic -Wno-unused -fabi-version=0 -mavx
CXXFLAGS = -I $(HOME)/include -L $(HOME)/lib -O3 -funroll-loops -std=c++0x -fopenmp -W -Wall -Wno-unused -fabi-version=0 -mavx
MATHFLAGS = -ffast-math
LDFLAGS = -L $(HOME)/lib
LIB_MPI = -lgomp
#================PAPI==================
#Add PAPI_MEM define to use papi to report memory consumption?
CXXFLAGS += -DPAPI_MEM
testpackage: CXXFLAGS += -DPAPI_MEM


#======== Allocator =========
#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc
#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it
CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE
testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE

#======== Libraries ===========

Expand All @@ -70,15 +69,14 @@ INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg

INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid

#INC_JEMALLOC = -I$(LIBRARY_PREFIX)/include/jemalloc
#LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/lib -ljemalloc
INC_PAPI = -I$(LIBRARY_PREFIX)/papi-5.4.3/src
LIB_PAPI = -L$(LIBRARY_PREFIX)/papi-5.4.3/src -lpapi

INC_JEMALLOC = -I/home/kostis/libraries/jemalloc
LIB_JEMALLOC = -L/home/kostis/libraries/jemalloc/lib -ljemalloc
INC_JEMALLOC = -I$(LIBRARY_PREFIX)/libraries/jemalloc
LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/libraries/jemalloc/lib -ljemalloc

LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof
INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include
INC_TOPO =

INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen-eigen-07105f7124f9

63 changes: 41 additions & 22 deletions tools/vlsvdiff.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,18 +421,18 @@ bool convertMesh(vlsvinterface::Reader& vlsvReader,
cerr << "ERROR, failed to get array info for '" << _varToExtract << "' at " << __FILE__ << " " << __LINE__ << endl;
return false;
}
char *variableBuffer = new char[variableVectorSize * variableDataSize];
float *variablePtrFloat = reinterpret_cast<float *>(variableBuffer);
double *variablePtrDouble = reinterpret_cast<double *>(variableBuffer);
uint *variablePtrUint = reinterpret_cast<uint *>(variableBuffer);
int *variablePtrInt = reinterpret_cast<int *>(variableBuffer);


if (gridName==gridType::SpatialGrid){

// Read the mesh array one node (of a spatial cell) at a time
// and create a map which contains each cell's CellID and variable to be extracted
char *variableBuffer = new char[variableVectorSize * variableDataSize];
float *variablePtrFloat = reinterpret_cast<float *>(variableBuffer);
double *variablePtrDouble = reinterpret_cast<double *>(variableBuffer);
uint *variablePtrUint = reinterpret_cast<uint *>(variableBuffer);
int *variablePtrInt = reinterpret_cast<int *>(variableBuffer);
//Get local cell ids:
//Get local cell ids:
vector<uint64_t> local_cells;
if ( vlsvReader.getCellIds( local_cells, meshName) == false ) {
cerr << "Failed to read cell ids at " << __FILE__ << " " << __LINE__ << endl;
Expand Down Expand Up @@ -509,7 +509,7 @@ bool convertMesh(vlsvinterface::Reader& vlsvReader,
std::array<int32_t,3> taskSize,taskStart;
std::array<int32_t,3> taskEnd;
int readOffset=0;
int readSize;
size_t readSize;
int index,my_x,my_y,my_z;
orderedData->clear();

Expand All @@ -536,11 +536,6 @@ bool convertMesh(vlsvinterface::Reader& vlsvReader,
readSize= taskSize[0] * taskSize[1] * taskSize[2];
std::vector<Real> readIn(variableVectorSize * variableDataSize*readSize);

if (vlsvReader.readArray("VARIABLE", variableAttributes, readOffset,readSize, (char*)readIn.data()) == false){
cerr << "ERROR, failed to read variable '" << _varToExtract << "' at " << __FILE__ << " " << __LINE__ << endl;
variableSuccess = false;
break;
}

int counter2=0;
uint64_t globalindex;
Expand All @@ -551,20 +546,39 @@ bool convertMesh(vlsvinterface::Reader& vlsvReader,

//Get global index
globalindex= x + y*xcells + z*xcells*ycells;
Real extract;
extract = (Real) readIn[counter+compToExtract];

if (vlsvReader.readArray("VARIABLE", variableAttributes, readOffset+counter,1, variableBuffer) == false) {
cerr << "ERROR, failed to read variable '" << _varToExtract << "' at " << __FILE__ << " " << __LINE__ << endl;
variableSuccess = false;
abort();
break;
}

// Get the variable value
Real extract = NAN;

switch (variableDataType) {
case datatype::type::FLOAT:
if(variableDataSize == sizeof(float)) extract = (Real)(variablePtrFloat[compToExtract]);
if(variableDataSize == sizeof(double)) extract = (Real)(variablePtrDouble[compToExtract]);
break;
case datatype::type::UINT:
extract = (Real)(variablePtrUint[compToExtract]);
break;
case datatype::type::INT:
extract = (Real)(variablePtrInt[compToExtract]);
break;
case datatype::type::UNKNOWN:
cerr << "ERROR, BAD DATATYPE AT " << __FILE__ << " " << __LINE__ << endl;
break;
}
orderedData->insert(pair<uint64_t, Real>(globalindex, extract));
counter+=variableVectorSize;
counter++;

}
}
}


readOffset+=readSize;


}
}else{
cerr<<"meshName not recognized\t" << __FILE__ << " " << __LINE__ <<endl;
Expand All @@ -577,6 +591,7 @@ bool convertMesh(vlsvinterface::Reader& vlsvReader,
if (variableSuccess == false) {
cerr << "ERROR reading array VARIABLE " << varToExtract << endl;
}
delete variableBuffer;
return meshSuccess && variableSuccess;
}

Expand Down Expand Up @@ -1531,11 +1546,15 @@ bool process2Files(const string fileName1,
}


map<string,string>::const_iterator it = attributes.find("--meshname");
if (cloneMesh(fileName1,outputFile,it->second,orderedData1) == false) {
std::cerr<<"Failed"<<std::endl;
return false;
}
}

singleStatistics(&orderedData1, &size, &mini, &maxi, &avg, &stdev); //CONTINUE
// Clone mesh from input file to diff file
map<string,string>::const_iterator it = attributes.find("--meshname");
if (cloneMesh(fileName1,outputFile,it->second,orderedData1) == false) return false;
}
outputStats(&size, &mini, &maxi, &avg, &stdev, verboseOutput, false);

singleStatistics(&orderedData2, &size, &mini, &maxi, &avg, &stdev);
Expand Down

0 comments on commit 396d3fd

Please sign in to comment.