From 208c8da9b99b753938a9553110381c3d5640780d Mon Sep 17 00:00:00 2001 From: kstppd Date: Mon, 1 Mar 2021 17:49:11 +0200 Subject: [PATCH 1/6] This fixes a bug that would show irrational stats when --diff was not passed as an arguement and handles fsgrid datatypes correctly. --- MAKE/Makefile.minihpc | 84 +++++++++++++++++++++++++++++++++++++++++++ tools/vlsvdiff.cpp | 65 +++++++++++++++++++++------------ 2 files changed, 126 insertions(+), 23 deletions(-) create mode 100644 MAKE/Makefile.minihpc diff --git a/MAKE/Makefile.minihpc b/MAKE/Makefile.minihpc new file mode 100644 index 000000000..52148d6e6 --- /dev/null +++ b/MAKE/Makefile.minihpc @@ -0,0 +1,84 @@ +#Kostis Papadakis +#======== Vectorization ========== +#Set vector backend type for vlasov solvers, sets precision and length. +#NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION) +#Options: +# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER +# AVX512: VEC8D_AGNER, VEC16F_AGNER +# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK + +ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) +#Single-precision + VECTORCLASS = VEC4F_FALLBACK +else +#Double-precision + VECTORCLASS = VEC4D_FALLBACK +endif + + +#======== Allocator ========= +#Use TBB malloc + +CMP = mpic++ +LNK = mpic++ + +PAPI_FLAG = + +#======== Vectorization ========== +#Set vector backend type for vlasov solvers, sets precision and length. +#NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION) +#Options: +# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER +# AVX512: VEC8D_AGNER, VEC16F_AGNER +# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK + +ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) +#Single-precision + VECTORCLASS = VEC8F_AGNER +else +#Double-precision + VECTORCLASS = VEC4D_AGNER +endif + +FLAGS = +#CXXFLAGS = -I $(HOME)/include -L $(HOME)/lib -g -funroll-loops -std=c++0x -fopenmp -W -Wall -pedantic -Wno-unused -fabi-version=0 -mavx +CXXFLAGS = -I $(HOME)/include -L $(HOME)/lib -O3 -funroll-loops -std=c++0x -fopenmp -W -Wall -Wno-unused -fabi-version=0 -mavx +MATHFLAGS = -ffast-math +LDFLAGS = -L $(HOME)/lib +LIB_MPI = -lgomp + + +#======== Libraries =========== + +LIBRARY_PREFIX = $(HOME)/libs + +INC_VECTORCLASS=-I$(LIBRARY_PREFIX)/version1 + +INC_BOOST = -I/usr/include/boost/ +LIB_BOOST = -L/usr/lib/x86_64-linux-gnu/ -lboost_program_options + +INC_ZOLTAN = -I$(LIBRARY_PREFIX)/zoltan/include +LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/zoltan/lib -lzoltan + +INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv +LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv + +INC_SILO = -I$(LIBRARY_PREFIX)/include +LIB_SILO = -L$(LIBRARY_PREFIX)/lib -lsilo + +INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg + +INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid + +#INC_JEMALLOC = -I$(LIBRARY_PREFIX)/include/jemalloc +#LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/lib -ljemalloc + +INC_JEMALLOC = -I$(LIBRARY_PREFIX)/jemalloc +LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/jemalloc/lib -ljemalloc + +LIB_PROFILE=-L/home/kstppd/libs/phiprof/lib -lphiprof +INC_PROFILE=-I/home/kstppd/libs/phiprof/include +INC_TOPO = + +INC_EIGEN = -I$(LIBRARY_PREFIX) + diff --git a/tools/vlsvdiff.cpp b/tools/vlsvdiff.cpp index 18244e1c4..96a3dd0c2 100644 --- a/tools/vlsvdiff.cpp +++ b/tools/vlsvdiff.cpp @@ -256,7 +256,7 @@ bool HandleFsGrid(const string& inputFileName, patch["vectorsize"]=to_string(vectorsize2); //Override MESH_DOMAIN_SIZES - std::array meshDomainSize({globalIds.size(), 0}); + std::array meshDomainSize({globalIds.size(), 0}); output.writeArray("MESH_DOMAIN_SIZES",patch ,1,vectorsize2, &meshDomainSize[0]); @@ -421,18 +421,18 @@ bool convertMesh(vlsvinterface::Reader& vlsvReader, cerr << "ERROR, failed to get array info for '" << _varToExtract << "' at " << __FILE__ << " " << __LINE__ << endl; return false; } + char *variableBuffer = new char[variableVectorSize * variableDataSize]; + float *variablePtrFloat = reinterpret_cast(variableBuffer); + double *variablePtrDouble = reinterpret_cast(variableBuffer); + uint *variablePtrUint = reinterpret_cast(variableBuffer); + int *variablePtrInt = reinterpret_cast(variableBuffer); if (gridName==gridType::SpatialGrid){ // Read the mesh array one node (of a spatial cell) at a time // and create a map which contains each cell's CellID and variable to be extracted - char *variableBuffer = new char[variableVectorSize * variableDataSize]; - float *variablePtrFloat = reinterpret_cast(variableBuffer); - double *variablePtrDouble = reinterpret_cast(variableBuffer); - uint *variablePtrUint = reinterpret_cast(variableBuffer); - int *variablePtrInt = reinterpret_cast(variableBuffer); - //Get local cell ids: + //Get local cell ids: vector local_cells; if ( vlsvReader.getCellIds( local_cells, meshName) == false ) { cerr << "Failed to read cell ids at " << __FILE__ << " " << __LINE__ << endl; @@ -509,7 +509,7 @@ bool convertMesh(vlsvinterface::Reader& vlsvReader, std::array taskSize,taskStart; std::array taskEnd; int readOffset=0; - int readSize; + size_t readSize; int index,my_x,my_y,my_z; orderedData->clear(); @@ -536,11 +536,6 @@ bool convertMesh(vlsvinterface::Reader& vlsvReader, readSize= taskSize[0] * taskSize[1] * taskSize[2]; std::vector readIn(variableVectorSize * variableDataSize*readSize); - if (vlsvReader.readArray("VARIABLE", variableAttributes, readOffset,readSize, (char*)readIn.data()) == false){ - cerr << "ERROR, failed to read variable '" << _varToExtract << "' at " << __FILE__ << " " << __LINE__ << endl; - variableSuccess = false; - break; - } int counter2=0; uint64_t globalindex; @@ -551,20 +546,39 @@ bool convertMesh(vlsvinterface::Reader& vlsvReader, //Get global index globalindex= x + y*xcells + z*xcells*ycells; - Real extract; - extract = (Real) readIn[counter+compToExtract]; + if (vlsvReader.readArray("VARIABLE", variableAttributes, readOffset+counter,1, variableBuffer) == false) { + cerr << "ERROR, failed to read variable '" << _varToExtract << "' at " << __FILE__ << " " << __LINE__ << endl; + variableSuccess = false; + abort(); + break; + } + + // Get the variable value + Real extract = NAN; + + switch (variableDataType) { + case datatype::type::FLOAT: + if(variableDataSize == sizeof(float)) extract = (Real)(variablePtrFloat[compToExtract]); + if(variableDataSize == sizeof(double)) extract = (Real)(variablePtrDouble[compToExtract]); + break; + case datatype::type::UINT: + extract = (Real)(variablePtrUint[compToExtract]); + break; + case datatype::type::INT: + extract = (Real)(variablePtrInt[compToExtract]); + break; + case datatype::type::UNKNOWN: + cerr << "ERROR, BAD DATATYPE AT " << __FILE__ << " " << __LINE__ << endl; + break; + } orderedData->insert(pair(globalindex, extract)); - counter+=variableVectorSize; + counter++; } } } - - readOffset+=readSize; - - } }else{ cerr<<"meshName not recognized\t" << __FILE__ << " " << __LINE__ <::const_iterator it = attributes.find("--meshname"); + if (cloneMesh(fileName1,outputFile,it->second,orderedData1) == false) { + std::cerr<<"Failed"<::const_iterator it = attributes.find("--meshname"); - if (cloneMesh(fileName1,outputFile,it->second,orderedData1) == false) return false; - } outputStats(&size, &mini, &maxi, &avg, &stdev, verboseOutput, false); singleStatistics(&orderedData2, &size, &mini, &maxi, &avg, &stdev); From d9873b7871e7ab8d35675eee5676f28047c1d948 Mon Sep 17 00:00:00 2001 From: kstppd Date: Mon, 1 Mar 2021 18:02:23 +0200 Subject: [PATCH 2/6] 64bit FsGrid indexing --- tools/vlsvdiff.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/vlsvdiff.cpp b/tools/vlsvdiff.cpp index 96a3dd0c2..ef1eb7a5e 100644 --- a/tools/vlsvdiff.cpp +++ b/tools/vlsvdiff.cpp @@ -256,7 +256,7 @@ bool HandleFsGrid(const string& inputFileName, patch["vectorsize"]=to_string(vectorsize2); //Override MESH_DOMAIN_SIZES - std::array meshDomainSize({globalIds.size(), 0}); + std::array meshDomainSize({globalIds.size(), 0}); output.writeArray("MESH_DOMAIN_SIZES",patch ,1,vectorsize2, &meshDomainSize[0]); From bcc89d861ff4b635dcfa6867488a6c36d8fa67f9 Mon Sep 17 00:00:00 2001 From: kstppd Date: Mon, 1 Mar 2021 18:08:56 +0200 Subject: [PATCH 3/6] Component sanity check --- tools/vlsvdiff.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/vlsvdiff.cpp b/tools/vlsvdiff.cpp index ef1eb7a5e..b6399368a 100644 --- a/tools/vlsvdiff.cpp +++ b/tools/vlsvdiff.cpp @@ -1778,6 +1778,10 @@ int main(int argn,char* args[]) { // 4th arg is its component, 0 for scalars, 2 for z component etc uint compToExtract = atoi(argsVector[4].c_str()); + if (compToExtract > 2){ + std::cerr<<"Component\t"< 5 ) { From 60f9bb80d3a6a39e889d4e93d8b83325f66096da Mon Sep 17 00:00:00 2001 From: kstppd Date: Mon, 1 Mar 2021 21:32:08 +0200 Subject: [PATCH 4/6] Remove component check since there are variables with more than 3 components --- tools/vlsvdiff.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/vlsvdiff.cpp b/tools/vlsvdiff.cpp index b6399368a..ef1eb7a5e 100644 --- a/tools/vlsvdiff.cpp +++ b/tools/vlsvdiff.cpp @@ -1778,10 +1778,6 @@ int main(int argn,char* args[]) { // 4th arg is its component, 0 for scalars, 2 for z component etc uint compToExtract = atoi(argsVector[4].c_str()); - if (compToExtract > 2){ - std::cerr<<"Component\t"< 5 ) { From 284eb7a34395a89e6adb78b00401007a55f49321 Mon Sep 17 00:00:00 2001 From: kstppd Date: Tue, 2 Mar 2021 01:23:29 +0200 Subject: [PATCH 5/6] Makefile cleanup --- MAKE/Makefile.kstppd | 50 +++++++++----------------- MAKE/Makefile.minihpc | 84 ------------------------------------------- 2 files changed, 17 insertions(+), 117 deletions(-) delete mode 100644 MAKE/Makefile.minihpc diff --git a/MAKE/Makefile.kstppd b/MAKE/Makefile.kstppd index 6b2dc29cd..d23b4d4c2 100644 --- a/MAKE/Makefile.kstppd +++ b/MAKE/Makefile.kstppd @@ -1,51 +1,36 @@ -#Kostis Papadakis -#======== Vectorization ========== -#Set vector backend type for vlasov solvers, sets precision and length. -#NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION) -#Options: -# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER -# AVX512: VEC8D_AGNER, VEC16F_AGNER -# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK - -ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) -#Single-precision - VECTORCLASS = VEC4F_FALLBACK -else -#Double-precision - VECTORCLASS = VEC4D_FALLBACK -endif - - -#======== Allocator ========= -#Use TBB malloc - +#Kostis Laptop +#Compiler & Linker CMP = mpic++ LNK = mpic++ -PAPI_FLAG = #======== Vectorization ========== #Set vector backend type for vlasov solvers, sets precision and length. #NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION) -#Options: +#Options; # AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER # AVX512: VEC8D_AGNER, VEC16F_AGNER # Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) #Single-precision - VECTORCLASS = VEC8F_AGNER + VECTORCLASS = VEC4F_FALLBACK else #Double-precision - VECTORCLASS = VEC4D_AGNER + VECTORCLASS = VEC4D_FALLBACK endif + +#======== Flags ========= + FLAGS = -#CXXFLAGS = -I $(HOME)/include -L $(HOME)/lib -g -funroll-loops -std=c++0x -fopenmp -W -Wall -pedantic -Wno-unused -fabi-version=0 -mavx -CXXFLAGS = -I $(HOME)/include -L $(HOME)/lib -O3 -funroll-loops -std=c++0x -fopenmp -W -Wall -Wno-unused -fabi-version=0 -mavx +CXXFLAGS = -O3 -funroll-loops -std=c++17 -fopenmp -W -Wall -Wno-unused -Wno-unused-parameter -Wno-missing-braces -fabi-version=0 -mavx +testpackage: CXXFLAGS = -g -ggdb -O2 -fopenmp -funroll-loops -std=c++17 -fabi-version=0 -mno-avx -mno-fma -fno-unsafe-math-optimizations +CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE -DPAPI_MEM MATHFLAGS = -ffast-math +testpackage: MATHFLAGS = -fno-unsafe-math-optimizations LDFLAGS = -L $(HOME)/lib -LIB_MPI = -lgomp +LIB_MPI = -lgomp -lpapi #======== Libraries =========== @@ -70,15 +55,14 @@ INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid -#INC_JEMALLOC = -I$(LIBRARY_PREFIX)/include/jemalloc -#LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/lib -ljemalloc +INC_PAPI = -I$(LIBRARY_PREFIX)/papi-5.4.3/src +LIB_PAPI = -L$(LIBRARY_PREFIX)/papi-5.4.3/src -lpapi -INC_JEMALLOC = -I/home/kostis/libraries/jemalloc -LIB_JEMALLOC = -L/home/kostis/libraries/jemalloc/lib -ljemalloc +INC_JEMALLOC = -I$(LIBRARY_PREFIX)/libraries/jemalloc +LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/libraries/jemalloc/lib -ljemalloc LIB_PROFILE = -L$(LIBRARY_PREFIX)/phiprof/lib -lphiprof INC_PROFILE = -I$(LIBRARY_PREFIX)/phiprof/include -INC_TOPO = INC_EIGEN = -I$(LIBRARY_PREFIX)/eigen-eigen-07105f7124f9 diff --git a/MAKE/Makefile.minihpc b/MAKE/Makefile.minihpc deleted file mode 100644 index 52148d6e6..000000000 --- a/MAKE/Makefile.minihpc +++ /dev/null @@ -1,84 +0,0 @@ -#Kostis Papadakis -#======== Vectorization ========== -#Set vector backend type for vlasov solvers, sets precision and length. -#NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION) -#Options: -# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER -# AVX512: VEC8D_AGNER, VEC16F_AGNER -# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK - -ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) -#Single-precision - VECTORCLASS = VEC4F_FALLBACK -else -#Double-precision - VECTORCLASS = VEC4D_FALLBACK -endif - - -#======== Allocator ========= -#Use TBB malloc - -CMP = mpic++ -LNK = mpic++ - -PAPI_FLAG = - -#======== Vectorization ========== -#Set vector backend type for vlasov solvers, sets precision and length. -#NOTE this has to have the same precision as the distribution function define (DISTRIBUTION_FP_PRECISION) -#Options: -# AVX: VEC4D_AGNER, VEC4F_AGNER, VEC8F_AGNER -# AVX512: VEC8D_AGNER, VEC16F_AGNER -# Fallback: VEC4D_FALLBACK, VEC4F_FALLBACK, VEC8F_FALLBACK - -ifeq ($(DISTRIBUTION_FP_PRECISION),SPF) -#Single-precision - VECTORCLASS = VEC8F_AGNER -else -#Double-precision - VECTORCLASS = VEC4D_AGNER -endif - -FLAGS = -#CXXFLAGS = -I $(HOME)/include -L $(HOME)/lib -g -funroll-loops -std=c++0x -fopenmp -W -Wall -pedantic -Wno-unused -fabi-version=0 -mavx -CXXFLAGS = -I $(HOME)/include -L $(HOME)/lib -O3 -funroll-loops -std=c++0x -fopenmp -W -Wall -Wno-unused -fabi-version=0 -mavx -MATHFLAGS = -ffast-math -LDFLAGS = -L $(HOME)/lib -LIB_MPI = -lgomp - - -#======== Libraries =========== - -LIBRARY_PREFIX = $(HOME)/libs - -INC_VECTORCLASS=-I$(LIBRARY_PREFIX)/version1 - -INC_BOOST = -I/usr/include/boost/ -LIB_BOOST = -L/usr/lib/x86_64-linux-gnu/ -lboost_program_options - -INC_ZOLTAN = -I$(LIBRARY_PREFIX)/zoltan/include -LIB_ZOLTAN = -L$(LIBRARY_PREFIX)/zoltan/lib -lzoltan - -INC_VLSV = -I$(LIBRARY_PREFIX)/vlsv -LIB_VLSV = -L$(LIBRARY_PREFIX)/vlsv -lvlsv - -INC_SILO = -I$(LIBRARY_PREFIX)/include -LIB_SILO = -L$(LIBRARY_PREFIX)/lib -lsilo - -INC_DCCRG = -I$(LIBRARY_PREFIX)/dccrg - -INC_FSGRID = -I$(LIBRARY_PREFIX)/fsgrid - -#INC_JEMALLOC = -I$(LIBRARY_PREFIX)/include/jemalloc -#LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/lib -ljemalloc - -INC_JEMALLOC = -I$(LIBRARY_PREFIX)/jemalloc -LIB_JEMALLOC = -L$(LIBRARY_PREFIX)/jemalloc/lib -ljemalloc - -LIB_PROFILE=-L/home/kstppd/libs/phiprof/lib -lphiprof -INC_PROFILE=-I/home/kstppd/libs/phiprof/include -INC_TOPO = - -INC_EIGEN = -I$(LIBRARY_PREFIX) - From 4e090c7550d5d73c892150baab8143ea825095ae Mon Sep 17 00:00:00 2001 From: kstppd Date: Tue, 2 Mar 2021 13:26:14 +0200 Subject: [PATCH 6/6] Makefile cleanup revised --- MAKE/Makefile.kstppd | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/MAKE/Makefile.kstppd b/MAKE/Makefile.kstppd index d23b4d4c2..887c7c102 100644 --- a/MAKE/Makefile.kstppd +++ b/MAKE/Makefile.kstppd @@ -33,6 +33,20 @@ LDFLAGS = -L $(HOME)/lib LIB_MPI = -lgomp -lpapi + + +#================PAPI================== +#Add PAPI_MEM define to use papi to report memory consumption? +CXXFLAGS += -DPAPI_MEM +testpackage: CXXFLAGS += -DPAPI_MEM + + +#======== Allocator ========= +#Use jemalloc instead of system malloc to reduce memory fragmentation? https://github.com/jemalloc/jemalloc +#Configure jemalloc with --with-jemalloc-prefix=je_ when installing it +CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE +testpackage: CXXFLAGS += -DUSE_JEMALLOC -DJEMALLOC_NO_DEMANGLE + #======== Libraries =========== LIBRARY_PREFIX = $(HOME)