adding the example for mpi_tool (including the test_mpi.nc)

GRIDINFORMER.py

@@ -811,6 +811,19 @@ def c2p(x,y):
         else:
             return (x < 0) * 270 + (x > 0) * 90
 
+    def GrtCirDist(lon1,lat1,lon2,lat2,R=6.37122E6):
+        a2r = lambda x: x/180. * math.pi
+
+        term1      = math.sin(a2r(lat1)) * math.sin(a2r(lat2))
+        term2      = math.cos(a2r(lat1)) * math.cos(a2r(lat2))
+        term3      = math.cos(a2r( lon2 - lon1  ))
+
+        deltaSigma = math.acos( term1 + term2 * term3 ) * 180./math.pi
+        numDist    = deltaSigma/360. * 2 * R * math.pi
+
+        return { "deltaSigma" : deltaSigma, 
+                 "dist"       : numDist     }
+
 class TOOLS:
     """ TOOLS is contains:
         timestamp
@@ -1229,7 +1242,7 @@ class DATA_READER:
     """
     The DATA_READER is based on my old work: gridtrans.py. 
     """
-    def __init__(self, STR_NULL="noData", NUM_NULL=-999.999):
+    def __init__(self, STR_NULL="noData", NUM_NULL=-999.999, STR_endian='b'):
         self.STR_NULL=STR_NULL
         self.NUM_NULL=NUM_NULL
         self.endian   = STR_endian
@@ -1351,9 +1364,9 @@ def READCSV(self, sourcefile):
         return result_arr,result_arr_text
 
     def IbtracsReader(self, strFileIn):
-        fileIn    = open(strFileIn, 'r')
-        arrLines  = fileIn.readlines()
-        numLength = len(arrLines)
+        fileIn       = open(strFileIn, 'r')
+        arrLines     = fileIn.readlines()
+        numTimesteps = len(arrLines)
 
         arrITEM15  = ["BASIN", "TIME",   "NATURE", 
                       "LAT"  ,  "LON", "WMO_WIND", "WMO_PRES",
@@ -1375,15 +1388,17 @@ def IbtracsReader(self, strFileIn):
 
         # Made the output dic
         dicOut    = {}
+        dicOut["TIMESTEPS"] = numTimesteps
+
         for item in arrITEM16:
-            dicOut[item] = [ "" for n in range(numLength) ]
+            dicOut[item] = [ "" for n in range(numTimesteps) ]
 
+        arrDATE = []
         # Analysing the dat and into the dictionary output
         for ind,line in enumerate(arrLines):
             arr_tmp = re.split("\s", line)
             numChk  = len(arr_tmp)
             if numChk == 17:
-                print(numChk , arr_tmp)
                 for ind2 in range(16):
                     if arrFMT16[ind2] == "str":
                         dicOut[ arrITEM16[ind2] ][ind] = str(arr_tmp[ind2]) 
@@ -1395,9 +1410,9 @@ def IbtracsReader(self, strFileIn):
                             dicOut[ arrITEM16[ind2] ][ ind ] = 0.0
                         else:
                             dicOut[ arrITEM16[ind2] ][ ind ] = float(arr_tmp[ ind2 ]) 
-
+                    if arrITEM16[ind2] == "DATE":
+                        arrDATE = InputTool.readDateTime(arr_tmp[ind2])[arrITEM16[ind2]]
             elif numChk == 16:
-                print(numChk , arr_tmp)
                 for ind2 in range(15):
                     if arrFMT15[ind2] == "str":
                         dicOut[ arrITEM15[ind2] ][ind] = str(arr_tmp[ind2]) 
@@ -1409,7 +1424,8 @@ def IbtracsReader(self, strFileIn):
                             dicOut[ arrITEM15[ind2] ][ ind ] = -999.999
                         else:
                             dicOut[ arrITEM15[ind2] ][ ind ] = float(arr_tmp[ ind2 ]) 
-
+                    if arrITEM16[ind2] == "DATE":
+                        dicOut[ "DATE" ][ ind ] = arrDATE             
             else:
                 print("something wrong with the dat file")
         return dicOut  

examples/mpi_tool.py

@@ -0,0 +1,90 @@
+### 
+#  This is the example for using MPI_TOOLS
+#     1) Creating the geometry for mpi rank for the input array
+#     2) Creating the netCDF file
+###
+
+import math, sys,os, time, re
+import netCDF4 as NC
+from mpi4py import MPI
+from GRIDINFORMER import MPI_TOOLS as MT
+
+IF_PARALLEL = True
+
+# MPI configuration
+# Initialing the MPI Commworld
+comm     = MPI.COMM_WORLD
+# Get the rank number (aka index of the CPU)
+NUM_MPI_RANK = comm.Get_rank()
+# Get the total size of CPUs
+NUM_MPI_SIZE = comm.Get_size()
+
+numNX = 200
+numNY = 100
+numNT =   6
+
+# Design Cores, can be the same as NUM_MPI_SIZE
+# numCores   = 48
+numCores   = NUM_MPI_SIZE
+
+numCoresNX =  6
+
+MPI_SET = MT(MPI_SIZE=NUM_MPI_SIZE, MPI_RANK = NUM_MPI_RANK,\
+             NUM_NX_END=numNX ,\
+             NUM_NY_END=numNY ,\
+             NUM_NX_CORES=numCoresNX)
+
+MPI_SET.CPU_GEOMETRY_2D()
+ARR_RANK_DESIGN = MPI_SET.ARR_RANK_DESIGN
+
+# Preparing the NC file
+STR_OUTPUT_FILE_TMP = "./test_mpi.nc" 
+
+if not os.path.exists(STR_OUTPUT_FILE_TMP):
+    if NUM_MPI_RANK == 0:
+
+        MPI_SET.MPI_MESSAGE("Starting to create File ")
+        NC_OUT_TMP         = NC.Dataset(STR_OUTPUT_FILE_TMP, 'w')
+        # Copying Dimensions:
+        NC_OUT_TMP.createDimension( "NX", numNX            )
+        NC_OUT_TMP.createDimension( "NY", numNY            )
+        NC_OUT_TMP.createDimension( "NT", numNT            )
+
+        # Adding statistics name:
+        NC_OUT_TMP.createVariable ('MAP2D', "f4", ('NY', 'NX'                ) )
+        NC_OUT_TMP.createVariable ('MAP3D', "i8", ('NT', 'NY', 'NX'          ) )
+        MPI_SET.MPI_MESSAGE("Finishing Creating the NC output{}".format(STR_OUTPUT_FILE_TMP))
+        NC_OUT_TMP.close()
+    else: 
+        time.sleep(3)
+        MPI_SET.MPI_MESSAGE("Done Waiting")
+
+
+    NC_OUT     = NC.Dataset(STR_OUTPUT_FILE_TMP, 'a', parallel=IF_PARALLEL)
+else:
+    NC_OUT     = NC.Dataset(STR_OUTPUT_FILE_TMP, 'a', parallel=IF_PARALLEL)
+
+
+MPI_SET.MPI_MESSAGE("Starting to work!")
+
+NUM_NX_START = ARR_RANK_DESIGN[NUM_MPI_RANK]["NX_START"]
+NUM_NX_END   = ARR_RANK_DESIGN[NUM_MPI_RANK]["NX_END"]
+NUM_NY_START = ARR_RANK_DESIGN[NUM_MPI_RANK]["NY_START"]
+NUM_NY_END   = ARR_RANK_DESIGN[NUM_MPI_RANK]["NY_END"]
+print(NUM_MPI_RANK, NUM_NX_END, NUM_NY_END)
+VAR_IN2D = NC_OUT.variables["MAP2D"]
+for j in range(NUM_NY_START, NUM_NY_END):
+    for i in range(NUM_NX_START, NUM_NX_END):
+        VAR_IN2D[j,i] = NUM_MPI_RANK
+
+MPI_SET.MPI_MESSAGE("End of MAP2D     ")
+VAR_IN3D = NC_OUT.variables["MAP3D"]
+for j in range(NUM_NY_START, NUM_NY_END):
+    for i in range(NUM_NX_START, NUM_NX_END):
+        for z in range(numNT):
+            VAR_IN3D[z,j,i] = NUM_MPI_RANK * z 
+
+MPI_SET.MPI_MESSAGE("End of MAP3D     ")
+
+
+NC_OUT.close()