diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..e4249f0 Binary files /dev/null and b/.DS_Store differ diff --git a/BUFR_TABLES/BUFRCREX_31_0_0_TableB_en.txt b/BUFR_TABLES/BUFRCREX_31_0_0_TableB_en.txt new file mode 100755 index 0000000..ab41483 --- /dev/null +++ b/BUFR_TABLES/BUFRCREX_31_0_0_TableB_en.txt @@ -0,0 +1,10 @@ +"No","ClassNo","ClassName_en","FXY","ElementName_en","Note_en","BUFR_Unit","BUFR_Scale","BUFR_ReferenceValue","BUFR_DataWidth_Bits","CREX_Unit","CREX_Scale","CREX_DataWidth_Char","Status" +1225.00,"22","Oceanographic elements","022069","Spectral wave density",,"m2/Hz","3","0","22","m2/Hz","3","7","Operational" +1236.00,"22","Oceanographic elements","022080","Waveband central frequency",,"Hz","3","0","10","Hz","3","4","Operational" +1252.00,"22","Oceanographic elements","022096","Spectral band width",,"/s","3","0","4","/s","3","2","Operational" +1560.00,"31","Data description operator qualifiers","031001","Delayed descriptor replication factor",,"Numeric","0","0","8",,,,"Operational" +1734.00,"42","Oceanographic elements","042011","a1 coefficient of the directional Fourier series",,"Numeric","4","-20000","15","Numeric","4","6","Test" +1735.00,"42","Oceanographic elements","042012","b1 coefficient of the directional Fourier series",,"Numeric","4","-20000","15","Numeric","4","6","Test" +1736.00,"42","Oceanographic elements","042013","a2 coefficient of the directional Fourier series",,"Numeric","4","-20000","15","Numeric","4","6","Test" +1737.00,"42","Oceanographic elements","042014","b2 coefficient of the directional Fourier series",,"Numeric","4","-20000","15","Numeric","4","6","Test" +1738.00,"42","Oceanographic elements","042015","Check factor K",,"Numeric","2","0","12","Numeric","2","4","Test" \ No newline at end of file diff --git a/BUFR_TABLES/BUFR_31_0_0_TableA_en.txt b/BUFR_TABLES/BUFR_31_0_0_TableA_en.txt new file mode 100755 index 0000000..0846288 --- /dev/null +++ b/BUFR_TABLES/BUFR_31_0_0_TableA_en.txt @@ -0,0 +1,34 @@ +"No","CodeFigure","Meaning_en","Status" +1.00,"0","Surface data - land","Operational" +2.00,"1","Surface data - sea","Operational" +3.00,"2","Vertical soundings (other than satellite)","Operational" +4.00,"3","Vertical soundings (satellite)","Operational" +5.00,"4","Single level upper-air data (other than satellite)","Operational" +6.00,"5","Single level upper-air data (satellite)","Operational" +7.00,"6","Radar data","Operational" +8.00,"7","Synoptic features","Operational" +9.00,"8","Physical/chemical constituents","Operational" +10.00,"9","Dispersal and transport","Operational" +11.00,"10","Radiological data","Operational" +12.00,"11","BUFR tables, complete replacement or update","Operational" +14.00,"12","Surface data (satellite)","Operational" +15.00,"13","Forecasts","Operational" +16.00,"14","Warnings","Operational" +17.00,"15-19","Reserved","Operational" +19.00,"20","Status information","Operational" +20.00,"21","Radiances (satellite measured)","Operational" +21.00,"22","Radar (satellite) but not altimeter and scatterometer","Operational" +22.00,"23","Lidar (satellite)","Operational" +23.00,"24","Scatterometry (satellite)","Operational" +24.00,"25","Altimetry (satellite)","Operational" +25.00,"26","Spectrometry (satellite)","Operational" +26.00,"27","Gravity measurement (satellite)","Operational" +27.00,"28","Precision orbit (satellite)","Operational" +28.00,"29","Space environment (satellite)","Operational" +29.00,"30","Calibration datasets (satellite)","Operational" +30.00,"31","Oceanographic data","Operational" +31.00,"32-100","Reserved","Operational" +32.00,"101","Image data (satellite)","Operational" +33.00,"102-239","Reserved","Operational" +34.00,"240-254","For experimental use","Operational" +35.00,"255","Other category","Operational" diff --git a/BUFR_TABLES/BUFR_31_0_0_TableC_en.txt b/BUFR_TABLES/BUFR_31_0_0_TableC_en.txt new file mode 100755 index 0000000..52ca493 --- /dev/null +++ b/BUFR_TABLES/BUFR_31_0_0_TableC_en.txt @@ -0,0 +1,29 @@ +"No","FXY","OperatorName_en","OperationDefinition_en","Note_en","Status" +1.00,"201YYY","Change data width","Add (YYY-128) bits to the data width given for each data element in Table B, other than CCITT IA5 (character) data, code or flag tables.",,"Operational" +3.00,"202YYY","Change scale","Add YYY-128 to the scale for each data element in Table B, other than CCITT IA5 (character) data, code or flag tables.",,"Operational" +5.00,"203YYY","Change reference values","Subsequent element descriptors define new reference values for corresponding Table B entries. Each new reference value is represented by YYY bits in the Data section. Definition of new reference values is concluded by coding this operator with YYY = 255. Negative reference values shall be represented by a positive integer with the left-most bit (bit 1) set to 1.",,"Operational" +6.00,"204YYY","Add associated field","Precede each data element with YYY bits of information. This operation associates a data field (e.g. quality control information) of YYY bits with each data element.",,"Operational" +7.00,"205YYY","Signify character","YYY characters (CCITT International Alphabet No. 5) are inserted as a data field of YYY x 8 bits in length.",,"Operational" +9.00,"206YYY","Signify data width for the immediately following local descriptor","YYY bits of data are described by the immediately following descriptor.",,"Operational" +10.00,"207YYY","Increase scale, reference value and data width","For Table B elements, which are not CCITT IA5 (character data), code tables, or flag tables: 1. Add YYY to the existing scale factor 2. Multiply the existing reference value by 10**YYY 3. Calculate ((10 x YYY) + 2) ÷ 3, disregard any fractional remainder and add the result to the existing bit width.",,"Operational" +12.00,"208YYY","Change width of CCITT IA5 field","YYY characters from CCITT International Alphabet No. 5 (representing YYY x 8 bits in length) replace the specified data width given for each CCITT IA5 element in Table B.",,"Operational" +14.00,"221YYY","Data not present","Data values present in Section 4 (Data section) corresponding to the following YYY descriptors shall be limited to data from Classes 01-09, and Class 31.",,"Operational" +15.00,"222000","Quality information follows","The values of Class 33 elements which follow relate to the data defined by the data present bit-map.",,"Operational" +16.00,"223000","Substituted values operator","The substituted values which follow relate to the data defined by the data present bit-map.",,"Operational" +17.00,"223255","Substituted values marker operator","This operator shall signify a data item containing a substituted value; the element descriptor for the substituted value is obtained by the application of the data present bit-map associated with the substituted values operator.",,"Operational" +18.00,"224000","First-order statistical values follow","The statistical values which follow relate to the data defined by the data present bit-map.",,"Operational" +19.00,"224255","First-order statistical values marker operator","This operator shall signify a data item containing a first-order statistical value of the type indicated by the preceding 0 08 023 element descriptor; the element descriptor to which the first-order statistic relates is obtained by the application of the data present bit-map associated with the first-order statistical values follow operator; first-order statistical values shall be represented as defined by this element descriptor.",,"Operational" +20.00,"225000","Difference statistical values follow","The statistical values which follow relate to the data defined by the data present bit-map.",,"Operational" +21.00,"225255","Difference statistical values marker operator","This operator shall signify a data item containing a difference statistical value of the type indicated by the preceding 0 08 024 element descriptor; the element descriptor to which the difference statistical value relates is obtained by the application of the data present bit-map associated with the difference statistical values follow operator; difference statistical values shall be represented as defined by this element descriptor, but with a reference value of -2**n and a data width of (n+1), where n is the data width given by the original descriptor. This special reference value allows the statistical difference values to be centred around zero.",,"Operational" +22.00,"232000","Replaced/retained values follow","The replaced/retained values which follow relate to the data defined by the data present bit-map.",,"Operational" +23.00,"232255","Replaced/retained value marker operator","This operator shall signify a data item containing the original of an element which has been replaced by a substituted value. The element descriptor for the retained value is obtained by the application of the data present bit-map associated with the substituted values operator.",,"Operational" +24.00,"235000","Cancel backward data reference","This operator terminates all previously defined back-ward reference and cancels any previously defined data present bit-map; it causes the next data present bit-map to refer to the data descriptors which immediately precede the operator to which it relates.",,"Operational" +25.00,"236000","Define data present bit-map","This operator defines the data present bit-map which follows for possible re-use; only one data present bit-map may be defined between this operator and the cancel use defined data present bit-map operator.",,"Operational" +26.00,"237000","Use defined data present bit-map","This operator causes the defined data present bit-map to be used again.",,"Operational" +27.00,"237255","Cancel use defined data present bit-map","This operator cancels the re-use of the defined data present bit-map.",,"Operational" +28.00,"241000","Define event","This operator denotes the beginning of the definition of an event","(see Note 19).","Operational" +30.00,"241255","Cancel define event","This operator denotes the conclusion of the event definition that was begun via the previous 2 41 000 operator.",,"Operational" +32.00,"242000","Define conditioning event","This operator denotes the beginning of the definition of a conditioning event","(see Note 19).","Operational" +34.00,"242255","Cancel define conditioning event","This operator denotes the conclusion of the conditioning event definition that was begun via the previous 2 42 000 operator.",,"Operational" +36.00,"243000","Categorical forecast values follow","The values which follow are categorical forecast values","(see Note 20).","Operational" +38.00,"243255","Cancel categorical forecast values follow","This operator denotes the conclusion of the definition of categorical forecast values that was begun via the previous 2 43 000 operator.",,"Operational" diff --git a/BUFR_TABLES/BUFR_31_0_0_TableD_en.txt b/BUFR_TABLES/BUFR_31_0_0_TableD_en.txt new file mode 100644 index 0000000..7773b63 --- /dev/null +++ b/BUFR_TABLES/BUFR_31_0_0_TableD_en.txt @@ -0,0 +1,15 @@ +"No","Category","CategoryOfSequences_en","FXY1","Title_en","SubTitle_en","FXY2","ElementName_en","ElementDescription_en","Note_en","Status" +8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"112000","Delayed replication of 12 descriptors",,,"Test" +8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"031001","Delayed descriptor replication factor",,,"Test" +8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"201144","Add YYY-128 to the width for each data element in Table B, other than CCITT IA5 (character) data, code or flag tables.",,,"Test" +8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"202133","Add YYY-128 to the scale for each data element in Table B, other than CCITT IA5 (character) data, code or flag tables.",,,"Test" +8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"022080","Waveband central frequency (Hz)",,,"Test" +8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"022096","Spectral band width (Hz)",,,"Test" +8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"022069","Spectral wave density (m2 Hz-1)",,,"Test" +8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"202000","Add YYY-128 to the scale for each data element in Table B, other than CCITT IA5 (character) data, code or flag tables.",,,"Test" +8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"201000","Add YYY-128 to the width for each data element in Table B, other than CCITT IA5 (character) data, code or flag tables.",,,"Test" +8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"042011","a1 coefficient of the directional Fourier series",,,"Test" +8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"042012","b1 coefficient of the directional Fourier series",,,"Test" +8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"042013","a2 coefficient of the directional Fourier series",,,"Test" +8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"042014","b2 coefficient of the directional Fourier series",,,"Test" +8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"042015","Check factor K",,,"Test" diff --git a/README.md b/README.md index 01f7336..7ea1b00 100644 --- a/README.md +++ b/README.md @@ -1 +1,8 @@ # BUFR_python_waves + +Development code used to encode first five Fourier components for wave spectrum in BUFR. + +Usage: + python3 bufr_wave.py + +Configuration information is read from waverider_first_five.json diff --git a/bufr_wave.py b/bufr_wave.py new file mode 100644 index 0000000..46edc12 --- /dev/null +++ b/bufr_wave.py @@ -0,0 +1,112 @@ +from expand_sequence import * +import json +import sys +from bitarray import bitarray +def main( argv ): + pd.set_option('display.max_columns', 150) + pd.set_option('display.max_rows', 200) + pd.set_option('display.width', 300 ) + + # Load message headers / sections + msg_file = 'waverider_first_five.json' + with open(msg_file) as bufrmsg: + msg = json.load( bufrmsg ) + + # ---------------------------------------------------------------------------------- + # need to do the next code block better, at the moment every subset has to have the + # same set of delayed replicators. In future need to loop over different subsets + # separately. + # ---------------------------------------------------------------------------------- + + # read in data file + datain = pd.read_csv(msg['datafile']) + replicators = [ datain.shape[0] ] + nsubsets = 1 + # get descriptors to pack + descriptors = expand_sequence( msg['section3']['descriptors']['value'], replicators ) + descriptors.reset_index( inplace=True ) + print(descriptors) + # calculate message length + data_length = descriptors.BUFR_DataWidth_Bits.sum() * nsubsets + if data_length % 8 > 0: + data_length = data_length + (8 - data_length % 8) + print(data_length) + data_length = int( data_length / 8) + # ---------------------------------------------------------------------------------- + msg['section4']['length']['value'] = data_length + 4 + msg['section4']['data']['width'] = data_length + msg_length = 0 + msg_length += 8 + msg_length += msg['section1']['length']['value'] + if msg['section1']['optional_section']['value'] == 1: + msg_length += msg['section2']['length']['value'] + msg_length += msg['section3']['length']['value'] + msg_length += msg['section4']['length']['value'] + msg_length += 4 + + + # Set length in section 0 + msg['section0']['length']['value'] = msg_length + + # Pack data section + + # first we need to transpose and unstack our data frame to single series + # get columns to write + towrite = datain.loc[:,['freq','bandwidth','energy','a1','b1','a2','b2','check_factor']].copy() + towrite = towrite.values.flatten() + # add number of repeats at start + replicators = [ datain.shape[0] ] + towrite = pd.np.insert( towrite, 0, replicators[0]) + # now convert back to data frame + towrite = pd.DataFrame({'data':towrite}).transpose() + # ---------------------------------------------------------------------------------- + # move to pack_section function + # ---------------------------------------------------------------------------------- + rsum = 0 + for i in range( towrite.shape[1] ): + scale = descriptors.loc[i,'BUFR_Scale'] + width = descriptors.loc[i,'BUFR_DataWidth_Bits'] + rsum += width + offset = descriptors.loc[i,'BUFR_ReferenceValue'] + units = descriptors.loc[i,'BUFR_Unit'] + msng = pow(2, width) - 1 + fld = towrite.columns[i] + if units == 'CCITT IA5': + towrite[fld] = towrite[fld].apply( lambda x: ''.join( format( ord(y), 'b').zfill(8) for y in x.rjust( int(width/8)) )) + towrite[fld] = towrite[fld].apply( lambda x: x.zfill( width ) ) + else: + towrite[fld] = towrite[fld].apply( lambda x: int(round(x * pow(10,scale) - offset))) + towrite[fld] = towrite[fld].fillna( msng ) + towrite[fld] = towrite[fld].apply( lambda x: format( int(x), 'b').zfill( width ) ) + + bitsOut = towrite.apply( lambda x: x.sum(), axis = 1).sum() + nbytes = int( len(bitsOut) / 8 ) + pack = len(bitsOut) % 8 + if pack > 0: + bitsOut = bitsOut + ''.zfill( 8 - pack ) + nbytes = int( len(bitsOut) / 8 ) + + assert nbytes == data_length + + msg['section4']['data']['value'] = bitsOut + + # Write to file + + bitsOut = '' + # pack sections + bitsOut += pack_section( msg['section0'] ) + bitsOut += pack_section( msg['section1'] ) + if msg['section1']['optional_section']['value'] == 1: + bitsOut += pack_section( msg['section2'] ) + bitsOut += pack_section( msg['section3'] ) + bitsOut += pack_section( msg['section4'] ) + bitsOut += pack_section( msg['section5'] ) + + # now write to file + #file_out = open('waves_first_five.bin', 'wb') + file_out = open(msg['outputfile'], 'wb') + bitarray( bitsOut ).tofile(file_out) + file_out.close() + +if __name__ == '__main__': + main(sys.argv[1:]) \ No newline at end of file diff --git a/dws-drifter-first5_v2.bufr b/dws-drifter-first5_v2.bufr new file mode 100644 index 0000000..51c6fe1 Binary files /dev/null and b/dws-drifter-first5_v2.bufr differ diff --git a/dws-drifter-first5_v2.dat b/dws-drifter-first5_v2.dat new file mode 100644 index 0000000..01c5fea --- /dev/null +++ b/dws-drifter-first5_v2.dat @@ -0,0 +1,117 @@ +300234063341760, 2019-04-11 17:00:00, 1.30, 6.92, 6.84, 4, 116, +0.03125000, 0.01140000, -0.2321, -0.3658, -0.1533, 0.4857, +0.03515625, 0.01940000, 0.0774, -0.4225, 0.0675, 0.0168, +0.03906250, 0.01160000, 0.0198, -0.2801, -0.1420, -0.3262, +0.04296875, 0.02570000, 0.0298, 0.3372, -0.4022, 0.1172, +0.04687500, 0.02470000, 0.4798, 0.1024, 0.1710, -0.0582, +0.05078125, 0.01610000, 0.3131, -0.2473, -0.1771, -0.0841, +0.05468750, 0.03080000, 0.0169, -0.1276, -0.2385, -0.2076, +0.05859375, 0.01480000, -0.4683, -0.0901, 0.3506, 0.2452, +0.06250000, 0.01970000, 0.0224, -0.3761, -0.0624, 0.2695, +0.06640625, 0.03880000, 0.2856, 0.0382, 0.3540, -0.1667, +0.07031250, 0.06490000, 0.2867, -0.2485, -0.3451, -0.1752, +0.07421875, 0.16540000, 0.3881, -0.5799, -0.1486, -0.5653, +0.07812500, 0.77050000, 0.5193, -0.7697, -0.3231, -0.8560, +0.08203125, 1.06170000, 0.4595, -0.8405, -0.5449, -0.7974, +0.08593750, 0.91490000, 0.4170, -0.8869, -0.6092, -0.7319, +0.08984375, 0.75220000, 0.3726, -0.8908, -0.7058, -0.6829, +0.09375000, 0.61620000, 0.3974, -0.8770, -0.6518, -0.7086, +0.09765625, 0.79640000, 0.4703, -0.8275, -0.5111, -0.8187, +0.10156250, 1.10680000, 0.4726, -0.8510, -0.5067, -0.8152, +0.10546875, 0.83620000, 0.4466, -0.8630, -0.5497, -0.7704, +0.10937500, 0.94060000, 0.4358, -0.8649, -0.5574, -0.7336, +0.11328125, 0.47070000, 0.5014, -0.8182, -0.3957, -0.8072, +0.11718750, 0.58720000, 0.2584, -0.9074, -0.6954, -0.5360, +0.12109375, 0.24290000, 0.2146, -0.9162, -0.8320, -0.3909, +0.12500000, 1.24930000, 0.3718, -0.8933, -0.6792, -0.6890, +0.12890625, 1.48990000, 0.4260, -0.8683, -0.5639, -0.7517, +0.13281250, 0.59530000, 0.3738, -0.8418, -0.5556, -0.5829, +0.13671875, 1.27440000, 0.2789, -0.9216, -0.7857, -0.4989, +0.14062500, 1.22880000, 0.5083, -0.8266, -0.4237, -0.8510, +0.14453125, 1.57240000, 0.4150, -0.8585, -0.5219, -0.7094, +0.14843750, 0.90570000, 0.2080, -0.8791, -0.6340, -0.3429, +0.15234375, 0.49070000, 0.2327, -0.8611, -0.5711, -0.3702, +0.15625000, 0.66140000, 0.4434, -0.8426, -0.5226, -0.7203, +0.16015625, 0.58730000, 0.3453, -0.8468, -0.5761, -0.5097, +0.16406250, 0.37900000, 0.4381, -0.7864, -0.4108, -0.5698, +0.16796875, 0.44500000, 0.2796, -0.8712, -0.7062, -0.3859, +0.17187500, 0.25130000, 0.0873, -0.9265, -0.8042, -0.1080, +0.17578125, 0.06640000, 0.1373, -0.6147, -0.2766, 0.2114, +0.17968750, 0.29830000, 0.3692, -0.8551, -0.6111, -0.6541, +0.18359375, 0.22200000, 0.3759, -0.8371, -0.5938, -0.6555, +0.18750000, 0.51700000, 0.1850, -0.9199, -0.7817, -0.3824, +0.19140625, 0.33790000, 0.2897, -0.8827, -0.6525, -0.5026, +0.19531250, 0.49060000, 0.3635, -0.8672, -0.6574, -0.5472, +0.19921875, 0.43600000, 0.3455, -0.8612, -0.6949, -0.5506, +0.20312500, 0.38860000, 0.1616, -0.8718, -0.7260, -0.1988, +0.20703125, 0.37270000, -0.0352, -0.8843, -0.7160, 0.1958, +0.21093750, 0.29680000, -0.0315, -0.9040, -0.7547, 0.1337, +0.21484375, 0.21600000, 0.1478, -0.8870, -0.7817, -0.3372, +0.21875000, 0.13360000, 0.1399, -0.8120, -0.4020, -0.2148, +0.22265625, 0.07870000, 0.4853, -0.6819, -0.0901, -0.5755, +0.22656250, 0.06860000, 0.5311, -0.6191, -0.0879, -0.5749, +0.23046875, 0.07430000, 0.0661, -0.8485, -0.7464, -0.0585, +0.23437500, 0.14390000, -0.0010, -0.9037, -0.8495, 0.0935, +0.23828125, 0.22630000, 0.3085, -0.8472, -0.5193, -0.4409, +0.24218750, 0.11190000, 0.6476, -0.6480, 0.0795, -0.7780, +0.24609375, 0.17080000, 0.4754, -0.7151, -0.1394, -0.5895, +0.25000000, 0.11240000, 0.1677, -0.7053, -0.1868, -0.0117, +0.25390625, 0.10100000, 0.0531, -0.8687, -0.7220, 0.0418, +0.25781250, 0.11730000, 0.1096, -0.8652, -0.9015, -0.1287, +0.26171875, 0.05720000, -0.0244, -0.8578, -0.8865, 0.0905, +0.26562500, 0.10070000, 0.0975, -0.8555, -0.7897, -0.1269, +0.26953125, 0.10470000, 0.2469, -0.8329, -0.7200, -0.3742, +0.27343750, 0.07780000, 0.2181, -0.8102, -0.5269, -0.3701, +0.27734375, 0.05000000, 0.0263, -0.6860, -0.4283, 0.0731, +0.28125000, 0.09810000, -0.1746, -0.7945, -0.7385, 0.5295, +0.28515625, 0.06770000, -0.0838, -0.7398, -0.3486, 0.2832, +0.28906250, 0.03170000, -0.2898, -0.6692, -0.4108, 0.3548, +0.29296875, 0.02510000, 0.0866, -0.8302, -0.7930, -0.1553, +0.29687500, 0.01880000, 0.0015, -0.6748, -0.5291, -0.1153, +0.30078125, 0.01670000, 0.1548, -0.3327, 0.4492, -0.0803, +0.30468750, 0.04630000, -0.2205, -0.7481, -0.3636, 0.4471, +0.30859375, 0.05100000, -0.0891, -0.8896, -0.6897, 0.2773, +0.31250000, 0.02120000, -0.0185, -0.7684, -0.5950, 0.2615, +0.31640625, 0.03420000, -0.1358, -0.8123, -0.6859, 0.3955, +0.32031250, 0.02350000, -0.1192, -0.7072, -0.4657, 0.2020, +0.32421875, 0.00730000, 0.1076, -0.6267, -0.7195, -0.1066, +0.32812500, 0.01970000, 0.1167, -0.7740, -0.6328, -0.0194, +0.33203125, 0.00640000, 0.6188, -0.3740, 0.4270, -0.3459, +0.33593750, 0.01540000, 0.1441, -0.5727, -0.4750, -0.0696, +0.33984375, 0.00860000, -0.0711, -0.4317, -0.1364, 0.3289, +0.34375000, 0.01330000, -0.2519, -0.4232, -0.0496, 0.3250, +0.34765625, 0.00470000, -0.2251, -0.4952, -0.1475, -0.0308, +0.35156250, 0.01760000, 0.0336, -0.6259, -0.5638, -0.0124, +0.35546875, 0.03000000, -0.2131, -0.7790, -0.4548, 0.2139, +0.35937500, 0.00880000, -0.0569, -0.3267, 0.4862, -0.1453, +0.36328125, 0.00350000, 0.2532, -0.4317, 0.2122, -0.1161, +0.36718750, 0.00730000, -0.2179, -0.2576, -0.2457, -0.0011, +0.37109375, 0.01750000, -0.2178, -0.5419, -0.3502, -0.0424, +0.37500000, 0.01650000, -0.3482, -0.7152, -0.4676, 0.3899, +0.37890625, 0.00520000, -0.0066, -0.6714, -0.5439, 0.0595, +0.38281250, 0.01690000, 0.1952, -0.7665, -0.7229, -0.1642, +0.38671875, 0.00900000, 0.3139, -0.4300, -0.1615, -0.0647, +0.39062500, 0.01240000, 0.1935, -0.4855, -0.3832, -0.2075, +0.39453125, 0.00780000, 0.2856, -0.5381, -0.1572, 0.0139, +0.39843750, 0.00450000, -0.1594, -0.3399, -0.1355, 0.1540, +0.40234375, 0.00940000, 0.0589, -0.6023, -0.5909, -0.1724, +0.40625000, 0.01580000, 0.0437, -0.4562, -0.4921, -0.2341, +0.41015625, 0.01010000, -0.3149, -0.3027, 0.2098, 0.6899, +0.41406250, 0.00670000, -0.3820, -0.2946, 0.4400, 0.4772, +0.41796875, 0.01920000, -0.3022, -0.4439, -0.2496, 0.4309, +0.42187500, 0.01060000, 0.0769, -0.1706, 0.0041, 0.0962, +0.42578125, 0.01010000, 0.1625, -0.2435, -0.5953, -0.0614, +0.42968750, 0.01060000, 0.2486, -0.1556, 0.4272, -0.5841, +0.43359375, 0.00910000, 0.4083, 0.1140, 0.0690, 0.6781, +0.43750000, 0.00500000, 0.0637, -0.2223, -0.4514, 0.2502, +0.44140625, 0.00570000, 0.0077, 0.1684, -0.1297, -0.5429, +0.44531250, 0.00660000, -0.0108, -0.0561, -0.6620, 0.0103, +0.44921875, 0.00490000, -0.0759, -0.4674, -0.1578, 0.5775, +0.45312500, 0.01130000, 0.0574, -0.3038, -0.3263, 0.5646, +0.45703125, 0.00710000, 0.1610, 0.4087, -0.2312, -0.3342, +0.46093750, 0.00440000, -0.0927, 0.5037, -0.3203, -0.4314, +0.46484375, 0.00880000, -0.0488, -0.0466, -0.1750, 0.3216, +0.46875000, 0.00710000, -0.0805, -0.4308, -0.4621, 0.5731, +0.47265625, 0.00470000, 0.2918, -0.3375, -0.2913, -0.0651, +0.47656250, 0.00540000, 0.1920, -0.0449, 0.1169, -0.1122, +0.48046875, 0.00400000, -0.1041, 0.1946, 0.1357, -0.6278, diff --git a/expand_sequence.py b/expand_sequence.py new file mode 100644 index 0000000..90ef0be --- /dev/null +++ b/expand_sequence.py @@ -0,0 +1,104 @@ +import pandas as pd +bufr_tables = "./BUFR_TABLES/" +table_A = pd.read_csv(bufr_tables+'/BUFR_31_0_0_TableA_en.txt',sep=',', dtype='object') +table_B = pd.read_csv(bufr_tables+'/BUFRCREX_31_0_0_TableB_en.txt',sep=',', dtype='object') +table_B['BUFR_DataWidth_Bits'] = table_B['BUFR_DataWidth_Bits'].map(int) +table_B['BUFR_Scale'] = table_B['BUFR_Scale'].map(int) +table_B['BUFR_ReferenceValue'] = table_B['BUFR_ReferenceValue'].map(float) +table_C = pd.read_csv(bufr_tables+'/BUFR_31_0_0_TableC_en.txt',sep=',', dtype='object') +table_D = pd.read_csv(bufr_tables+'/BUFR_31_0_0_TableD_en.txt',sep=',', dtype='object') + +operators = { + '01':{'name':'change_data_width', 'field':'BUFR_DataWidth_Bits', 'value':0}, + '02':{'name':'change_scale', 'field':'BUFR_Scale', 'value':0}, + '03':{'name':'change_reference_value', 'field':'BUFR_ReferenceValue', 'value':0}, + '04':{'name':'add_associated_value', 'field':None, 'value':0}, + '05':{'name':'signify_character', 'field':None, 'value':0}, + '06':{'name':'signify_width', 'field':None, 'value':0}, + '07':{'name':'increase_scale_ref_width', 'field':None, 'value':0}, + '08':{'name':'change_width_of_ccitt', 'field':'BUFR_DataWidth_Bits', 'value':0}, +} + +def expand_sequence( descriptors, replicators ): + # change return value to pandas df + # FXY, kind, units, width, scale, offset + return_value = pd.DataFrame( columns = table_B.columns ) + ix = 0 + while ix < len( descriptors ): + if descriptors[ix][0] == '0': + # get values from table B and apply operators + tmpRow = table_B[ table_B['FXY'] == descriptors[ix] ].copy() + for op in operators: + if operators[ op ]['value'] > 0: + if op in ['01','02']: # add yyy - 128 bits / scale to field excluding code / flag tables and CCITT data + exclude = ['CCITT IA5','Code table', 'Flag table'] + tmpRow.loc[ (~ tmpRow['BUFR_Unit'].isin(exclude)), operators[ op ]['field'] ] = \ + tmpRow.loc[(~ tmpRow['BUFR_Unit'].isin(exclude)), operators[op]['field']] + operators[op]['value'] - 128 + if op == '08' : + tmpRow.loc[ tmpRow['BUFR_Unit'] == 'CCITT IA5' , operators[ op ]['field'] ] = \ + operators[op]['value'] * 8 + + return_value = pd.concat( [return_value, tmpRow] ) + ix += 1 + elif descriptors[ix][0] == '1': + nelements = int(descriptors[ix][1:3]) + nreplications = int( descriptors[ix][3:6]) + if nreplications == 0: + #next item is delayed descriptor, add to return value + return_value = pd.concat( [return_value, expand_sequence( [descriptors[ix+1]], replicators ) ] ) + nreplications = replicators.pop(0) + items = descriptors[(ix+2):(ix+nelements+2)] + for iy in range(nreplications): + return_value = pd.concat([return_value, expand_sequence( items , replicators )] ) + ix += nelements + 2 + else: + items = descriptors[(ix+1):(ix+nelements+1)] + for iy in range(nreplications): + return_value = pd.concat( [return_value, expand_sequence( items , replicators )]) + ix += nelements + 1 + elif descriptors[ix][0] == '2': + xx = descriptors[ix][1:3] + yyy = descriptors[ix][3:6] + operators[ xx ]['value'] = int( yyy ) + ix += 1 + else : #descriptors[ix][0] == '3' : + items = list( table_D[ table_D['FXY1'] == descriptors[ix]].FXY2 ) + return_value = pd.concat( [return_value, expand_sequence( items , replicators )]) + ix += 1 + return( return_value ) + +def pack_section( section ): + bits = '' + # encode section 0 and add to bits + for key in section: + print( section[key] ) + if section[key]['width'] > 0: + if section[key]['kind'] == 'CCITT IA5' : + word = ''.join( format( ord(x), 'b').zfill(8) for x in section[key]['value']) + word = word.rjust( section[key]['width'] * 8) + bits += word + elif section[key]['kind'] == 'bin': + # data already binary + bits += section[key]['value'] + else: + # should be int or list + if isinstance( section[key]['value'], int ): + if pd.np.isnan(section[key]['value']): + word = pow(2, section[key]['width'] * 8) - 1 + word = format(word, 'b').zfill(section[key]['width'] * 8) + else: + word = format(int(section[key]['value']), 'b').zfill(section[key]['width'] * 8) + bits += word + elif isinstance( section[key]['value'], list ): + assert key == 'descriptors' + for item in section[key]['value']: + F = int(item[0]) + XX = int(item[1:3]) + YYY = int(item[3:6]) + word = '' + word += format( F , 'b').zfill(2) + word += format( XX , 'b').zfill(6) + word += format( YYY, 'b').zfill(8) + bits += word + return( bits ) + diff --git a/waverider_first_five.json b/waverider_first_five.json new file mode 100644 index 0000000..b03a2de --- /dev/null +++ b/waverider_first_five.json @@ -0,0 +1,50 @@ +{ + "datafile":"waverider_test_data.txt", + "outputfile":"waverider_test_data.bin", + "section0":{ + "bufr":{"width":4,"value":"BUFR","kind":"CCITT IA5"}, + "length":{"width":3,"value":0,"kind":"int"}, + "version":{"width":1,"value":4,"kind":"int"} + }, + "section1" : { + "length": {"width":3, "value":22, "kind":"int"}, + "master_table": {"width":1, "value":0, "kind":"int"}, + "originating_centre": {"width":2, "value":65535, "kind":"int"}, + "sub_centre": {"width":2, "value":65535, "kind":"int"}, + "update_sequence": {"width":1, "value":0, "kind":"int"}, + "optional_section": {"width":1, "value":0, "kind":"int"}, + "data_category": {"width":1, "value":1, "kind":"int"}, + "international_sub_category": {"width":1, "value":6, "kind":"int"}, + "local_sub_category": {"width":1, "value":255, "kind":"int"}, + "master_table_verions": {"width":1, "value":0, "kind":"int"}, + "local_table_version": {"width":1, "value":255, "kind":"int"}, + "year": {"width":2, "value":2019, "kind":"int"}, + "month": {"width":1, "value":2, "kind":"int"}, + "day": {"width":1, "value":27, "kind":"int"}, + "hour": {"width":1, "value":23, "kind":"int"}, + "minute": {"width":1, "value":6, "kind":"int"}, + "second": {"width":1, "value":0, "kind":"int"}, + "optional": {"width":0, "value":null, "kind":"int"} + }, + "section2": { + "length": {"width":3, "value":4, "kind":"int"}, + "zero": {"width":1, "value":0, "kind":"int"}, + "local_use": {"width":0, "value":null, "kind":"int"} + }, + "section3":{ + "length": {"width":3, "value":9, "kind":"int"}, + "zero": {"width":1, "value":0, "kind":"int"}, + "number_subsets": {"width":2, "value":1, "kind":"int"}, + "flags": {"width":1, "value":128, "kind":"int"}, + "descriptors": {"width":2, "value":["315010"], "kind":"list"} + }, + "section4" : { + "length": {"width":3, "value":null, "kind":"int"}, + "zero": {"width":1, "value":0, "kind":"int"}, + "data": {"width":0, "value":null, "kind":"bin"} + }, + "section5": { + "sevens":{"width":4, "value":"7777", "kind":"CCITT IA5"} + }, + "payload":null +} diff --git a/waverider_test_data.bin b/waverider_test_data.bin new file mode 100644 index 0000000..df5fc7a Binary files /dev/null and b/waverider_test_data.bin differ diff --git a/waverider_test_data.txt b/waverider_test_data.txt new file mode 100644 index 0000000..e438402 --- /dev/null +++ b/waverider_test_data.txt @@ -0,0 +1,65 @@ +freq,bandwidth,energy,dmean,a1,b1,a2,b2,check_factor +0.025,0.005,0.0001,36,0.1056,0.0776,-0.3223,0.592,2.55 +0.03,0.005,0.0002,57,0.031,0.0484,-0.0301,0.7026,2.55 +0.035,0.005,0.0008,214,-0.319,-0.2111,-0.0244,0.2374,2.55 +0.04,0.005,0.001,192,-0.4395,-0.0967,0.0221,0.6923,2.55 +0.045,0.005,0.0018,69,0.0607,0.1553,-0.3969,0.6557,2.55 +0.05,0.005,0.0028,191,-0.3054,-0.0594,0.2324,0.1339,2.55 +0.055,0.005,0.0179,209,-0.4874,-0.2733,-0.0749,0.5403,2.07 +0.06,0.005,0.0583,206,-0.6751,-0.3362,0.0831,0.4243,1.51 +0.065,0.005,0.0517,202,-0.6666,-0.2727,0.2719,0.2946,1.62 +0.07,0.005,0.3578,211,-0.7937,-0.471,0.4624,0.8023,1.29 +0.075,0.005,0.3286,208,-0.7986,-0.4224,0.5436,0.6388,1.33 +0.08,0.005,0.0759,212,-0.6807,-0.4269,0.2098,0.4518,1.53 +0.085,0.005,0.1768,211,-0.743,-0.441,0.3926,0.589,1.24 +0.09,0.005,0.1405,206,-0.7859,-0.3913,0.5619,0.5106,1.39 +0.095,0.005,0.0985,222,-0.559,-0.5022,0.1194,0.2524,1.27 +0.1013,0.0075,0.0591,221,-0.4691,-0.4011,0.0978,-0.1577,1.39 +0.11,0.01,0.1005,278,0.1144,-0.7953,-0.4749,-0.3859,1.46 +0.12,0.01,0.1099,277,0.0911,-0.7663,-0.3595,-0.3419,1.29 +0.13,0.01,0.1336,284,0.2161,-0.879,-0.6524,-0.434,1.21 +0.14,0.01,0.0577,285,0.2235,-0.8217,-0.5003,-0.4593,1.25 +0.15,0.01,0.0619,284,0.2073,-0.843,-0.5463,-0.4093,1.34 +0.16,0.01,0.0588,285,0.2351,-0.8642,-0.5768,-0.4024,1.19 +0.17,0.01,0.0266,273,0.0375,-0.8377,-0.5152,-0.1148,1.19 +0.18,0.01,0.0178,277,0.0986,-0.8293,-0.5519,-0.2276,1.2 +0.19,0.01,0.023,277,0.0991,-0.8337,-0.5998,-0.2954,1.1 +0.2,0.01,0.0253,277,0.0916,-0.7702,-0.376,-0.2655,1.19 +0.21,0.01,0.0124,263,-0.0968,-0.7575,-0.4719,0.0399,1.2 +0.22,0.01,0.0058,261,-0.1189,-0.7782,-0.5917,0.1954,1.29 +0.23,0.01,0.0039,281,0.1287,-0.6623,-0.3972,-0.2995,1.21 +0.24,0.01,0.0048,261,-0.1145,-0.7496,-0.4286,0.128,1.13 +0.25,0.01,0.0032,275,0.0628,-0.667,-0.2201,-0.0974,1.24 +0.26,0.01,0.003,268,-0.0216,-0.7456,-0.4245,0.0011,1.11 +0.27,0.01,0.0017,259,-0.1342,-0.6596,-0.3201,0.1656,1.22 +0.28,0.01,0.0017,246,-0.2636,-0.5877,-0.2287,0.3508,1.12 +0.29,0.01,0.0018,229,-0.4419,-0.5078,0.1456,0.2431,1.14 +0.3,0.01,0.0025,218,-0.4746,-0.367,0.2568,0.01,1.13 +0.31,0.01,0.0024,222,-0.4463,-0.401,0.3427,0.2834,1.03 +0.32,0.01,0.0016,268,-0.0103,-0.3561,-0.0433,-0.0718,1.1 +0.33,0.01,0.0017,263,-0.056,-0.4383,-0.0574,0.0896,1.08 +0.34,0.01,0.0018,232,-0.3722,-0.4727,-0.0056,0.3563,1.22 +0.35,0.01,0.0021,228,-0.4595,-0.5026,0.1241,0.5837,1.08 +0.36,0.01,0.002,236,-0.3546,-0.5257,0.0669,0.4862,1.14 +0.37,0.01,0.001,278,0.0566,-0.3936,0.0752,0.0628,1.33 +0.38,0.01,0.0013,289,0.1671,-0.4734,0.0402,-0.1557,1.07 +0.39,0.01,0.0012,289,0.2014,-0.5707,-0.065,-0.3986,1.15 +0.4,0.01,0.0018,289,0.2468,-0.6995,-0.3648,-0.4095,0.88 +0.41,0.01,0.0016,284,0.172,-0.6994,-0.212,-0.252,0.91 +0.42,0.01,0.001,299,0.2436,-0.4345,-0.0349,-0.2854,1.02 +0.43,0.01,0.0007,308,0.255,-0.3297,0.1737,-0.2711,1.13 +0.44,0.01,0.0007,263,-0.0745,-0.5829,-0.249,0.1757,1.08 +0.45,0.01,0.0005,274,0.0269,-0.3881,-0.0108,-0.0074,1.01 +0.46,0.01,0.0007,285,0.1634,-0.6005,-0.1792,-0.069,0.95 +0.47,0.01,0.0007,270,-0.0027,-0.6121,-0.2293,0.1348,0.8 +0.48,0.01,0.0007,277,0.0749,-0.6297,-0.1913,0.0684,1 +0.49,0.01,0.0008,292,0.2433,-0.5946,-0.0865,-0.2575,0.76 +0.5,0.01,0.0005,288,0.1608,-0.4939,-0.1458,0.104,0.99 +0.51,0.01,0.0007,287,0.1992,-0.6673,-0.3169,-0.0723,0.85 +0.52,0.01,0.0009,292,0.2738,-0.6693,-0.2296,-0.1927,0.7 +0.53,0.01,0.0009,301,0.3435,-0.5789,-0.138,-0.0693,0.74 +0.54,0.01,0.0007,296,0.2766,-0.5556,-0.0237,0.0399,0.83 +0.55,0.01,0.001,304,0.3689,-0.5573,-0.0396,-0.2633,0.66 +0.56,0.01,0.0012,309,0.4326,-0.5319,0.0618,-0.336,0.65 +0.57,0.01,0.0013,305,0.4045,-0.5798,-0.2622,-0.3905,0.57 +0.58,0.01,0.0009,298,0.2612,-0.4939,-0.0497,-0.1528,0.63 \ No newline at end of file