First commit

BUFR_TABLES/BUFRCREX_31_0_0_TableB_en.txt

@@ -0,0 +1,10 @@
+"No","ClassNo","ClassName_en","FXY","ElementName_en","Note_en","BUFR_Unit","BUFR_Scale","BUFR_ReferenceValue","BUFR_DataWidth_Bits","CREX_Unit","CREX_Scale","CREX_DataWidth_Char","Status"
+1225.00,"22","Oceanographic elements","022069","Spectral wave density",,"m2/Hz","3","0","22","m2/Hz","3","7","Operational"
+1236.00,"22","Oceanographic elements","022080","Waveband central frequency",,"Hz","3","0","10","Hz","3","4","Operational"
+1252.00,"22","Oceanographic elements","022096","Spectral band width",,"/s","3","0","4","/s","3","2","Operational"
+1560.00,"31","Data description operator qualifiers","031001","Delayed descriptor replication factor",,"Numeric","0","0","8",,,,"Operational"
+1734.00,"42","Oceanographic elements","042011","a1 coefficient of the directional Fourier series",,"Numeric","4","-20000","15","Numeric","4","6","Test"
+1735.00,"42","Oceanographic elements","042012","b1 coefficient of the directional Fourier series",,"Numeric","4","-20000","15","Numeric","4","6","Test"
+1736.00,"42","Oceanographic elements","042013","a2 coefficient of the directional Fourier series",,"Numeric","4","-20000","15","Numeric","4","6","Test"
+1737.00,"42","Oceanographic elements","042014","b2 coefficient of the directional Fourier series",,"Numeric","4","-20000","15","Numeric","4","6","Test"
+1738.00,"42","Oceanographic elements","042015","Check factor K",,"Numeric","2","0","12","Numeric","2","4","Test"

BUFR_TABLES/BUFR_31_0_0_TableA_en.txt

@@ -0,0 +1,34 @@
+"No","CodeFigure","Meaning_en","Status"
+1.00,"0","Surface data - land","Operational"
+2.00,"1","Surface data - sea","Operational"
+3.00,"2","Vertical soundings (other than satellite)","Operational"
+4.00,"3","Vertical soundings (satellite)","Operational"
+5.00,"4","Single level upper-air data (other than satellite)","Operational"
+6.00,"5","Single level upper-air data (satellite)","Operational"
+7.00,"6","Radar data","Operational"
+8.00,"7","Synoptic features","Operational"
+9.00,"8","Physical/chemical constituents","Operational"
+10.00,"9","Dispersal and transport","Operational"
+11.00,"10","Radiological data","Operational"
+12.00,"11","BUFR tables, complete replacement or update","Operational"
+14.00,"12","Surface data (satellite)","Operational"
+15.00,"13","Forecasts","Operational"
+16.00,"14","Warnings","Operational"
+17.00,"15-19","Reserved","Operational"
+19.00,"20","Status information","Operational"
+20.00,"21","Radiances (satellite measured)","Operational"
+21.00,"22","Radar (satellite) but not altimeter and scatterometer","Operational"
+22.00,"23","Lidar (satellite)","Operational"
+23.00,"24","Scatterometry (satellite)","Operational"
+24.00,"25","Altimetry (satellite)","Operational"
+25.00,"26","Spectrometry (satellite)","Operational"
+26.00,"27","Gravity measurement (satellite)","Operational"
+27.00,"28","Precision orbit (satellite)","Operational"
+28.00,"29","Space environment (satellite)","Operational"
+29.00,"30","Calibration datasets (satellite)","Operational"
+30.00,"31","Oceanographic data","Operational"
+31.00,"32-100","Reserved","Operational"
+32.00,"101","Image data (satellite)","Operational"
+33.00,"102-239","Reserved","Operational"
+34.00,"240-254","For experimental use","Operational"
+35.00,"255","Other category","Operational"

BUFR_TABLES/BUFR_31_0_0_TableC_en.txt

@@ -0,0 +1,29 @@
+"No","FXY","OperatorName_en","OperationDefinition_en","Note_en","Status"
+1.00,"201YYY","Change data width","Add (YYY-128) bits to the data width given for each data element in Table B, other than CCITT IA5 (character) data, code or flag tables.",,"Operational"
+3.00,"202YYY","Change scale","Add YYY-128 to the scale for each data element in Table B, other than CCITT IA5 (character) data, code or flag tables.",,"Operational"
+5.00,"203YYY","Change reference values","Subsequent element descriptors define new reference values for corresponding Table B entries. Each new reference value is represented by YYY bits in the Data section. Definition of new reference values is concluded by coding this operator with YYY = 255. Negative reference values shall be represented by a positive integer with the left-most bit (bit 1) set to 1.",,"Operational"
+6.00,"204YYY","Add associated field","Precede each data element with YYY bits of information.  This operation associates a data field (e.g. quality control information) of YYY bits with each data element.",,"Operational"
+7.00,"205YYY","Signify character","YYY characters (CCITT International Alphabet No. 5) are inserted as a data field of YYY x 8 bits in length.",,"Operational"
+9.00,"206YYY","Signify data width for the immediately following local descriptor","YYY bits of data are described by the immediately following descriptor.",,"Operational"
+10.00,"207YYY","Increase scale, reference value and data width","For Table B elements, which are not CCITT IA5 (character data), code tables, or flag tables:   1. Add YYY to the existing scale factor   2. Multiply the existing reference value by 10**YYY   3. Calculate ((10 x YYY) + 2) ÷ 3, disregard any fractional remainder and add the result to the existing bit width.",,"Operational"
+12.00,"208YYY","Change width of CCITT IA5 field","YYY characters from CCITT International Alphabet No. 5 (representing YYY x 8 bits in length) replace the specified data width given for each CCITT IA5 element in Table B.",,"Operational"
+14.00,"221YYY","Data not present","Data values present in Section 4 (Data section) corresponding to the following YYY descriptors shall be limited to data from Classes 01-09, and Class 31.",,"Operational"
+15.00,"222000","Quality information follows","The values of Class 33 elements which follow relate to the data defined by the data present bit-map.",,"Operational"
+16.00,"223000","Substituted values operator","The substituted values which follow relate to the data defined by the data present bit-map.",,"Operational"
+17.00,"223255","Substituted values marker operator","This operator shall signify a data item containing a substituted value; the element descriptor for the substituted value is obtained by the application of the data present bit-map associated with the substituted values operator.",,"Operational"
+18.00,"224000","First-order statistical values follow","The statistical values which follow relate to the data defined by the data present bit-map.",,"Operational"
+19.00,"224255","First-order statistical values marker operator","This operator shall signify a data item containing a first-order statistical value of the type indicated by the preceding 0 08 023 element descriptor; the element descriptor to which the first-order statistic relates is obtained by the application of the data present bit-map associated with the first-order statistical values follow operator; first-order statistical values shall be represented as defined by this element descriptor.",,"Operational"
+20.00,"225000","Difference statistical values follow","The statistical values which follow relate to the data defined by the data present bit-map.",,"Operational"
+21.00,"225255","Difference statistical values marker operator","This operator shall signify a data item containing a difference statistical value of the type indicated by the preceding 0 08 024 element descriptor; the element descriptor to which the difference statistical value relates is obtained by the application of the data present bit-map associated with the difference statistical values follow operator; difference statistical values shall be represented as defined by this element descriptor, but with a reference value of -2**n and a data width of (n+1), where n is the data width given by the original descriptor. This special reference value allows the statistical difference values to be centred around zero.",,"Operational"
+22.00,"232000","Replaced/retained values follow","The replaced/retained values which follow relate to the data defined by the data present bit-map.",,"Operational"
+23.00,"232255","Replaced/retained value marker operator","This operator shall signify a data item containing the original of an element which has been replaced by a substituted value.  The element descriptor for the retained value is obtained by the application of the data present bit-map associated with the substituted values operator.",,"Operational"
+24.00,"235000","Cancel backward data reference","This operator terminates all previously defined back-ward reference and cancels any previously defined data present bit-map; it causes the next data present bit-map to refer to the data descriptors which immediately precede the operator to which it relates.",,"Operational"
+25.00,"236000","Define data present bit-map","This operator defines the data present bit-map which follows for possible re-use; only one data present bit-map may be defined between this operator and the cancel use defined data present bit-map operator.",,"Operational"
+26.00,"237000","Use defined data present bit-map","This operator causes the defined data present bit-map to be used again.",,"Operational"
+27.00,"237255","Cancel use defined data present bit-map","This operator cancels the re-use of the defined data present bit-map.",,"Operational"
+28.00,"241000","Define event","This operator denotes the beginning of the definition of an event","(see Note 19).","Operational"
+30.00,"241255","Cancel define event","This operator denotes the conclusion of the event definition that was begun via the previous 2 41 000 operator.",,"Operational"
+32.00,"242000","Define conditioning event","This operator denotes the beginning of the definition of a conditioning event","(see Note 19).","Operational"
+34.00,"242255","Cancel define conditioning event","This operator denotes the conclusion of the conditioning event definition that was begun via the previous 2 42 000 operator.",,"Operational"
+36.00,"243000","Categorical forecast values follow","The values which follow are categorical forecast values","(see Note 20).","Operational"
+38.00,"243255","Cancel categorical forecast values follow","This operator denotes the conclusion of the definition of categorical forecast values that was begun via the previous 2 43 000 operator.",,"Operational"

BUFR_TABLES/BUFR_31_0_0_TableD_en.txt

@@ -0,0 +1,15 @@
+"No","Category","CategoryOfSequences_en","FXY1","Title_en","SubTitle_en","FXY2","ElementName_en","ElementDescription_en","Note_en","Status"
+8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"112000","Delayed replication of 12 descriptors",,,"Test"
+8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"031001","Delayed descriptor replication factor",,,"Test"
+8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"201144","Add YYY-128 to the width for each data element in Table B, other than CCITT IA5 (character) data, code or flag tables.",,,"Test"
+8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"202133","Add YYY-128 to the scale for each data element in Table B, other than CCITT IA5 (character) data, code or flag tables.",,,"Test"
+8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"022080","Waveband central frequency (Hz)",,,"Test"
+8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"022096","Spectral band width (Hz)",,,"Test"
+8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"022069","Spectral wave density (m2 Hz-1)",,,"Test"
+8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"202000","Add YYY-128 to the scale for each data element in Table B, other than CCITT IA5 (character) data, code or flag tables.",,,"Test"
+8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"201000","Add YYY-128 to the width for each data element in Table B, other than CCITT IA5 (character) data, code or flag tables.",,,"Test"
+8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"042011","a1 coefficient of the directional Fourier series",,,"Test"
+8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"042012","b1 coefficient of the directional Fourier series",,,"Test"
+8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"042013","a2 coefficient of the directional Fourier series",,,"Test"
+8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"042014","b2 coefficient of the directional Fourier series",,,"Test"
+8000.00,"15","Oceanographic report sequences","315010","First five Fourier components of the wave spectrum",,"042015","Check factor K",,,"Test"

README.md

@@ -1 +1,8 @@
 # BUFR_python_waves
+
+Development code used to encode first five Fourier components for wave spectrum in BUFR.
+
+Usage:
+   python3 bufr_wave.py
+
+Configuration information is read from waverider_first_five.json

bufr_wave.py

@@ -0,0 +1,112 @@
+from expand_sequence import *
+import json
+import sys
+from bitarray import bitarray
+def main( argv ):
+    pd.set_option('display.max_columns', 150)
+    pd.set_option('display.max_rows', 200)
+    pd.set_option('display.width', 300 )
+
+    # Load message headers / sections
+    msg_file = 'waverider_first_five.json'
+    with open(msg_file) as bufrmsg:
+        msg = json.load( bufrmsg )
+
+    # ----------------------------------------------------------------------------------
+    # need to do the next code block better, at the moment every subset has to have the
+    # same set of delayed replicators. In future need to loop over different subsets
+    # separately.
+    # ----------------------------------------------------------------------------------
+
+    # read in data file
+    datain = pd.read_csv(msg['datafile'])
+    replicators = [ datain.shape[0] ]
+    nsubsets = 1
+    # get descriptors to pack
+    descriptors = expand_sequence( msg['section3']['descriptors']['value'], replicators )
+    descriptors.reset_index( inplace=True )
+    print(descriptors)
+    # calculate message length
+    data_length = descriptors.BUFR_DataWidth_Bits.sum() * nsubsets
+    if data_length % 8 > 0:
+        data_length = data_length + (8 - data_length % 8)
+    print(data_length)
+    data_length = int( data_length / 8)
+    # ----------------------------------------------------------------------------------
+    msg['section4']['length']['value'] = data_length + 4
+    msg['section4']['data']['width'] = data_length
+    msg_length = 0
+    msg_length += 8
+    msg_length += msg['section1']['length']['value']
+    if msg['section1']['optional_section']['value'] == 1:
+        msg_length += msg['section2']['length']['value']
+    msg_length += msg['section3']['length']['value']
+    msg_length += msg['section4']['length']['value']
+    msg_length += 4
+
+
+    # Set length in section 0
+    msg['section0']['length']['value'] = msg_length
+
+    # Pack data section
+
+    # first we need to transpose and unstack our data frame to single series
+    # get columns to write
+    towrite = datain.loc[:,['freq','bandwidth','energy','a1','b1','a2','b2','check_factor']].copy()
+    towrite = towrite.values.flatten()
+    # add number of repeats at start
+    replicators = [ datain.shape[0] ]
+    towrite = pd.np.insert( towrite, 0, replicators[0])
+    # now convert back to data frame
+    towrite = pd.DataFrame({'data':towrite}).transpose()
+    # ----------------------------------------------------------------------------------
+    # move to pack_section function
+    # ----------------------------------------------------------------------------------
+    rsum = 0
+    for i in range( towrite.shape[1] ):
+        scale  = descriptors.loc[i,'BUFR_Scale']
+        width  = descriptors.loc[i,'BUFR_DataWidth_Bits']
+        rsum += width
+        offset = descriptors.loc[i,'BUFR_ReferenceValue']
+        units  = descriptors.loc[i,'BUFR_Unit']
+        msng   = pow(2, width) - 1
+        fld = towrite.columns[i]
+        if units == 'CCITT IA5':
+            towrite[fld] = towrite[fld].apply( lambda x: ''.join( format( ord(y), 'b').zfill(8) for y in x.rjust( int(width/8)) ))
+            towrite[fld] = towrite[fld].apply( lambda x: x.zfill( width ) )
+        else:
+            towrite[fld] = towrite[fld].apply( lambda x: int(round(x * pow(10,scale) - offset)))
+            towrite[fld] = towrite[fld].fillna( msng )
+            towrite[fld] = towrite[fld].apply( lambda x: format( int(x), 'b').zfill( width ) )
+
+    bitsOut = towrite.apply( lambda x: x.sum(), axis = 1).sum()
+    nbytes = int( len(bitsOut) / 8 )
+    pack = len(bitsOut) % 8
+    if pack > 0:
+        bitsOut = bitsOut + ''.zfill( 8 - pack )
+    nbytes = int( len(bitsOut) / 8 )
+
+    assert nbytes == data_length
+
+    msg['section4']['data']['value'] = bitsOut
+
+    # Write to file
+
+    bitsOut = ''
+    # pack sections
+    bitsOut += pack_section( msg['section0'] )
+    bitsOut += pack_section( msg['section1'] )
+    if msg['section1']['optional_section']['value'] == 1:
+        bitsOut += pack_section( msg['section2'] )
+    bitsOut += pack_section( msg['section3'] )
+    bitsOut += pack_section( msg['section4'] )
+    bitsOut += pack_section( msg['section5'] )
+
+    # now write to file
+    #file_out = open('waves_first_five.bin', 'wb')
+    file_out = open(msg['outputfile'], 'wb')
+    bitarray( bitsOut ).tofile(file_out)
+    file_out.close()
+
+if __name__ == '__main__':
+    main(sys.argv[1:])