encoding fix

README.md

@@ -26,20 +26,25 @@ The processed output is available in the `policies-out.csv` file.
 ### Options
 #### Python
 ```
-$ python fgpoliciestocsv.py  -h
+$ python fgpoliciestocsv.py -h
 Usage: fgpoliciestocsv.py [options]
 
 Options:
   -h, --help            show this help message and exit
-  -i INPUT_FILE, --input-file=INPUT_FILE
-                        <INPUT_FILE>: Fortigate configuration file. Ex:
+
+  Main parameters:
+    -i INPUT_FILE, --input-file=INPUT_FILE
+                        Partial or full Fortigate configuration file. Ex:
                         fgfw.cfg
-  -o OUTPUT_FILE, --output-file=OUTPUT_FILE
-                        <OUTPUT_FILE>: output csv file (default './policies-
-                        out.csv')
-  -n, --newline         <NEWLINE> : insert a newline between each policy for
-                        better readability
-  -s, --skip-header     <SKIP_HEADER> : do not print the csv header
+    -o OUTPUT_FILE, --output-file=OUTPUT_FILE
+                        Output csv file (default ./policies-out.csv)
+    -s, --skip-header   Do not print the csv header
+    -n, --newline       Insert a newline between each policy for better
+                        readability
+    -d DELIMITER, --delimiter=DELIMITER
+                        CSV delimiter (default ";")
+    -e ENCODING, --encoding=ENCODING
+                        Input file encoding (default "utf8")
 ```
 
 #### Perl
@@ -79,7 +84,7 @@ For a policy, an empty value in the `action` column might mean `deny`, as this i
 
 Requirements
 ------------
-* Python >= 2.4
+* Python >= 2.7
 * Perl
 
 Copyright and license

fgaddressestocsv.py

@@ -23,8 +23,8 @@
 from __future__ import division
 from __future__ import print_function
 
-from codecs import open
-from os import path 
+from os import path
+import io
 import sys
 import re
 import csv
@@ -43,11 +43,12 @@
 main_grp.add_option('-s', '--skip-header', help='Do not print the csv header', action='store_true', default=False)
 main_grp.add_option('-n', '--newline', help='Insert a newline between each group for better readability', action='store_true', default=False)
 main_grp.add_option('-d', '--delimiter', help='CSV delimiter (default ";")', default=';')
+main_grp.add_option('-e', '--encoding', help='Input file encoding (default "utf8")', default='utf8')
 parser.option_groups.extend([main_grp])
 
 # Python 2 and 3 compatibility
 if (sys.version_info < (3, 0)):
-    fd_read_options = 'rb'
+    fd_read_options = 'r'
     fd_write_options = 'wb'
 else:
     fd_read_options = 'r'
@@ -87,7 +88,7 @@ def parse(options):
 
     order_keys = []
 
-    with open(options.input_file, mode=fd_read_options) as fd_input:
+    with io.open(options.input_file, mode=fd_read_options, encoding=options.encoding) as fd_input:
         for line in fd_input:
             line = line.strip()
 
@@ -131,7 +132,7 @@ def generate_csv(results, keys, options):
         Generate a plain csv file
     """
     if results and keys:
-        with open(options.output_file, mode=fd_write_options) as fd_output:
+        with io.open(options.output_file, mode=fd_write_options) as fd_output:
             spamwriter = csv.writer(fd_output, delimiter=options.delimiter, quoting=csv.QUOTE_ALL, lineterminator='\n')
 
             if not(options.skip_header):

fggroupstocsv.py

@@ -23,8 +23,8 @@
 from __future__ import division
 from __future__ import print_function
 
-from codecs import open
-from os import path 
+from os import path
+import io
 import sys
 import re
 import csv
@@ -43,11 +43,12 @@
 main_grp.add_option('-s', '--skip-header', help='Do not print the csv header', action='store_true', default=False)
 main_grp.add_option('-n', '--newline', help='Insert a newline between each group for better readability', action='store_true', default=False)
 main_grp.add_option('-d', '--delimiter', help='CSV delimiter (default ";")', default=';')
+main_grp.add_option('-e', '--encoding', help='Input file encoding (default "utf8")', default='utf8')
 parser.option_groups.extend([main_grp])
 
 # Python 2 and 3 compatibility
 if (sys.version_info < (3, 0)):
-    fd_read_options = 'rb'
+    fd_read_options = 'r'
     fd_write_options = 'wb'
 else:
     fd_read_options = 'r'
@@ -87,7 +88,7 @@ def parse(options):
 
     order_keys = []
 
-    with open(options.input_file, mode=fd_read_options) as fd_input:
+    with io.open(options.input_file, mode=fd_read_options, encoding=options.encoding) as fd_input:
         for line in fd_input:
             line = line.strip()
 
@@ -129,7 +130,7 @@ def generate_csv(results, keys, options):
         Generate a plain ';' separated csv file
     """
     if results and keys:
-        with open(options.output_file, mode=fd_write_options) as fd_output:
+        with io.open(options.output_file, mode=fd_write_options) as fd_output:
             spamwriter = csv.writer(fd_output, delimiter=options.delimiter, quoting=csv.QUOTE_ALL, lineterminator='\n')
 
             if not(options.skip_header):

fgpoliciestocsv.py

@@ -23,8 +23,8 @@
 from __future__ import division
 from __future__ import print_function
 
-from codecs import open
 from os import path 
+import io
 import sys
 import re
 import csv
@@ -43,11 +43,12 @@
 main_grp.add_option('-s', '--skip-header', help='Do not print the csv header', action='store_true', default=False)
 main_grp.add_option('-n', '--newline', help='Insert a newline between each policy for better readability', action='store_true', default=False)
 main_grp.add_option('-d', '--delimiter', help='CSV delimiter (default ";")', default=';')
+main_grp.add_option('-e', '--encoding', help='Input file encoding (default "utf8")', default='utf8')
 parser.option_groups.extend([main_grp])
 
 # Python 2 and 3 compatibility
 if (sys.version_info < (3, 0)):
-    fd_read_options = 'rb'
+    fd_read_options = 'r'
     fd_write_options = 'wb'
 else:
     fd_read_options = 'r'
@@ -90,7 +91,7 @@ def parse(options):
 
     order_keys = []
 
-    with open(options.input_file, mode=fd_read_options) as fd_input:
+    with io.open(options.input_file, mode=fd_read_options, encoding=options.encoding) as fd_input:
         for line in fd_input:
             line = line.strip()
 
@@ -111,9 +112,9 @@ def parse(options):
             if in_policy_block:
                 if p_policy_number.search(line) and not(skip_ssl_vpn_policy_block):
                     policy_number = p_policy_number.search(line).group('policy_number')
-                    policy_elem['id'] = policy_number
+                    policy_elem[u'id'] = policy_number
                     if not('id' in order_keys):
-                        order_keys.append('id')
+                        order_keys.append(u'id')
 
                 # We match a setting
                 if p_policy_set.search(line) and not(skip_ssl_vpn_policy_block):
@@ -151,7 +152,7 @@ def generate_csv(results, keys, options):
         Generate a plain csv file
     """
     if results and keys:
-        with open(options.output_file, mode=fd_write_options) as fd_output:
+        with io.open(options.output_file, mode=fd_write_options) as fd_output:
             spamwriter = csv.writer(fd_output, delimiter=options.delimiter, quoting=csv.QUOTE_ALL, lineterminator='\n')
 
             if not(options.skip_header):

fgservicestocsv.py

@@ -23,8 +23,8 @@
 from __future__ import division
 from __future__ import print_function
 
-from codecs import open
 from os import path 
+import io
 import sys
 import re
 import csv
@@ -43,11 +43,12 @@
 main_grp.add_option('-s', '--skip-header', help='Do not print the csv header', action='store_true', default=False)
 main_grp.add_option('-n', '--newline', help='Insert a newline between each group for better readability', action='store_true', default=False)
 main_grp.add_option('-d', '--delimiter', help='CSV delimiter (default ";")', default=';')
+main_grp.add_option('-e', '--encoding', help='Input file encoding (default "utf8")', default='utf8')
 parser.option_groups.extend([main_grp])
 
 # Python 2 and 3 compatibility
 if (sys.version_info < (3, 0)):
-    fd_read_options = 'rb'
+    fd_read_options = 'r'
     fd_write_options = 'wb'
 else:
     fd_read_options = 'r'
@@ -87,7 +88,7 @@ def parse(options):
 
     order_keys = []
 
-    with open(options.input_file, mode=fd_read_options) as fd_input:
+    with io.open(options.input_file, mode=fd_read_options, encoding=options.encoding) as fd_input:
         for line in fd_input:
             line = line.strip()
 
@@ -131,7 +132,7 @@ def generate_csv(results, keys, options):
         Generate a plain ';' separated csv file
     """
     if results and keys:
-        with open(options.output_file, mode=fd_write_options) as fd_output:
+        with io.open(options.output_file, mode=fd_write_options) as fd_output:
             spamwriter = csv.writer(fd_output, delimiter=options.delimiter, quoting=csv.QUOTE_ALL, lineterminator='\n')
 
             if not(options.skip_header):