Encoding

- Add option `--encoding` to control file input and output encoding

Author: jelu

dsc_datatool/__init__.py

@@ -49,6 +49,7 @@ def process(self, datasets)
 generators = {}
 transformers = {}
 process_dataset = {}
+encoding = 'utf-8'
 
 
 class Dataset(object):
@@ -339,6 +340,8 @@ def _process(datasets, generators, transformers, outputs):
         help='Set the special DSC skipped key. (default to "-:SKIPPED:-")')
     parser.add_argument('--skipped-sum-key', nargs=1, default='-:SKIPPED_SUM:-',
         help='Set the special DSC skipped sum key. (default to "-:SKIPPED_SUM:-")')
+    parser.add_argument('--encoding', nargs=1, default='utf-8',
+        help='Encoding to use for all files, default utf-8.')
     parser.add_argument('-v', '--verbose', action='count', default=0,
         help='Increase the verbose level, can be given multiple times.')
     parser.add_argument('-V', '--version', action='version', version='%(prog)s v'+__version__,

dsc_datatool/generator/client_subnet_authority.py

@@ -13,7 +13,7 @@
 from urllib.request import Request, urlopen
 from io import StringIO
 
-from dsc_datatool import Generator, Dataset, Dimension, args
+from dsc_datatool import Generator, Dataset, Dimension, args, encoding
 
 
 _whois2rir = {
@@ -97,7 +97,7 @@ def __init__(self, opts):
             if not isinstance(csvs, list):
                 csvs = [ csvs ]
             for file in csvs:
-                with open(file, newline='') as csvfile:
+                with open(file, newline='', encoding=encoding) as csvfile:
                     self._read(csvfile)
         elif opts.get('fetch', 'no').lower() == 'yes':
             urls = opts.get('url', [ urlv4, urlv6 ])

dsc_datatool/input/dat.py

@@ -9,7 +9,7 @@
 
 import re
 
-from dsc_datatool import Input, Dataset, Dimension, process_dataset
+from dsc_datatool import Input, Dataset, Dimension, process_dataset, encoding
 
 
 _dataset1d = [
@@ -78,7 +78,7 @@ def process(self, dir):
 
     def process1d(self, file, name):
         datasets = []
-        with open(file, 'r') as f:
+        with open(file, 'r', encoding=encoding) as f:
             for l in f.readlines():
                 if re.match(r'^#', l):
                     continue
@@ -103,7 +103,7 @@ def process1d(self, file, name):
 
     def process2d(self, file, name, field):
         datasets = []
-        with open(file, 'r') as f:
+        with open(file, 'r', encoding=encoding) as f:
             for l in f.readlines():
                 if re.match(r'^#', l):
                     continue
@@ -135,7 +135,7 @@ def process2d(self, file, name, field):
 
     def process3d(self, file, name, first, second):
         datasets = []
-        with open(file, 'r') as f:
+        with open(file, 'r', encoding=encoding) as f:
             for l in f.readlines():
                 if re.match(r'^#', l):
                     continue

dsc_datatool/output/influxdb.py

@@ -11,7 +11,7 @@
 import sys
 import atexit
 
-from dsc_datatool import Output, args
+from dsc_datatool import Output, args, encoding
 
 
 _re = re.compile(r'([,=\s])')
@@ -67,9 +67,9 @@ def __init__(self, opts):
         append = opts.get('append', False)
         if file:
             if append:
-                self.fh = open(file, 'a')
+                self.fh = open(file, 'a', encoding=encoding)
             else:
-                self.fh = open(file, 'w')
+                self.fh = open(file, 'w', encoding=encoding)
             atexit.register(self.close)
         else:
             self.fh = sys.stdout

dsc_datatool/output/prometheus.py

@@ -11,7 +11,7 @@
 import sys
 import atexit
 
-from dsc_datatool import Output, args
+from dsc_datatool import Output, args, encoding
 
 
 _re = re.compile(r'([\\\n"])')
@@ -52,9 +52,9 @@ def __init__(self, opts):
         append = opts.get('append', False)
         if file:
             if append:
-                self.fh = open(file, 'a')
+                self.fh = open(file, 'a', encoding=encoding)
             else:
-                self.fh = open(file, 'w')
+                self.fh = open(file, 'w', encoding=encoding)
             atexit.register(self.close)
         else:
             self.fh = sys.stdout

dsc_datatool/transformer/labler.py

@@ -9,7 +9,7 @@
 
 import yaml
 
-from dsc_datatool import Transformer
+from dsc_datatool import Transformer, encoding
 
 
 def _process(label, d):
@@ -43,7 +43,7 @@ def __init__(self, opts):
         Transformer.__init__(self, opts)
         if not 'yaml' in opts:
             raise Exception('yaml=file option required')
-        f = open(opts.get('yaml'), 'r')
+        f = open(opts.get('yaml'), 'r', encoding=encoding)
         try:
             self.label = yaml.full_load(f)
         except AttributeError:

tests/test.sh

@@ -55,3 +55,21 @@ dsc-datatool \
 
 sort -s "$base/test.gold3" > "$base/test.gold3.tmp"
 diff -u "$base/test.gold3.tmp" "$base/test.out"
+
+dsc-datatool \
+  -vvv \
+  -s test-server-åäö \
+  -n test-node \
+  --output ";InfluxDB;dml=1;database=dsc" \
+  --transform ";Labler;*;yaml=$base/labler.yaml" \
+  --transform ";ReRanger;rcode_vs_replylen;range=/64;pad_to=5" \
+  --transform ";ReRanger;qtype_vs_qnamelen;range=/16;pad_to=3" \
+  --transform ";ReRanger;client_port_range;key=low;range=/2048;pad_to=5" \
+  --transform ";ReRanger;edns_bufsiz,priming_queries;key=low;range=/512;pad_to=5;allow_invalid_keys=1" \
+  --transform ";ReRanger;priming_responses;key=low;range=/128;pad_to=4" \
+  --transform ";NetRemap;client_subnet,client_subnet2,client_addr_vs_rcode,ipv6_rsn_abusers;net=16" \
+  --generator ";client_subnet_authority;csv=$base/ipv4-address-space.csv;csv=$base/ipv6-unicast-address-assignments.csv" \
+  --xml "$base/utf8.xml" | sort -s > "$base/test4.out"
+
+sort -s "$base/test4.gold" > "$base/test4.gold.tmp"
+diff -u "$base/test4.gold.tmp" "$base/test4.out"

tests/test4.gold

@@ -0,0 +1,5 @@
+# CONTEXT-DATABASE: dsc
+# DML
+pcap_stats,server=test-server-åäö,node=test-node,ifname=eth0åäö,pcap_stat=filter_received value=5625 1563520560000000000
+pcap_stats,server=test-server-åäö,node=test-node,ifname=eth0åäö,pcap_stat=kernel_dropped value=731 1563520560000000000
+pcap_stats,server=test-server-åäö,node=test-node,ifname=eth0åäö,pcap_stat=pkts_captured value=4894 1563520560000000000

tests/utf8.xml

@@ -0,0 +1,13 @@
+<dscdata>
+<array name="pcap_stats" dimensions="2" start_time="1563520560" stop_time="1563520621">
+  <dimension number="1" type="ifname"/>
+  <dimension number="2" type="pcap_stat"/>
+  <data>
+    <ifname val="eth0åäö">
+      <pcap_stat val="filter_received" count="5625"/>
+      <pcap_stat val="pkts_captured" count="4894"/>
+      <pcap_stat val="kernel_dropped" count="731"/>
+    </ifname>
+  </data>
+</array>
+</dscdata>