postgresql output: escape all null bytes

while null bytes (`\0`, not SQL "NULL") in TEXT and JSON/JSONB fields are valid, data containing null bytes can cause troubles in some combinations of clients, servers and each settings.
To prevent unhandled errors, and data which can't be inserted into the database, all null bytes are escaped

fixes #2203

Author: wagner-intevation

CHANGELOG.md

@@ -94,6 +94,8 @@ CHANGELOG
 - `intelmq.bots.outputs.templated_smtp.output`:
   - Add new function `from_json()` (which just calls `json.loads()` in the standard Python environment), meaning the Templated SMTP output bot can take strings containing JSON documents and do the formatting itself (PR#2120 by Karl-Johan Karlsson).
   - Lift restriction on requirement jinja2 < 3 (PR#2158 by Sebastian Wagner).
+- `intelmq.bots.outputs.sql`:
+  - For PostgreSQL, escape Nullbytes in text to prevent "unsupported Unicode escape sequence" issues (PR#2223 by Sebastian Wagner, fixes #2203).
 
 ### Documentation
 - Feeds: Add documentation for newly supported dataplane feeds, see above (PR#2102 by Mikk Margus Möll).

docs/user/bots.rst

@@ -4025,7 +4025,8 @@ The parameters marked with 'PostgreSQL' will be sent to libpq via psycopg2. Chec
 * `sslmode`: PostgreSQL sslmode, can be `'disable'`, `'allow'`, `'prefer'` (default), `'require'`, `'verify-ca'` or `'verify-full'`. See postgresql docs: https://www.postgresql.org/docs/current/static/libpq-connect.html#libpq-connect-sslmode
 * `table`: name of the database table into which events are to be inserted
 
-**PostgreSQL**
+PostgreSQL
+~~~~~~~~~~
 
 You have two basic choices to run PostgreSQL:
 
@@ -4080,7 +4081,13 @@ if the user `intelmq` can authenticate):
 
    psql -h localhost intelmq-events intelmq </tmp/initdb.sql
 
-**SQLite**
+**PostgreSQL and null characters**
+
+While null characters (`\0`, not SQL "NULL") in TEXT and JSON/JSONB fields are valid, data containing null characters can cause troubles in some combinations of clients, servers and each settings.
+To prevent unhandled errors and data which can't be inserted into the database, all null characters are escaped (`\\u0000`) before insertion.
+
+SQLite
+~~~~~~
 
 Similarly to PostgreSQL, you can use `intelmq_psql_initdb` to create initial SQL statements
 from `harmonization.conf`. The script will create the required table layout

intelmq/bots/outputs/sql/output.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2019 Edvard Rejthar
+# SPDX-FileCopyrightText: 2019 Edvard Rejthar, 2022 Intevation GmbH
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
@@ -37,7 +37,7 @@ def process(self):
         event = self.receive_message().to_dict(jsondict_as_string=self.jsondict_as_string)
 
         keys = '", "'.join(event.keys())
-        values = list(event.values())
+        values = self.prepare_values(event.values())
         fvalues = len(values) * f'{self.format_char}, '
         query = ('INSERT INTO {table} ("{keys}") VALUES ({values})'
                  ''.format(table=self.table, keys=keys, values=fvalues[:-2]))
@@ -46,5 +46,14 @@ def process(self):
             self.con.commit()
             self.acknowledge_message()
 
+    def prepare_values(self, values):
+        if self._engine_name == self.POSTGRESQL:
+            # escape JSON-encoded NULL characters. JSON escapes them once, but we need to escape them twice,
+            # so that Postgres does not encounter a NULL char while decoding it
+            # https://github.com/certtools/intelmq/issues/2203
+            return [value.replace('\\u0000', '\\\\u0000') if isinstance(value, str) else value for value in values]
+        else:
+            return list(values)
+
 
 BOT = SQLOutputBot

intelmq/lib/test.py

@@ -188,7 +188,7 @@ def log(name, *args, **kwargs):
             return logger
         return log
 
-    def prepare_bot(self, parameters={}, destination_queues=None):
+    def prepare_bot(self, parameters={}, destination_queues=None, prepare_source_queue: bool = True):
         """
         Reconfigures the bot with the changed attributes.
 
@@ -238,7 +238,8 @@ def prepare_bot(self, parameters={}, destination_queues=None):
         self.pipe.set_queues(parameters.source_queue, "source")
         self.pipe.set_queues(parameters.destination_queues, "destination")
 
-        self.prepare_source_queue()
+        if prepare_source_queue:
+            self.prepare_source_queue()
 
     def prepare_source_queue(self):
         if self.input_message is not None:

intelmq/tests/bots/outputs/sql/test_output_postgresql.py

@@ -1,8 +1,9 @@
-# SPDX-FileCopyrightText: 2019 Sebastian Wagner
+# SPDX-FileCopyrightText: 2019 Sebastian Wagner, 2022 Intevation GmbH
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 # -*- coding: utf-8 -*-
+import json
 import os
 import unittest
 
@@ -28,6 +29,10 @@
                "extra.asn": 64496,
                "extra.ip": "192.0.2.1",
                }
+INPUT_NULL = {"__type": "Event",
+              "classification.type": "undetermined",
+              "extra.payload": '{"text": "M41\u0012)3U>\bxӾ6\u0000\u0013M6M6M4M4]4y]4ӭ4"}',
+              }
 
 
 @test.skip_database()
@@ -77,6 +82,18 @@ def test_extra(self):
         from_db = {k: v for k, v in self.cur.fetchone().items() if v is not None}
         self.assertEqual(from_db['extra'], {"asn": 64496, "ip": "192.0.2.1"})
 
+    def test_extra_nullbyte(self):
+        """
+        Test a Nullbyte in an extra-field
+        https://github.com/certtools/intelmq/issues/2203
+        """
+        self.input_message = INPUT_NULL
+        self.run_bot()
+        self.cur.execute('SELECT "extra" FROM tests WHERE "classification.type" = \'undetermined\'')
+        self.assertEqual(self.cur.rowcount, 1)
+        from_db = {k: v for k, v in self.cur.fetchone().items() if v is not None}
+        self.assertEqual(from_db['extra'], {"payload": '{"text": "M41\u0012)3U>\bxӾ6\\u0000\u0013M6M6M4M4]4y]4ӭ4"}'})
+
     @classmethod
     def tearDownClass(cls):
         if not os.environ.get('INTELMQ_TEST_DATABASES'):
@@ -122,6 +139,13 @@ def test_event(self):
         from_db = {k: v for k, v in self.cur.fetchone().items() if v is not None}
         self.assertDictEqual(from_db, OUTPUT1)
 
+    def test_prepare_null(self):
+        """ Test if a null character in extra is correctly removed. https://github.com/certtools/intelmq/issues/2203 """
+        values = [json.dumps({"special": "foo\x00bar"})]
+        self.prepare_bot(prepare_source_queue=False)
+        output = self.bot.prepare_values(values)
+        self.assertEqual(output, ['{"special": "foo\\\\u0000bar"}'])
+
     @classmethod
     def tearDownClass(cls):
         if not os.environ.get('INTELMQ_TEST_DATABASES'):