Print a warning on malformed line

If a line can't be parsed (e.g. `FOO: BAR` is invalid), it is printed as
a warning on stderr.

As usual, after such an error, Python-dotenv tries to parse the rest of
the file and the application can use the successfully parsed variable
bindings.

Author: bbc2

src/dotenv/main.py

@@ -2,19 +2,22 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import io
+import logging
 import os
 import re
 import shutil
 import sys
 import tempfile
-from typing import (Dict, Iterator, List, Match, Optional,  # noqa
+from typing import (Dict, Iterator, Match, Optional,  # noqa
                     Pattern, Union, TYPE_CHECKING, Text, IO, Tuple)
 import warnings
 from collections import OrderedDict
 from contextlib import contextmanager
 
 from .compat import StringIO, PY2, to_env
-from .parser import parse_stream
+from .parser import Binding, parse_stream
+
+logger = logging.getLogger(__name__)
 
 if TYPE_CHECKING:  # pragma: no cover
     if sys.version_info >= (3, 6):
@@ -30,6 +33,17 @@
 __posix_variable = re.compile(r'\$\{[^\}]*\}')  # type: Pattern[Text]
 
 
+def with_warn_for_invalid_lines(mappings):
+    # type: (Iterator[Binding]) -> Iterator[Binding]
+    for mapping in mappings:
+        if mapping.key is None or mapping.value is None:
+            logger.warning(
+                "Python-dotenv could not parse statement starting at line %s",
+                mapping.original.line,
+            )
+        yield mapping
+
+
 class DotEnv():
 
     def __init__(self, dotenv_path, verbose=False, encoding=None):
@@ -65,7 +79,7 @@ def dict(self):
     def parse(self):
         # type: () -> Iterator[Tuple[Text, Text]]
         with self._get_stream() as stream:
-            for mapping in parse_stream(stream):
+            for mapping in with_warn_for_invalid_lines(parse_stream(stream)):
                 if mapping.key is not None and mapping.value is not None:
                     yield mapping.key, mapping.value
 
@@ -142,12 +156,12 @@ def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always"):
 
     with rewrite(dotenv_path) as (source, dest):
         replaced = False
-        for mapping in parse_stream(source):
+        for mapping in with_warn_for_invalid_lines(parse_stream(source)):
             if mapping.key == key_to_set:
                 dest.write(line_out)
                 replaced = True
             else:
-                dest.write(mapping.original)
+                dest.write(mapping.original.string)
         if not replaced:
             dest.write(line_out)
 
@@ -168,11 +182,11 @@ def unset_key(dotenv_path, key_to_unset, quote_mode="always"):
 
     removed = False
     with rewrite(dotenv_path) as (source, dest):
-        for mapping in parse_stream(source):
+        for mapping in with_warn_for_invalid_lines(parse_stream(source)):
             if mapping.key == key_to_unset:
                 removed = True
             else:
-                dest.write(mapping.original)
+                dest.write(mapping.original.string)
 
     if not removed:
         warnings.warn("key %s not removed from %s - key doesn't exist." % (key_to_unset, dotenv_path))  # type: ignore

src/dotenv/parser.py

@@ -11,6 +11,7 @@ def make_regex(string, extra_flags=0):
     return re.compile(to_text(string), re.UNICODE | extra_flags)
 
 
+_newline = make_regex(r"(\r\n|\n|\r)")
 _whitespace = make_regex(r"\s*", extra_flags=re.MULTILINE)
 _export = make_regex(r"(?:export[^\S\r\n]+)?")
 _single_quoted_key = make_regex(r"'([^']+)'")
@@ -25,9 +26,46 @@ def make_regex(string, extra_flags=0):
 _double_quote_escapes = make_regex(r"\\[\\'\"abfnrtv]")
 _single_quote_escapes = make_regex(r"\\[\\']")
 
-Binding = NamedTuple("Binding", [("key", Optional[Text]),
-                                 ("value", Optional[Text]),
-                                 ("original", Text)])
+
+class Position:
+    def __init__(self, chars, line):
+        # type: (int, int) -> None
+        self.chars = chars
+        self.line = line
+
+    @classmethod
+    def start(cls):
+        # type: () -> Position
+        return cls(chars=0, line=1)
+
+    def set(self, other):
+        # type: (Position) -> None
+        self.chars = other.chars
+        self.line = other.line
+
+    def advance(self, string):
+        # type: (Text) -> None
+        self.chars += len(string)
+        self.line += len(re.findall(_newline, string))
+
+
+Original = NamedTuple(
+    "Original",
+    [
+        ("string", Text),
+        ("line", int),
+    ],
+)
+
+
+Binding = NamedTuple(
+    "Binding",
+    [
+        ("key", Optional[Text]),
+        ("value", Optional[Text]),
+        ("original", Original),
+    ],
+)
 
 
 class Error(Exception):
@@ -38,39 +76,42 @@ class Reader:
     def __init__(self, stream):
         # type: (IO[Text]) -> None
         self.string = stream.read()
-        self.position = 0
-        self.mark = 0
+        self.position = Position.start()
+        self.mark = Position.start()
 
     def has_next(self):
         # type: () -> bool
-        return self.position < len(self.string)
+        return self.position.chars < len(self.string)
 
     def set_mark(self):
         # type: () -> None
-        self.mark = self.position
+        self.mark.set(self.position)
 
     def get_marked(self):
-        # type: () -> Text
-        return self.string[self.mark:self.position]
+        # type: () -> Original
+        return Original(
+            string=self.string[self.mark.chars:self.position.chars],
+            line=self.mark.line,
+        )
 
     def peek(self, count):
         # type: (int) -> Text
-        return self.string[self.position:self.position + count]
+        return self.string[self.position.chars:self.position.chars + count]
 
     def read(self, count):
         # type: (int) -> Text
-        result = self.string[self.position:self.position + count]
+        result = self.string[self.position.chars:self.position.chars + count]
         if len(result) < count:
             raise Error("read: End of string")
-        self.position += count
+        self.position.advance(result)
         return result
 
     def read_regex(self, regex):
         # type: (Pattern[Text]) -> Sequence[Text]
-        match = regex.match(self.string, self.position)
+        match = regex.match(self.string, self.position.chars)
         if match is None:
             raise Error("read_regex: Pattern not found")
-        self.position = match.end()
+        self.position.advance(self.string[match.start():match.end()])
         return match.groups()
 
 
@@ -131,10 +172,18 @@ def parse_binding(reader):
         value = parse_value(reader)
         reader.read_regex(_comment)
         reader.read_regex(_end_of_line)
-        return Binding(key=key, value=value, original=reader.get_marked())
+        return Binding(
+            key=key,
+            value=value,
+            original=reader.get_marked(),
+        )
     except Error:
         reader.read_regex(_rest_of_line)
-        return Binding(key=None, value=None, original=reader.get_marked())
+        return Binding(
+            key=None,
+            value=None,
+            original=reader.get_marked(),
+        )
 
 
 def parse_stream(stream):

tests/test_parser.py

@@ -2,83 +2,95 @@
 import pytest
 
 from dotenv.compat import StringIO
-from dotenv.parser import Binding, parse_stream
+from dotenv.parser import Binding, Original, parse_stream
 
 
 @pytest.mark.parametrize("test_input,expected", [
     (u"", []),
-    (u"a=b", [Binding(key=u"a", value=u"b", original=u"a=b")]),
-    (u"'a'=b", [Binding(key=u"a", value=u"b", original=u"'a'=b")]),
-    (u"[=b", [Binding(key=u"[", value=u"b", original=u"[=b")]),
-    (u" a = b ", [Binding(key=u"a", value=u"b", original=u" a = b ")]),
-    (u"export a=b", [Binding(key=u"a", value=u"b", original=u"export a=b")]),
-    (u" export 'a'=b", [Binding(key=u"a", value=u"b", original=u" export 'a'=b")]),
-    (u"# a=b", [Binding(key=None, value=None, original=u"# a=b")]),
-    (u"a=b#c", [Binding(key=u"a", value=u"b#c", original=u"a=b#c")]),
-    (u'a=b # comment', [Binding(key=u"a", value=u"b", original=u"a=b # comment")]),
-    (u"a=b space ", [Binding(key=u"a", value=u"b space", original=u"a=b space ")]),
-    (u"a='b space '", [Binding(key=u"a", value=u"b space ", original=u"a='b space '")]),
-    (u'a="b space "', [Binding(key=u"a", value=u"b space ", original=u'a="b space "')]),
-    (u"export export_a=1", [Binding(key=u"export_a", value=u"1", original=u"export export_a=1")]),
-    (u"export port=8000", [Binding(key=u"port", value=u"8000", original=u"export port=8000")]),
-    (u'a="b\nc"', [Binding(key=u"a", value=u"b\nc", original=u'a="b\nc"')]),
-    (u"a='b\nc'", [Binding(key=u"a", value=u"b\nc", original=u"a='b\nc'")]),
-    (u'a="b\nc"', [Binding(key=u"a", value=u"b\nc", original=u'a="b\nc"')]),
-    (u'a="b\\nc"', [Binding(key=u"a", value=u'b\nc', original=u'a="b\\nc"')]),
-    (u"a='b\\nc'", [Binding(key=u"a", value=u'b\\nc', original=u"a='b\\nc'")]),
-    (u'a="b\\"c"', [Binding(key=u"a", value=u'b"c', original=u'a="b\\"c"')]),
-    (u"a='b\\'c'", [Binding(key=u"a", value=u"b'c", original=u"a='b\\'c'")]),
-    (u"a=à", [Binding(key=u"a", value=u"à", original=u"a=à")]),
-    (u'a="à"', [Binding(key=u"a", value=u"à", original=u'a="à"')]),
-    (u'garbage', [Binding(key=None, value=None, original=u"garbage")]),
+    (u"a=b", [Binding(key=u"a", value=u"b", original=Original(string=u"a=b", line=1))]),
+    (u"'a'=b", [Binding(key=u"a", value=u"b", original=Original(string=u"'a'=b", line=1))]),
+    (u"[=b", [Binding(key=u"[", value=u"b", original=Original(string=u"[=b", line=1))]),
+    (u" a = b ", [Binding(key=u"a", value=u"b", original=Original(string=u" a = b ", line=1))]),
+    (u"export a=b", [Binding(key=u"a", value=u"b", original=Original(string=u"export a=b", line=1))]),
+    (u" export 'a'=b", [Binding(key=u"a", value=u"b", original=Original(string=u" export 'a'=b", line=1))]),
+    (u"# a=b", [Binding(key=None, value=None, original=Original(string=u"# a=b", line=1))]),
+    (u"a=b#c", [Binding(key=u"a", value=u"b#c", original=Original(string=u"a=b#c", line=1))]),
+    (u'a=b # comment', [Binding(key=u"a", value=u"b", original=Original(string=u"a=b # comment", line=1))]),
+    (u"a=b space ", [Binding(key=u"a", value=u"b space", original=Original(string=u"a=b space ", line=1))]),
+    (u"a='b space '", [Binding(key=u"a", value=u"b space ", original=Original(string=u"a='b space '", line=1))]),
+    (u'a="b space "', [Binding(key=u"a", value=u"b space ", original=Original(string=u'a="b space "', line=1))]),
+    (
+        u"export export_a=1",
+        [
+            Binding(key=u"export_a", value=u"1", original=Original(string=u"export export_a=1", line=1))
+        ],
+    ),
+    (u"export port=8000", [Binding(key=u"port", value=u"8000", original=Original(string=u"export port=8000", line=1))]),
+    (u'a="b\nc"', [Binding(key=u"a", value=u"b\nc", original=Original(string=u'a="b\nc"', line=1))]),
+    (u"a='b\nc'", [Binding(key=u"a", value=u"b\nc", original=Original(string=u"a='b\nc'", line=1))]),
+    (u'a="b\nc"', [Binding(key=u"a", value=u"b\nc", original=Original(string=u'a="b\nc"', line=1))]),
+    (u'a="b\\nc"', [Binding(key=u"a", value=u'b\nc', original=Original(string=u'a="b\\nc"', line=1))]),
+    (u"a='b\\nc'", [Binding(key=u"a", value=u'b\\nc', original=Original(string=u"a='b\\nc'", line=1))]),
+    (u'a="b\\"c"', [Binding(key=u"a", value=u'b"c', original=Original(string=u'a="b\\"c"', line=1))]),
+    (u"a='b\\'c'", [Binding(key=u"a", value=u"b'c", original=Original(string=u"a='b\\'c'", line=1))]),
+    (u"a=à", [Binding(key=u"a", value=u"à", original=Original(string=u"a=à", line=1))]),
+    (u'a="à"', [Binding(key=u"a", value=u"à", original=Original(string=u'a="à"', line=1))]),
+    (u'garbage', [Binding(key=None, value=None, original=Original(string=u"garbage", line=1))]),
     (
         u"a=b\nc=d",
         [
-            Binding(key=u"a", value=u"b", original=u"a=b\n"),
-            Binding(key=u"c", value=u"d", original=u"c=d"),
+            Binding(key=u"a", value=u"b", original=Original(string=u"a=b\n", line=1)),
+            Binding(key=u"c", value=u"d", original=Original(string=u"c=d", line=2)),
+        ],
+    ),
+    (
+        u"a=b\rc=d",
+        [
+            Binding(key=u"a", value=u"b", original=Original(string=u"a=b\r", line=1)),
+            Binding(key=u"c", value=u"d", original=Original(string=u"c=d", line=2)),
         ],
     ),
     (
         u"a=b\r\nc=d",
         [
-            Binding(key=u"a", value=u"b", original=u"a=b\r\n"),
-            Binding(key=u"c", value=u"d", original=u"c=d"),
+            Binding(key=u"a", value=u"b", original=Original(string=u"a=b\r\n", line=1)),
+            Binding(key=u"c", value=u"d", original=Original(string=u"c=d", line=2)),
         ],
     ),
     (
         u'a=\nb=c',
         [
-            Binding(key=u"a", value=u'', original=u'a=\n'),
-            Binding(key=u"b", value=u'c', original=u"b=c"),
+            Binding(key=u"a", value=u'', original=Original(string=u'a=\n', line=1)),
+            Binding(key=u"b", value=u'c', original=Original(string=u"b=c", line=2)),
         ]
     ),
     (
         u'a=b\n\nc=d',
         [
-            Binding(key=u"a", value=u"b", original=u"a=b\n"),
-            Binding(key=u"c", value=u"d", original=u"\nc=d"),
+            Binding(key=u"a", value=u"b", original=Original(string=u"a=b\n", line=1)),
+            Binding(key=u"c", value=u"d", original=Original(string=u"\nc=d", line=2)),
         ]
     ),
     (
         u'a="\nb=c',
         [
-            Binding(key=None, value=None, original=u'a="\n'),
-            Binding(key=u"b", value=u"c", original=u"b=c"),
+            Binding(key=None, value=None, original=Original(string=u'a="\n', line=1)),
+            Binding(key=u"b", value=u"c", original=Original(string=u"b=c", line=2)),
         ]
     ),
     (
         u'# comment\na="b\nc"\nd=e\n',
         [
-            Binding(key=None, value=None, original=u"# comment\n"),
-            Binding(key=u"a", value=u"b\nc", original=u'a="b\nc"\n'),
-            Binding(key=u"d", value=u"e", original=u"d=e\n"),
+            Binding(key=None, value=None, original=Original(string=u"# comment\n", line=1)),
+            Binding(key=u"a", value=u"b\nc", original=Original(string=u'a="b\nc"\n', line=2)),
+            Binding(key=u"d", value=u"e", original=Original(string=u"d=e\n", line=4)),
         ],
     ),
     (
         u'garbage[%$#\na=b',
         [
-            Binding(key=None, value=None, original=u"garbage[%$#\n"),
-            Binding(key=u"a", value=u"b", original=u'a=b'),
+            Binding(key=None, value=None, original=Original(string=u"garbage[%$#\n", line=1)),
+            Binding(key=u"a", value=u"b", original=Original(string=u'a=b', line=2)),
         ],
     ),
 ])