From 35ba9cc9d99d72c4830cc264fc949cf5b8dad126 Mon Sep 17 00:00:00 2001 From: Andrew Phillips Date: Mon, 19 Jul 2021 17:49:12 -0500 Subject: [PATCH 1/5] Fix source string exec --- birdseye/utils.py | 16 ++++++++++++---- tests/test_utils.py | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/birdseye/utils.py b/birdseye/utils.py index dfea09a..35e9526 100644 --- a/birdseye/utils.py +++ b/birdseye/utils.py @@ -199,11 +199,19 @@ def read_source_file(filename): if filename.endswith('.pyc'): filename = filename[:-1] - with open_with_encoding_check(filename) as f: + try: + with open_with_encoding_check(filename) as f: + return ''.join([ + '\n' if i < 2 and cookie_re.match(line) + else line + for i, line in enumerate(f) + ]) + + except FileNotFoundError: + import linecache + return ''.join([ - '\n' if i < 2 and cookie_re.match(line) - else line - for i, line in enumerate(f) + line for line in linecache.cache[filename][2] ]) diff --git a/tests/test_utils.py b/tests/test_utils.py index 212033a..1830b43 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -146,6 +146,30 @@ def test_cheap_repr(self): series = df[0] self.assertEqual(cheap_repr(series), "0 = 0; 1 = 100; 2 = 200; ...; 97 = 9700; 98 = 9800; 99 = 9900") + def test_read_source_file_as_string(self): + import linecache + import birdseye + + source = """ +from birdseye import eye + +@eye +def func(): + return 42 +""" + filename = "" + co = compile(source, filename, "exec") + linecache.cache[filename] = ( + len(source), + None, + [line + '\n' for line in source.splitlines()], + filename, + ) + exec(co, {"birdseye": birdseye}) + rv = read_source_file(filename) + self.assertEqual(rv, source) + return + if __name__ == '__main__': unittest.main() From 55194d9355961209ad15c6237272d52e84cf1b86 Mon Sep 17 00:00:00 2001 From: Andrew Phillips Date: Mon, 19 Jul 2021 18:22:13 -0500 Subject: [PATCH 2/5] Use getlines instead of direct cache access, remove explicit file open --- birdseye/utils.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/birdseye/utils.py b/birdseye/utils.py index 35e9526..86863b5 100644 --- a/birdseye/utils.py +++ b/birdseye/utils.py @@ -1,5 +1,6 @@ import ast import json +import linecache import ntpath import os import sys @@ -199,21 +200,12 @@ def read_source_file(filename): if filename.endswith('.pyc'): filename = filename[:-1] - try: - with open_with_encoding_check(filename) as f: - return ''.join([ - '\n' if i < 2 and cookie_re.match(line) - else line - for i, line in enumerate(f) + return ''.join([ + '\n' if i < 2 and cookie_re.match(line) + else line + for i, line in linecache.getlines(filename) ]) - except FileNotFoundError: - import linecache - - return ''.join([ - line for line in linecache.cache[filename][2] - ]) - def source_without_decorators(tokens, function_node): def_token = tokens.find_token(function_node.first_token, token.NAME, 'def') From d1a93bf7a2210f63d33cfb271a30037fdcb898d3 Mon Sep 17 00:00:00 2001 From: Andrew Phillips Date: Mon, 19 Jul 2021 18:29:53 -0500 Subject: [PATCH 3/5] Fix loop --- birdseye/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/birdseye/utils.py b/birdseye/utils.py index 86863b5..adcd0a6 100644 --- a/birdseye/utils.py +++ b/birdseye/utils.py @@ -203,7 +203,7 @@ def read_source_file(filename): return ''.join([ '\n' if i < 2 and cookie_re.match(line) else line - for i, line in linecache.getlines(filename) + for line in linecache.getlines(filename) ]) From 831bf7cc525b90d0b2a57c35b764ae1eaf89bce4 Mon Sep 17 00:00:00 2001 From: Andrew Phillips Date: Mon, 19 Jul 2021 18:32:43 -0500 Subject: [PATCH 4/5] Fix loop --- birdseye/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/birdseye/utils.py b/birdseye/utils.py index adcd0a6..51959d0 100644 --- a/birdseye/utils.py +++ b/birdseye/utils.py @@ -203,7 +203,7 @@ def read_source_file(filename): return ''.join([ '\n' if i < 2 and cookie_re.match(line) else line - for line in linecache.getlines(filename) + for i, line in enumerate(linecache.getlines(filename)) ]) From acfbd63a0943c44c21d6d3c3f379132be246299b Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Sun, 1 Aug 2021 15:45:34 +0200 Subject: [PATCH 5/5] Handle encoding in read_source_file --- birdseye/utils.py | 51 ++++++++++++++++----------------------------- tests/test_utils.py | 2 ++ 2 files changed, 20 insertions(+), 33 deletions(-) diff --git a/birdseye/utils.py b/birdseye/utils.py index 51959d0..65f64c6 100644 --- a/birdseye/utils.py +++ b/birdseye/utils.py @@ -1,4 +1,5 @@ import ast +import io import json import linecache import ntpath @@ -7,6 +8,7 @@ import token import types from sys import version_info + from littleutils import strip_required_prefix # noinspection PyUnreachableCode @@ -166,45 +168,28 @@ def default(self, o): return method() -try: - - # Python 3 - from tokenize import open as open_with_encoding_check - -except ImportError: - - # Python 2 - from lib2to3.pgen2.tokenize import detect_encoding - import io - - - def open_with_encoding_check(filename): # type: ignore - """Open a file in read only mode using the encoding detected by - detect_encoding(). - """ - fp = io.open(filename, 'rb') - try: - encoding, lines = detect_encoding(fp.readline) - fp.seek(0) - text = io.TextIOWrapper(fp, encoding, line_buffering=True) - text.mode = 'r' - return text - except: - fp.close() - raise - - def read_source_file(filename): - from lib2to3.pgen2.tokenize import cookie_re + if PY3: + from tokenize import detect_encoding, cookie_re + else: + from lib2to3.pgen2.tokenize import detect_encoding, cookie_re + + lines = linecache.getlines(filename) + text = ''.join(lines) - if filename.endswith('.pyc'): - filename = filename[:-1] + if not isinstance(text, Text): + encoding = detect_encoding(io.BytesIO(text).readline)[0] + text = text.decode(encoding) # noqa + lines = [line.decode(encoding) for line in lines] + # In python 2 it's a syntax error to parse unicode + # with an encoding declaration, so we remove it but + # leave empty lines in its place to keep line numbers the same return ''.join([ '\n' if i < 2 and cookie_re.match(line) else line - for i, line in enumerate(linecache.getlines(filename)) - ]) + for i, line in enumerate(lines) + ]) def source_without_decorators(tokens, function_node): diff --git a/tests/test_utils.py b/tests/test_utils.py index 1830b43..41fb1f8 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,7 @@ # coding=utf8 import ast +import linecache import unittest from tempfile import mkstemp @@ -99,6 +100,7 @@ def test_open_with_encoding_check(self): def write(stuff): with open(filename, 'wb') as f: f.write(stuff) + linecache.cache.pop(filename, None) def read(): return read_source_file(filename).strip()