From 1f7d64608b5c7f4c3d96b01b33e18ebf9dec8490 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 6 Aug 2021 14:33:30 -0500 Subject: [PATCH 1/8] bpo-44605: Teach @total_ordering() to work with metaclasses (GH-27633) --- Lib/functools.py | 24 ++++++++-------- Lib/test/test_functools.py | 28 +++++++++++++++++++ .../2021-08-06-09-43-50.bpo-44605.q4YSBZ.rst | 1 + 3 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-08-06-09-43-50.bpo-44605.q4YSBZ.rst diff --git a/Lib/functools.py b/Lib/functools.py index 357c1dfd909fa9..77ec852805c104 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -88,84 +88,84 @@ def wraps(wrapped, def _gt_from_lt(self, other, NotImplemented=NotImplemented): 'Return a > b. Computed by @total_ordering from (not a < b) and (a != b).' - op_result = self.__lt__(other) + op_result = type(self).__lt__(self, other) if op_result is NotImplemented: return op_result return not op_result and self != other def _le_from_lt(self, other, NotImplemented=NotImplemented): 'Return a <= b. Computed by @total_ordering from (a < b) or (a == b).' - op_result = self.__lt__(other) + op_result = type(self).__lt__(self, other) if op_result is NotImplemented: return op_result return op_result or self == other def _ge_from_lt(self, other, NotImplemented=NotImplemented): 'Return a >= b. Computed by @total_ordering from (not a < b).' - op_result = self.__lt__(other) + op_result = type(self).__lt__(self, other) if op_result is NotImplemented: return op_result return not op_result def _ge_from_le(self, other, NotImplemented=NotImplemented): 'Return a >= b. Computed by @total_ordering from (not a <= b) or (a == b).' - op_result = self.__le__(other) + op_result = type(self).__le__(self, other) if op_result is NotImplemented: return op_result return not op_result or self == other def _lt_from_le(self, other, NotImplemented=NotImplemented): 'Return a < b. Computed by @total_ordering from (a <= b) and (a != b).' - op_result = self.__le__(other) + op_result = type(self).__le__(self, other) if op_result is NotImplemented: return op_result return op_result and self != other def _gt_from_le(self, other, NotImplemented=NotImplemented): 'Return a > b. Computed by @total_ordering from (not a <= b).' - op_result = self.__le__(other) + op_result = type(self).__le__(self, other) if op_result is NotImplemented: return op_result return not op_result def _lt_from_gt(self, other, NotImplemented=NotImplemented): 'Return a < b. Computed by @total_ordering from (not a > b) and (a != b).' - op_result = self.__gt__(other) + op_result = type(self).__gt__(self, other) if op_result is NotImplemented: return op_result return not op_result and self != other def _ge_from_gt(self, other, NotImplemented=NotImplemented): 'Return a >= b. Computed by @total_ordering from (a > b) or (a == b).' - op_result = self.__gt__(other) + op_result = type(self).__gt__(self, other) if op_result is NotImplemented: return op_result return op_result or self == other def _le_from_gt(self, other, NotImplemented=NotImplemented): 'Return a <= b. Computed by @total_ordering from (not a > b).' - op_result = self.__gt__(other) + op_result = type(self).__gt__(self, other) if op_result is NotImplemented: return op_result return not op_result def _le_from_ge(self, other, NotImplemented=NotImplemented): 'Return a <= b. Computed by @total_ordering from (not a >= b) or (a == b).' - op_result = self.__ge__(other) + op_result = type(self).__ge__(self, other) if op_result is NotImplemented: return op_result return not op_result or self == other def _gt_from_ge(self, other, NotImplemented=NotImplemented): 'Return a > b. Computed by @total_ordering from (a >= b) and (a != b).' - op_result = self.__ge__(other) + op_result = type(self).__ge__(self, other) if op_result is NotImplemented: return op_result return op_result and self != other def _lt_from_ge(self, other, NotImplemented=NotImplemented): 'Return a < b. Computed by @total_ordering from (not a >= b).' - op_result = self.__ge__(other) + op_result = type(self).__ge__(self, other) if op_result is NotImplemented: return op_result return not op_result diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 78a8a5fcc0feaa..fbf5578872e6b0 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1163,6 +1163,34 @@ def test_pickle(self): method_copy = pickle.loads(pickle.dumps(method, proto)) self.assertIs(method_copy, method) + + def test_total_ordering_for_metaclasses_issue_44605(self): + + @functools.total_ordering + class SortableMeta(type): + def __new__(cls, name, bases, ns): + return super().__new__(cls, name, bases, ns) + + def __lt__(self, other): + if not isinstance(other, SortableMeta): + pass + return self.__name__ < other.__name__ + + def __eq__(self, other): + if not isinstance(other, SortableMeta): + pass + return self.__name__ == other.__name__ + + class B(metaclass=SortableMeta): + pass + + class A(metaclass=SortableMeta): + pass + + self.assertTrue(A < B) + self.assertFalse(A > B) + + @functools.total_ordering class Orderable_LT: def __init__(self, value): diff --git a/Misc/NEWS.d/next/Library/2021-08-06-09-43-50.bpo-44605.q4YSBZ.rst b/Misc/NEWS.d/next/Library/2021-08-06-09-43-50.bpo-44605.q4YSBZ.rst new file mode 100644 index 00000000000000..93783923e15b3d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-06-09-43-50.bpo-44605.q4YSBZ.rst @@ -0,0 +1 @@ +The @functools.total_ordering() decorator now works with metaclasses. From 0ffdced3b64ba5886fcde64266a31a15712da284 Mon Sep 17 00:00:00 2001 From: Jack DeVries <58614260+jdevries3133@users.noreply.github.com> Date: Fri, 6 Aug 2021 16:05:16 -0400 Subject: [PATCH 2/8] bpo-27752: improve documentation of csv.Dialect (GH-26795) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Doc/library/csv.rst | 34 +++++++++++++------ .../2021-06-18-18-04-53.bpo-27752.NEByNk.rst | 1 + 2 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2021-06-18-18-04-53.bpo-27752.NEByNk.rst diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index cb03f8da20235f..899ce0225ce7f3 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -94,8 +94,8 @@ The :mod:`csv` module defines the following functions: :class:`Dialect` class or one of the strings returned by the :func:`list_dialects` function. The other optional *fmtparams* keyword arguments can be given to override individual formatting parameters in the current - dialect. For full details about the dialect and formatting parameters, see - section :ref:`csv-fmt-params`. To make it + dialect. For full details about dialects and formatting parameters, see + the :ref:`csv-fmt-params` section. To make it as easy as possible to interface with modules which implement the DB API, the value :const:`None` is written as the empty string. While this isn't a reversible transformation, it makes it easier to dump SQL NULL data values to @@ -117,7 +117,7 @@ The :mod:`csv` module defines the following functions: Associate *dialect* with *name*. *name* must be a string. The dialect can be specified either by passing a sub-class of :class:`Dialect`, or by *fmtparams* keyword arguments, or both, with keyword arguments overriding - parameters of the dialect. For full details about the dialect and formatting + parameters of the dialect. For full details about dialects and formatting parameters, see section :ref:`csv-fmt-params`. @@ -225,9 +225,21 @@ The :mod:`csv` module defines the following classes: .. class:: Dialect - The :class:`Dialect` class is a container class relied on primarily for its - attributes, which are used to define the parameters for a specific - :class:`reader` or :class:`writer` instance. + The :class:`Dialect` class is a container class whose attributes contain + information for how to handle doublequotes, whitespace, delimiters, etc. + Due to the lack of a strict CSV specification, different applications + produce subtly different CSV data. :class:`Dialect` instances define how + :class:`reader` and :class:`writer` instances behave. + + All available :class:`Dialect` names are returned by :func:`list_dialects`, + and they can be registered with specific :class:`reader` and :class:`writer` + classes through their initializer (``__init__``) functions like this:: + + import csv + + with open('students.csv', 'w', newline='') as csvfile: + writer = csv.writer(csvfile, dialect='unix') + ^^^^^^^^^^^^^^ .. class:: excel() @@ -419,8 +431,8 @@ Reader objects (:class:`DictReader` instances and objects returned by the Return the next row of the reader's iterable object as a list (if the object was returned from :func:`reader`) or a dict (if it is a :class:`DictReader` - instance), parsed according to the current dialect. Usually you should call - this as ``next(reader)``. + instance), parsed according to the current :class:`Dialect`. Usually you + should call this as ``next(reader)``. Reader objects have the following public attributes: @@ -460,9 +472,9 @@ read CSV files (assuming they support complex numbers at all). .. method:: csvwriter.writerow(row) - Write the *row* parameter to the writer's file object, formatted according to - the current dialect. Return the return value of the call to the *write* method - of the underlying file object. + Write the *row* parameter to the writer's file object, formatted according + to the current :class:`Dialect`. Return the return value of the call to the + *write* method of the underlying file object. .. versionchanged:: 3.5 Added support of arbitrary iterables. diff --git a/Misc/NEWS.d/next/Documentation/2021-06-18-18-04-53.bpo-27752.NEByNk.rst b/Misc/NEWS.d/next/Documentation/2021-06-18-18-04-53.bpo-27752.NEByNk.rst new file mode 100644 index 00000000000000..ccb7767a6b6936 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2021-06-18-18-04-53.bpo-27752.NEByNk.rst @@ -0,0 +1 @@ +Documentation of csv.Dialect is more descriptive. From e9a6f1b78bf57d9f3f99547bd007d7cfc9724cfb Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 6 Aug 2021 21:44:15 +0100 Subject: [PATCH 3/8] bpo-41576: document BaseException in favor of bare except (GH-21917) --- Doc/tutorial/errors.rst | 17 +++++++++-------- .../2020-08-21-22-59-37.bpo-41576.7a6CQR.rst | 1 + 2 files changed, 10 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2020-08-21-22-59-37.bpo-41576.7a6CQR.rst diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 27e39dfe257716..e7a45a302c65a7 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -147,10 +147,10 @@ For example, the following code will print B, C, D in that order:: Note that if the *except clauses* were reversed (with ``except B`` first), it would have printed B, B, B --- the first matching *except clause* is triggered. -The last *except clause* may omit the exception name(s), to serve as a wildcard. -Use this with extreme caution, since it is easy to mask a real programming error -in this way! It can also be used to print an error message and then re-raise -the exception (allowing a caller to handle the exception as well):: +All exceptions inherit from :exc:`BaseException`, and so it can be used to serve +as a wildcard. Use this with extreme caution, since it is easy to mask a real +programming error in this way! It can also be used to print an error message and +then re-raise the exception (allowing a caller to handle the exception as well):: import sys @@ -162,10 +162,13 @@ the exception (allowing a caller to handle the exception as well):: print("OS error: {0}".format(err)) except ValueError: print("Could not convert data to an integer.") - except: - print("Unexpected error:", sys.exc_info()[0]) + except BaseException as err: + print(f"Unexpected {err=}, {type(err)=}") raise +Alternatively the last except clause may omit the exception name(s), however the exception +value must then be retrieved from ``sys.exc_info()[1]``. + The :keyword:`try` ... :keyword:`except` statement has an optional *else clause*, which, when present, must follow all *except clauses*. It is useful for code that must be executed if the *try clause* does not raise an exception. @@ -493,5 +496,3 @@ used in a way that ensures they are always cleaned up promptly and correctly. :: After the statement is executed, the file *f* is always closed, even if a problem was encountered while processing the lines. Objects which, like files, provide predefined clean-up actions will indicate this in their documentation. - - diff --git a/Misc/NEWS.d/next/Documentation/2020-08-21-22-59-37.bpo-41576.7a6CQR.rst b/Misc/NEWS.d/next/Documentation/2020-08-21-22-59-37.bpo-41576.7a6CQR.rst new file mode 100644 index 00000000000000..f74ef62ca47ab2 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2020-08-21-22-59-37.bpo-41576.7a6CQR.rst @@ -0,0 +1 @@ +document BaseException in favor of bare except \ No newline at end of file From 17c23167942498296f0bdfffe52e72d53d66d693 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Fri, 6 Aug 2021 23:35:13 +0200 Subject: [PATCH 4/8] bpo-42971: Add errno.EQFULL (macOS) (GH-24419) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Doc/library/errno.rst | 5 +++++ .../next/Library/2021-02-02-20-11-14.bpo-42971.OpVoFu.rst | 2 ++ Modules/errnomodule.c | 3 +++ 3 files changed, 10 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2021-02-02-20-11-14.bpo-42971.OpVoFu.rst diff --git a/Doc/library/errno.rst b/Doc/library/errno.rst index 1cbd51c582c0cf..93bdb6ca9b8e3c 100644 --- a/Doc/library/errno.rst +++ b/Doc/library/errno.rst @@ -637,3 +637,8 @@ defined by the module. The specific list of defined symbols is available as Quota exceeded +.. data:: EQFULL + + Interface output queue is full + + .. versionadded:: 3.11 diff --git a/Misc/NEWS.d/next/Library/2021-02-02-20-11-14.bpo-42971.OpVoFu.rst b/Misc/NEWS.d/next/Library/2021-02-02-20-11-14.bpo-42971.OpVoFu.rst new file mode 100644 index 00000000000000..97c8d2d79aa404 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-02-02-20-11-14.bpo-42971.OpVoFu.rst @@ -0,0 +1,2 @@ +Add definition of ``errno.EQFULL`` for platforms that define this constant +(such as macOS). diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index d99bed45bd6a23..bf6766e02349c0 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -920,6 +920,9 @@ errno_exec(PyObject *module) #ifdef ESHLIBVERS add_errcode("ESHLIBVERS", ESHLIBVERS, "Shared library version mismatch"); #endif +#ifdef EQFULL + add_errcode("EQFULL", EQFULL, "Interface output queue is full"); +#endif Py_DECREF(error_dict); return 0; From a40675c659cd8c0699f85ee9ac31660f93f8c2f5 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 7 Aug 2021 11:10:14 +0100 Subject: [PATCH 5/8] bpo-44856: Possible reference leak in error paths of update_bases() and __build_class__ (GH-27647) --- .../2021-08-07-01-26-12.bpo-44856.9rk3li.rst | 1 + Python/bltinmodule.c | 35 +++++++------------ 2 files changed, 14 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-08-07-01-26-12.bpo-44856.9rk3li.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-07-01-26-12.bpo-44856.9rk3li.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-07-01-26-12.bpo-44856.9rk3li.rst new file mode 100644 index 00000000000000..1111d01b726fa2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-08-07-01-26-12.bpo-44856.9rk3li.rst @@ -0,0 +1 @@ +Fix reference leaks in the error paths of ``update_bases()`` and ``__build_class__``. Patch by Pablo Galindo. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index bfe21ad6d0c4c7..761dc08d1d6413 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -72,6 +72,7 @@ update_bases(PyObject *bases, PyObject *const *args, Py_ssize_t nargs) /* If this is a first successful replacement, create new_bases list and copy previously encountered bases. */ if (!(new_bases = PyList_New(i))) { + Py_DECREF(new_base); goto error; } for (j = 0; j < i; j++) { @@ -82,6 +83,7 @@ update_bases(PyObject *bases, PyObject *const *args, Py_ssize_t nargs) } j = PyList_GET_SIZE(new_bases); if (PyList_SetSlice(new_bases, j, j, new_base) < 0) { + Py_DECREF(new_base); goto error; } Py_DECREF(new_base); @@ -103,8 +105,9 @@ static PyObject * builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns, *orig_bases; - PyObject *cls = NULL, *cell = NULL; + PyObject *func, *name, *winner, *prep; + PyObject *cls = NULL, *cell = NULL, *ns = NULL, *meta = NULL, *orig_bases = NULL; + PyObject *mkw = NULL, *bases = NULL; int isclass = 0; /* initialize to prevent gcc warning */ if (nargs < 2) { @@ -141,26 +144,20 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, else { mkw = _PyStack_AsDict(args + nargs, kwnames); if (mkw == NULL) { - Py_DECREF(bases); - return NULL; + goto error; } meta = _PyDict_GetItemIdWithError(mkw, &PyId_metaclass); if (meta != NULL) { Py_INCREF(meta); if (_PyDict_DelItemId(mkw, &PyId_metaclass) < 0) { - Py_DECREF(meta); - Py_DECREF(mkw); - Py_DECREF(bases); - return NULL; + goto error; } /* metaclass is explicitly given, check if it's indeed a class */ isclass = PyType_Check(meta); } else if (PyErr_Occurred()) { - Py_DECREF(mkw); - Py_DECREF(bases); - return NULL; + goto error; } } if (meta == NULL) { @@ -183,10 +180,7 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, winner = (PyObject *)_PyType_CalculateMetaclass((PyTypeObject *)meta, bases); if (winner == NULL) { - Py_DECREF(meta); - Py_XDECREF(mkw); - Py_DECREF(bases); - return NULL; + goto error; } if (winner != meta) { Py_DECREF(meta); @@ -208,10 +202,7 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, Py_DECREF(prep); } if (ns == NULL) { - Py_DECREF(meta); - Py_XDECREF(mkw); - Py_DECREF(bases); - return NULL; + goto error; } if (!PyMapping_Check(ns)) { PyErr_Format(PyExc_TypeError, @@ -252,13 +243,13 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, } error: Py_XDECREF(cell); - Py_DECREF(ns); - Py_DECREF(meta); + Py_XDECREF(ns); + Py_XDECREF(meta); Py_XDECREF(mkw); - Py_DECREF(bases); if (bases != orig_bases) { Py_DECREF(orig_bases); } + Py_DECREF(bases); return cls; } From ebecffdb6d5fffa4249f9a813f1fc1915926feb5 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 7 Aug 2021 20:18:10 -0700 Subject: [PATCH 6/8] bpo-44830 - Remove the broken Broken Mozilla devguide link. (GH-27664) --- Doc/bugs.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 8ffc8b14c53e4e..82819792a1ad8c 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -80,10 +80,6 @@ taken on the bug. Article which goes into some detail about how to create a useful bug report. This describes what kind of information is useful and why it is useful. - `Bug Report Writing Guidelines `_ - Information about writing a good bug report. Some of this is specific to the - Mozilla project, but describes general good practices. - .. _contributing-to-python: Getting started contributing to Python yourself From 0eec6276fdcdde5221370d92b50ea95851760c72 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 8 Aug 2021 08:49:44 +0300 Subject: [PATCH 7/8] bpo-44859: Improve error handling in sqlite3 and and raise more accurate exceptions. (GH-27654) * MemoryError is now raised instead of sqlite3.Warning when memory is not enough for encoding a statement to UTF-8 in Connection.__call__() and Cursor.execute(). * UnicodEncodeError is now raised instead of sqlite3.Warning when the statement contains surrogate characters in Connection.__call__() and Cursor.execute(). * TypeError is now raised instead of ValueError for non-string script argument in Cursor.executescript(). * ValueError is now raised for script containing the null character instead of truncating it in Cursor.executescript(). * Correctly handle exceptions raised when getting boolean value of the result of the progress handler. * Add many tests covering different corner cases. Co-authored-by: Erlend Egeberg Aasland --- Lib/sqlite3/test/dbapi.py | 33 ++++++++++-- Lib/sqlite3/test/hooks.py | 29 ++++++++++- Lib/sqlite3/test/regression.py | 23 +++++++- Lib/sqlite3/test/types.py | 52 ++++++++++++++++++- Lib/sqlite3/test/userfunctions.py | 48 +++++++++++++---- .../2021-08-07-17-28-56.bpo-44859.CCopjk.rst | 8 +++ Modules/_sqlite/clinic/cursor.c.h | 31 ++++++++++- Modules/_sqlite/connection.c | 14 ++--- Modules/_sqlite/cursor.c | 37 +++++-------- Modules/_sqlite/statement.c | 3 -- 10 files changed, 226 insertions(+), 52 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-08-07-17-28-56.bpo-44859.CCopjk.rst diff --git a/Lib/sqlite3/test/dbapi.py b/Lib/sqlite3/test/dbapi.py index 408f9945f2c970..5d7e5bba05bc45 100644 --- a/Lib/sqlite3/test/dbapi.py +++ b/Lib/sqlite3/test/dbapi.py @@ -26,7 +26,7 @@ import threading import unittest -from test.support import check_disallow_instantiation, threading_helper +from test.support import check_disallow_instantiation, threading_helper, bigmemtest from test.support.os_helper import TESTFN, unlink @@ -758,9 +758,35 @@ def test_script_error_normal(self): def test_cursor_executescript_as_bytes(self): con = sqlite.connect(":memory:") cur = con.cursor() - with self.assertRaises(ValueError) as cm: + with self.assertRaises(TypeError): cur.executescript(b"create table test(foo); insert into test(foo) values (5);") - self.assertEqual(str(cm.exception), 'script argument must be unicode.') + + def test_cursor_executescript_with_null_characters(self): + con = sqlite.connect(":memory:") + cur = con.cursor() + with self.assertRaises(ValueError): + cur.executescript(""" + create table a(i);\0 + insert into a(i) values (5); + """) + + def test_cursor_executescript_with_surrogates(self): + con = sqlite.connect(":memory:") + cur = con.cursor() + with self.assertRaises(UnicodeEncodeError): + cur.executescript(""" + create table a(s); + insert into a(s) values ('\ud8ff'); + """) + + @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') + @bigmemtest(size=2**31, memuse=3, dry_run=False) + def test_cursor_executescript_too_large_script(self, maxsize): + con = sqlite.connect(":memory:") + cur = con.cursor() + for size in 2**31-1, 2**31: + with self.assertRaises(sqlite.DataError): + cur.executescript("create table a(s);".ljust(size)) def test_connection_execute(self): con = sqlite.connect(":memory:") @@ -969,6 +995,7 @@ def suite(): CursorTests, ExtensionTests, ModuleTests, + OpenTests, SqliteOnConflictTests, ThreadTests, UninitialisedConnectionTests, diff --git a/Lib/sqlite3/test/hooks.py b/Lib/sqlite3/test/hooks.py index 1be6d380abd20a..43e3810d13df18 100644 --- a/Lib/sqlite3/test/hooks.py +++ b/Lib/sqlite3/test/hooks.py @@ -24,7 +24,7 @@ import sqlite3 as sqlite from test.support.os_helper import TESTFN, unlink - +from .userfunctions import with_tracebacks class CollationTests(unittest.TestCase): def test_create_collation_not_string(self): @@ -145,7 +145,6 @@ def progress(): """) self.assertTrue(progress_calls) - def test_opcode_count(self): """ Test that the opcode argument is respected. @@ -198,6 +197,32 @@ def progress(): con.execute("select 1 union select 2 union select 3").fetchall() self.assertEqual(action, 0, "progress handler was not cleared") + @with_tracebacks(['bad_progress', 'ZeroDivisionError']) + def test_error_in_progress_handler(self): + con = sqlite.connect(":memory:") + def bad_progress(): + 1 / 0 + con.set_progress_handler(bad_progress, 1) + with self.assertRaises(sqlite.OperationalError): + con.execute(""" + create table foo(a, b) + """) + + @with_tracebacks(['__bool__', 'ZeroDivisionError']) + def test_error_in_progress_handler_result(self): + con = sqlite.connect(":memory:") + class BadBool: + def __bool__(self): + 1 / 0 + def bad_progress(): + return BadBool() + con.set_progress_handler(bad_progress, 1) + with self.assertRaises(sqlite.OperationalError): + con.execute(""" + create table foo(a, b) + """) + + class TraceCallbackTests(unittest.TestCase): def test_trace_callback_used(self): """ diff --git a/Lib/sqlite3/test/regression.py b/Lib/sqlite3/test/regression.py index 6c093d7c2c36e0..ddf36e71819445 100644 --- a/Lib/sqlite3/test/regression.py +++ b/Lib/sqlite3/test/regression.py @@ -21,6 +21,7 @@ # 3. This notice may not be removed or altered from any source distribution. import datetime +import sys import unittest import sqlite3 as sqlite import weakref @@ -273,7 +274,7 @@ def test_connection_call(self): Call a connection with a non-string SQL request: check error handling of the statement constructor. """ - self.assertRaises(TypeError, self.con, 1) + self.assertRaises(TypeError, self.con, b"select 1") def test_collation(self): def collation_cb(a, b): @@ -344,6 +345,26 @@ def test_null_character(self): self.assertRaises(ValueError, cur.execute, " \0select 2") self.assertRaises(ValueError, cur.execute, "select 2\0") + def test_surrogates(self): + con = sqlite.connect(":memory:") + self.assertRaises(UnicodeEncodeError, con, "select '\ud8ff'") + self.assertRaises(UnicodeEncodeError, con, "select '\udcff'") + cur = con.cursor() + self.assertRaises(UnicodeEncodeError, cur.execute, "select '\ud8ff'") + self.assertRaises(UnicodeEncodeError, cur.execute, "select '\udcff'") + + @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') + @support.bigmemtest(size=2**31, memuse=4, dry_run=False) + def test_large_sql(self, maxsize): + # Test two cases: size+1 > INT_MAX and size+1 <= INT_MAX. + for size in (2**31, 2**31-2): + con = sqlite.connect(":memory:") + sql = "select 1".ljust(size) + self.assertRaises(sqlite.DataError, con, sql) + cur = con.cursor() + self.assertRaises(sqlite.DataError, cur.execute, sql) + del sql + def test_commit_cursor_reset(self): """ Connection.commit() did reset cursors, which made sqlite3 diff --git a/Lib/sqlite3/test/types.py b/Lib/sqlite3/test/types.py index 4f0e4f6d268392..b8926ffee22e87 100644 --- a/Lib/sqlite3/test/types.py +++ b/Lib/sqlite3/test/types.py @@ -23,11 +23,14 @@ import datetime import unittest import sqlite3 as sqlite +import sys try: import zlib except ImportError: zlib = None +from test import support + class SqliteTypeTests(unittest.TestCase): def setUp(self): @@ -45,6 +48,12 @@ def test_string(self): row = self.cur.fetchone() self.assertEqual(row[0], "Österreich") + def test_string_with_null_character(self): + self.cur.execute("insert into test(s) values (?)", ("a\0b",)) + self.cur.execute("select s from test") + row = self.cur.fetchone() + self.assertEqual(row[0], "a\0b") + def test_small_int(self): self.cur.execute("insert into test(i) values (?)", (42,)) self.cur.execute("select i from test") @@ -52,7 +61,7 @@ def test_small_int(self): self.assertEqual(row[0], 42) def test_large_int(self): - num = 2**40 + num = 123456789123456789 self.cur.execute("insert into test(i) values (?)", (num,)) self.cur.execute("select i from test") row = self.cur.fetchone() @@ -78,6 +87,45 @@ def test_unicode_execute(self): row = self.cur.fetchone() self.assertEqual(row[0], "Österreich") + def test_too_large_int(self): + for value in 2**63, -2**63-1, 2**64: + with self.assertRaises(OverflowError): + self.cur.execute("insert into test(i) values (?)", (value,)) + self.cur.execute("select i from test") + row = self.cur.fetchone() + self.assertIsNone(row) + + def test_string_with_surrogates(self): + for value in 0xd8ff, 0xdcff: + with self.assertRaises(UnicodeEncodeError): + self.cur.execute("insert into test(s) values (?)", (chr(value),)) + self.cur.execute("select s from test") + row = self.cur.fetchone() + self.assertIsNone(row) + + @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') + @support.bigmemtest(size=2**31, memuse=4, dry_run=False) + def test_too_large_string(self, maxsize): + with self.assertRaises(sqlite.InterfaceError): + self.cur.execute("insert into test(s) values (?)", ('x'*(2**31-1),)) + with self.assertRaises(OverflowError): + self.cur.execute("insert into test(s) values (?)", ('x'*(2**31),)) + self.cur.execute("select 1 from test") + row = self.cur.fetchone() + self.assertIsNone(row) + + @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') + @support.bigmemtest(size=2**31, memuse=3, dry_run=False) + def test_too_large_blob(self, maxsize): + with self.assertRaises(sqlite.InterfaceError): + self.cur.execute("insert into test(s) values (?)", (b'x'*(2**31-1),)) + with self.assertRaises(OverflowError): + self.cur.execute("insert into test(s) values (?)", (b'x'*(2**31),)) + self.cur.execute("select 1 from test") + row = self.cur.fetchone() + self.assertIsNone(row) + + class DeclTypesTests(unittest.TestCase): class Foo: def __init__(self, _val): @@ -163,7 +211,7 @@ def test_small_int(self): def test_large_int(self): # default - num = 2**40 + num = 123456789123456789 self.cur.execute("insert into test(i) values (?)", (num,)) self.cur.execute("select i from test") row = self.cur.fetchone() diff --git a/Lib/sqlite3/test/userfunctions.py b/Lib/sqlite3/test/userfunctions.py index 9681dbdde2b092..b4d5181777ebdf 100644 --- a/Lib/sqlite3/test/userfunctions.py +++ b/Lib/sqlite3/test/userfunctions.py @@ -33,28 +33,37 @@ from test.support import bigmemtest -def with_tracebacks(strings): +def with_tracebacks(strings, traceback=True): """Convenience decorator for testing callback tracebacks.""" - strings.append('Traceback') + if traceback: + strings.append('Traceback') def decorator(func): @functools.wraps(func) def wrapper(self, *args, **kwargs): # First, run the test with traceback enabled. - sqlite.enable_callback_tracebacks(True) - buf = io.StringIO() - with contextlib.redirect_stderr(buf): + with check_tracebacks(self, strings): func(self, *args, **kwargs) - tb = buf.getvalue() - for s in strings: - self.assertIn(s, tb) # Then run the test with traceback disabled. - sqlite.enable_callback_tracebacks(False) func(self, *args, **kwargs) return wrapper return decorator +@contextlib.contextmanager +def check_tracebacks(self, strings): + """Convenience context manager for testing callback tracebacks.""" + sqlite.enable_callback_tracebacks(True) + try: + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + yield + tb = buf.getvalue() + for s in strings: + self.assertIn(s, tb) + finally: + sqlite.enable_callback_tracebacks(False) + def func_returntext(): return "foo" def func_returntextwithnull(): @@ -408,9 +417,26 @@ def md5sum(t): del x,y gc.collect() + def test_func_return_too_large_int(self): + cur = self.con.cursor() + for value in 2**63, -2**63-1, 2**64: + self.con.create_function("largeint", 0, lambda value=value: value) + with check_tracebacks(self, ['OverflowError']): + with self.assertRaises(sqlite.DataError): + cur.execute("select largeint()") + + def test_func_return_text_with_surrogates(self): + cur = self.con.cursor() + self.con.create_function("pychr", 1, chr) + for value in 0xd8ff, 0xdcff: + with check_tracebacks(self, + ['UnicodeEncodeError', 'surrogates not allowed']): + with self.assertRaises(sqlite.OperationalError): + cur.execute("select pychr(?)", (value,)) + @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') @bigmemtest(size=2**31, memuse=3, dry_run=False) - def test_large_text(self, size): + def test_func_return_too_large_text(self, size): cur = self.con.cursor() for size in 2**31-1, 2**31: self.con.create_function("largetext", 0, lambda size=size: "b" * size) @@ -419,7 +445,7 @@ def test_large_text(self, size): @unittest.skipUnless(sys.maxsize > 2**32, 'requires 64bit platform') @bigmemtest(size=2**31, memuse=2, dry_run=False) - def test_large_blob(self, size): + def test_func_return_too_large_blob(self, size): cur = self.con.cursor() for size in 2**31-1, 2**31: self.con.create_function("largeblob", 0, lambda size=size: b"b" * size) diff --git a/Misc/NEWS.d/next/Library/2021-08-07-17-28-56.bpo-44859.CCopjk.rst b/Misc/NEWS.d/next/Library/2021-08-07-17-28-56.bpo-44859.CCopjk.rst new file mode 100644 index 00000000000000..ec9f774d66b8c4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-07-17-28-56.bpo-44859.CCopjk.rst @@ -0,0 +1,8 @@ +Improve error handling in :mod:`sqlite3` and raise more accurate exceptions. + +* :exc:`MemoryError` is now raised instead of :exc:`sqlite3.Warning` when memory is not enough for encoding a statement to UTF-8 in ``Connection.__call__()`` and ``Cursor.execute()``. +* :exc:`UnicodEncodeError` is now raised instead of :exc:`sqlite3.Warning` when the statement contains surrogate characters in ``Connection.__call__()`` and ``Cursor.execute()``. +* :exc:`TypeError` is now raised instead of :exc:`ValueError` for non-string script argument in ``Cursor.executescript()``. +* :exc:`ValueError` is now raised for script containing the null character instead of truncating it in ``Cursor.executescript()``. +* Correctly handle exceptions raised when getting boolean value of the result of the progress handler. +* Add many tests covering different corner cases. diff --git a/Modules/_sqlite/clinic/cursor.c.h b/Modules/_sqlite/clinic/cursor.c.h index d2c453b38b4b9e..07e15870146cf7 100644 --- a/Modules/_sqlite/clinic/cursor.c.h +++ b/Modules/_sqlite/clinic/cursor.c.h @@ -119,6 +119,35 @@ PyDoc_STRVAR(pysqlite_cursor_executescript__doc__, #define PYSQLITE_CURSOR_EXECUTESCRIPT_METHODDEF \ {"executescript", (PyCFunction)pysqlite_cursor_executescript, METH_O, pysqlite_cursor_executescript__doc__}, +static PyObject * +pysqlite_cursor_executescript_impl(pysqlite_Cursor *self, + const char *sql_script); + +static PyObject * +pysqlite_cursor_executescript(pysqlite_Cursor *self, PyObject *arg) +{ + PyObject *return_value = NULL; + const char *sql_script; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("executescript", "argument", "str", arg); + goto exit; + } + Py_ssize_t sql_script_length; + sql_script = PyUnicode_AsUTF8AndSize(arg, &sql_script_length); + if (sql_script == NULL) { + goto exit; + } + if (strlen(sql_script) != (size_t)sql_script_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = pysqlite_cursor_executescript_impl(self, sql_script); + +exit: + return return_value; +} + PyDoc_STRVAR(pysqlite_cursor_fetchone__doc__, "fetchone($self, /)\n" "--\n" @@ -270,4 +299,4 @@ pysqlite_cursor_close(pysqlite_Cursor *self, PyTypeObject *cls, PyObject *const exit: return return_value; } -/*[clinic end generated code: output=7b216aba2439f5cf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ace31a7481aa3f41 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 0dab3e85160e82..67160c4c449aa1 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -997,6 +997,14 @@ static int _progress_handler(void* user_arg) ret = _PyObject_CallNoArg((PyObject*)user_arg); if (!ret) { + /* abort query if error occurred */ + rc = -1; + } + else { + rc = PyObject_IsTrue(ret); + Py_DECREF(ret); + } + if (rc < 0) { pysqlite_state *state = pysqlite_get_state(NULL); if (state->enable_callback_tracebacks) { PyErr_Print(); @@ -1004,12 +1012,6 @@ static int _progress_handler(void* user_arg) else { PyErr_Clear(); } - - /* abort query if error occurred */ - rc = 1; - } else { - rc = (int)PyObject_IsTrue(ret); - Py_DECREF(ret); } PyGILState_Release(gilstate); diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 2f4494690f9557..7308f3062da4b9 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -728,21 +728,21 @@ pysqlite_cursor_executemany_impl(pysqlite_Cursor *self, PyObject *sql, /*[clinic input] _sqlite3.Cursor.executescript as pysqlite_cursor_executescript - sql_script as script_obj: object + sql_script: str / Executes multiple SQL statements at once. Non-standard. [clinic start generated code]*/ static PyObject * -pysqlite_cursor_executescript(pysqlite_Cursor *self, PyObject *script_obj) -/*[clinic end generated code: output=115a8132b0f200fe input=ba3ec59df205e362]*/ +pysqlite_cursor_executescript_impl(pysqlite_Cursor *self, + const char *sql_script) +/*[clinic end generated code: output=8fd726dde1c65164 input=1ac0693dc8db02a8]*/ { _Py_IDENTIFIER(commit); - const char* script_cstr; sqlite3_stmt* statement; int rc; - Py_ssize_t sql_len; + size_t sql_len; PyObject* result; if (!check_cursor(self)) { @@ -751,21 +751,12 @@ pysqlite_cursor_executescript(pysqlite_Cursor *self, PyObject *script_obj) self->reset = 0; - if (PyUnicode_Check(script_obj)) { - script_cstr = PyUnicode_AsUTF8AndSize(script_obj, &sql_len); - if (!script_cstr) { - return NULL; - } - - int max_length = sqlite3_limit(self->connection->db, - SQLITE_LIMIT_LENGTH, -1); - if (sql_len >= max_length) { - PyErr_SetString(self->connection->DataError, - "query string is too large"); - return NULL; - } - } else { - PyErr_SetString(PyExc_ValueError, "script argument must be unicode."); + sql_len = strlen(sql_script); + int max_length = sqlite3_limit(self->connection->db, + SQLITE_LIMIT_LENGTH, -1); + if (sql_len >= (unsigned)max_length) { + PyErr_SetString(self->connection->DataError, + "query string is too large"); return NULL; } @@ -782,7 +773,7 @@ pysqlite_cursor_executescript(pysqlite_Cursor *self, PyObject *script_obj) Py_BEGIN_ALLOW_THREADS rc = sqlite3_prepare_v2(self->connection->db, - script_cstr, + sql_script, (int)sql_len + 1, &statement, &tail); @@ -816,8 +807,8 @@ pysqlite_cursor_executescript(pysqlite_Cursor *self, PyObject *script_obj) if (*tail == (char)0) { break; } - sql_len -= (tail - script_cstr); - script_cstr = tail; + sql_len -= (tail - sql_script); + sql_script = tail; } error: diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c index 983df2d50c975d..2d5c72d13b7edb 100644 --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -56,9 +56,6 @@ pysqlite_statement_create(pysqlite_Connection *connection, PyObject *sql) Py_ssize_t size; const char *sql_cstr = PyUnicode_AsUTF8AndSize(sql, &size); if (sql_cstr == NULL) { - PyErr_Format(connection->Warning, - "SQL is of wrong type ('%s'). Must be string.", - Py_TYPE(sql)->tp_name); return NULL; } From 2b496e79293a8b80e8ba0e514e186b3b1467b64b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 8 Aug 2021 21:04:02 +0300 Subject: [PATCH 8/8] bpo-42053: Remove misleading check in os.fwalk() (GH-27669) os.fwalk() does not support integer as the first argument, and never supported. --- Lib/os.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 8cc70a11e9bc89..ab7ef3c17798b9 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -461,8 +461,7 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= dirs.remove('CVS') # don't visit CVS directories """ sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd) - if not isinstance(top, int) or not hasattr(top, '__index__'): - top = fspath(top) + top = fspath(top) # Note: To guard against symlink races, we use the standard # lstat()/open()/fstat() trick. if not follow_symlinks: