From 9711265182f163ba381e7800e3748ac28710f9ef Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 1 Nov 2022 15:51:05 -0700 Subject: [PATCH 1/8] gh-98925: Lower marshal recursion depth for WASI (GH-98938) For wasmtime 2.0, the stack depth cost is 6% higher. This causes the default max `marshal` recursion depth to blow the stack. As the default marshal depth is 2000 and Windows is set to 1000, split the difference and choose 1500 for WASI to be safe. --- .gitignore | 1 + Lib/test/test_marshal.py | 2 ++ .../2022-10-31-18-03-10.gh-issue-98925.zpdjVd.rst | 2 ++ Python/marshal.c | 2 ++ 4 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-10-31-18-03-10.gh-issue-98925.zpdjVd.rst diff --git a/.gitignore b/.gitignore index 6934faa91e9874..5055e6d225c796 100644 --- a/.gitignore +++ b/.gitignore @@ -116,6 +116,7 @@ PCbuild/win32/ Tools/unicode/data/ /autom4te.cache /build/ +/builddir/ /config.cache /config.log /config.status diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index fe4f368bed42f4..54c5a324897d23 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -260,6 +260,8 @@ def test_recursion_limit(self): #if os.name == 'nt' and support.Py_DEBUG: if os.name == 'nt': MAX_MARSHAL_STACK_DEPTH = 1000 + elif sys.platform == 'wasi': + MAX_MARSHAL_STACK_DEPTH = 1500 else: MAX_MARSHAL_STACK_DEPTH = 2000 for i in range(MAX_MARSHAL_STACK_DEPTH - 2): diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-31-18-03-10.gh-issue-98925.zpdjVd.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-31-18-03-10.gh-issue-98925.zpdjVd.rst new file mode 100644 index 00000000000000..7c965dc752f70b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-31-18-03-10.gh-issue-98925.zpdjVd.rst @@ -0,0 +1,2 @@ +Lower the recursion depth for marshal on WASI to support (in-development) +wasmtime 2.0. diff --git a/Python/marshal.c b/Python/marshal.c index 90a44050918006..2690f55766c83b 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -34,6 +34,8 @@ module marshal */ #if defined(MS_WINDOWS) #define MAX_MARSHAL_STACK_DEPTH 1000 +#elif defined(__wasi__) +#define MAX_MARSHAL_STACK_DEPTH 1500 #else #define MAX_MARSHAL_STACK_DEPTH 2000 #endif From 50a9b037a6603e73f0a850f5d43dbdd2656e60b1 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Tue, 1 Nov 2022 23:00:17 +0000 Subject: [PATCH 2/8] =?UTF-8?q?[doc]=20Update=20cookbook=20example=20for?= =?UTF-8?q?=20socket-based=20logging=20in=20a=20production=20sett=E2=80=A6?= =?UTF-8?q?=20(GH-98922)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: C.A.M. Gerlach --- Doc/howto/logging-cookbook.rst | 72 ++++++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index ae101e3cdf2501..bf6f54a841a7b9 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -765,13 +765,71 @@ serialization. Running a logging socket listener in production ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To run a logging listener in production, you may need to use a process-management tool -such as `Supervisor `_. `Here -`_ is a Gist which -provides the bare-bones files to run the above functionality using Supervisor: you -will need to change the ``/path/to/`` parts in the Gist to reflect the actual paths you -want to use. - +.. _socket-listener-gist: https://gist.github.com/vsajip/4b227eeec43817465ca835ca66f75e2b + +To run a logging listener in production, you may need to use a +process-management tool such as `Supervisor `_. +`Here is a Gist `__ +which provides the bare-bones files to run the above functionality using +Supervisor. It consists of the following files: + ++-------------------------+----------------------------------------------------+ +| File | Purpose | ++=========================+====================================================+ +| :file:`prepare.sh` | A Bash script to prepare the environment for | +| | testing | ++-------------------------+----------------------------------------------------+ +| :file:`supervisor.conf` | The Supervisor configuration file, which has | +| | entries for the listener and a multi-process web | +| | application | ++-------------------------+----------------------------------------------------+ +| :file:`ensure_app.sh` | A Bash script to ensure that Supervisor is running | +| | with the above configuration | ++-------------------------+----------------------------------------------------+ +| :file:`log_listener.py` | The socket listener program which receives log | +| | events and records them to a file | ++-------------------------+----------------------------------------------------+ +| :file:`main.py` | A simple web application which performs logging | +| | via a socket connected to the listener | ++-------------------------+----------------------------------------------------+ +| :file:`webapp.json` | A JSON configuration file for the web application | ++-------------------------+----------------------------------------------------+ +| :file:`client.py` | A Python script to exercise the web application | ++-------------------------+----------------------------------------------------+ + +The web application uses `Gunicorn `_, which is a +popular web application server that starts multiple worker processes to handle +requests. This example setup shows how the workers can write to the same log file +without conflicting with one another --- they all go through the socket listener. + +To test these files, do the following in a POSIX environment: + +#. Download `the Gist `__ + as a ZIP archive using the :guilabel:`Download ZIP` button. + +#. Unzip the above files from the archive into a scratch directory. + +#. In the scratch directory, run ``bash prepare.sh`` to get things ready. + This creates a :file:`run` subdirectory to contain Supervisor-related and + log files, and a :file:`venv` subdirectory to contain a virtual environment + into which ``bottle``, ``gunicorn`` and ``supervisor`` are installed. + +#. Run ``bash ensure_app.sh`` to ensure that Supervisor is running with + the above configuration. + +#. Run ``venv/bin/python client.py`` to exercise the web application, + which will lead to records being written to the log. + +#. Inspect the log files in the :file:`run` subdirectory. You should see the + most recent log lines in files matching the pattern :file:`app.log*`. They won't be in + any particular order, since they have been handled concurrently by different + worker processes in a non-deterministic way. + +#. You can shut down the listener and the web application by running + ``venv/bin/supervisorctl -c supervisor.conf shutdown``. + +You may need to tweak the configuration files in the unlikely event that the +configured ports clash with something else in your test environment. .. _context-info: From 65d1407737befc3e9430eda8ae14ab1c7f9d8c7a Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 2 Nov 2022 04:51:45 +0100 Subject: [PATCH 3/8] gh-92871: Postpone the removal of typing.{io,re} to 3.13 (#98958) --- Doc/library/typing.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 8c8286059d531e..0ec4499f94f5a7 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2009,7 +2009,7 @@ Other concrete types represent the types of I/O streams such as returned by :func:`open`. - .. deprecated-removed:: 3.8 3.12 + .. deprecated-removed:: 3.8 3.13 The ``typing.io`` namespace is deprecated and will be removed. These types should be directly imported from ``typing`` instead. @@ -2023,7 +2023,7 @@ Other concrete types ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or ``Match[bytes]``. - .. deprecated-removed:: 3.8 3.12 + .. deprecated-removed:: 3.8 3.13 The ``typing.re`` namespace is deprecated and will be removed. These types should be directly imported from ``typing`` instead. @@ -2868,7 +2868,7 @@ convenience. This is subject to change, and not all deprecations are listed. +----------------------------------+---------------+-------------------+----------------+ | Feature | Deprecated in | Projected removal | PEP/issue | +==================================+===============+===================+================+ -| ``typing.io`` and ``typing.re`` | 3.8 | 3.12 | :issue:`38291` | +| ``typing.io`` and ``typing.re`` | 3.8 | 3.13 | :issue:`38291` | | submodules | | | | +----------------------------------+---------------+-------------------+----------------+ | ``typing`` versions of standard | 3.9 | Undecided | :pep:`585` | From 898d0d9ad8328326ba5f7df9952e9dc287766ca3 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Wed, 2 Nov 2022 08:52:19 +0400 Subject: [PATCH 4/8] gh-92679: Clarify asyncio.loop.start_tls parameters (#92682) --- Doc/library/asyncio-eventloop.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 4f12074dd70098..d0a1ed2b99e55d 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -877,9 +877,14 @@ TLS Upgrade Upgrade an existing transport-based connection to TLS. - Return a new transport instance, that the *protocol* must start using - immediately after the *await*. The *transport* instance passed to - the *start_tls* method should never be used again. + Create a TLS coder/decoder instance and insert it between the *transport* + and the *protocol*. The coder/decoder implements both *transport*-facing + protocol and *protocol*-facing transport. + + Return the created two-interface instance. After *await*, the *protocol* + must stop using the original *transport* and communicate with the returned + object only because the coder caches *protocol*-side data and sporadically + exchanges extra TLS session packets with *transport*. Parameters: From 0689b99bb8c4f6058af43a52effaa8a25609dbed Mon Sep 17 00:00:00 2001 From: Stanley <46876382+slateny@users.noreply.github.com> Date: Tue, 1 Nov 2022 21:58:44 -0700 Subject: [PATCH 5/8] gh-96265: Formatting changes for faq/programming (#98242) * Formatting changes for faq/programming * Add missing method formatting, use non-literal formatting * Fix sphinx warnings * Some extra formatting missed earlier * More formatting suggestions from review Co-authored-by: C.A.M. Gerlach * Add missing colon, avoid referening external module Co-authored-by: C.A.M. Gerlach --- Doc/faq/programming.rst | 87 +++++++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 39 deletions(-) diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index ad281f826dd42c..f3c5b0f76494aa 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -25,8 +25,9 @@ Reference Manual `. You can also write your own debugger by using the code for pdb as an example. The IDLE interactive development environment, which is part of the standard -Python distribution (normally available as Tools/scripts/idle), includes a -graphical debugger. +Python distribution (normally available as +`Tools/scripts/idle3 `_), +includes a graphical debugger. PythonWin is a Python IDE that includes a GUI debugger based on pdb. The PythonWin debugger colors breakpoints and has quite a few cool features such as @@ -78,7 +79,8 @@ set of modules required by a program and bind these modules together with a Python binary to produce a single executable. One is to use the freeze tool, which is included in the Python source tree as -``Tools/freeze``. It converts Python byte code to C arrays; with a C compiler you can +`Tools/freeze `_. +It converts Python byte code to C arrays; with a C compiler you can embed all your modules into a new program, which is then linked with the standard Python modules. @@ -114,7 +116,7 @@ Core Language Why am I getting an UnboundLocalError when the variable has a value? -------------------------------------------------------------------- -It can be a surprise to get the UnboundLocalError in previously working +It can be a surprise to get the :exc:`UnboundLocalError` in previously working code when it is modified by adding an assignment statement somewhere in the body of a function. @@ -123,6 +125,7 @@ This code: >>> x = 10 >>> def bar(): ... print(x) + ... >>> bar() 10 @@ -133,7 +136,7 @@ works, but this code: ... print(x) ... x += 1 -results in an UnboundLocalError: +results in an :exc:`!UnboundLocalError`: >>> foo() Traceback (most recent call last): @@ -155,6 +158,7 @@ global: ... global x ... print(x) ... x += 1 + ... >>> foobar() 10 @@ -176,6 +180,7 @@ keyword: ... x += 1 ... bar() ... print(x) + ... >>> foo() 10 11 @@ -273,7 +278,7 @@ main.py:: import mod print(config.x) -Note that using a module is also the basis for implementing the Singleton design +Note that using a module is also the basis for implementing the singleton design pattern, for the same reason. @@ -291,9 +296,9 @@ using multiple imports per line uses less screen space. It's good practice if you import modules in the following order: -1. standard library modules -- e.g. ``sys``, ``os``, ``getopt``, ``re`` +1. standard library modules -- e.g. :mod:`sys`, :mod:`os`, :mod:`argparse`, :mod:`re` 2. third-party library modules (anything installed in Python's site-packages - directory) -- e.g. mx.DateTime, ZODB, PIL.Image, etc. + directory) -- e.g. :mod:`!dateutil`, :mod:`!requests`, :mod:`!PIL.Image` 3. locally developed modules It is sometimes necessary to move imports to a function or class to avoid @@ -471,7 +476,7 @@ object ``x`` refers to). After this assignment we have two objects (the ints Some operations (for example ``y.append(10)`` and ``y.sort()``) mutate the object, whereas superficially similar operations (for example ``y = y + [10]`` -and ``sorted(y)``) create a new object. In general in Python (and in all cases +and :func:`sorted(y) `) create a new object. In general in Python (and in all cases in the standard library) a method that mutates an object will return ``None`` to help avoid getting the two types of operations confused. So if you mistakenly write ``y.sort()`` thinking it will give you a sorted copy of ``y``, @@ -644,7 +649,7 @@ Sequences can be copied by slicing:: How can I find the methods or attributes of an object? ------------------------------------------------------ -For an instance x of a user-defined class, ``dir(x)`` returns an alphabetized +For an instance ``x`` of a user-defined class, :func:`dir(x) ` returns an alphabetized list of the names containing the instance attributes and methods and attributes defined by its class. @@ -669,9 +674,9 @@ callable. Consider the following code:: <__main__.A object at 0x16D07CC> Arguably the class has a name: even though it is bound to two names and invoked -through the name B the created instance is still reported as an instance of -class A. However, it is impossible to say whether the instance's name is a or -b, since both names are bound to the same value. +through the name ``B`` the created instance is still reported as an instance of +class ``A``. However, it is impossible to say whether the instance's name is ``a`` or +``b``, since both names are bound to the same value. Generally speaking it should not be necessary for your code to "know the names" of particular values. Unless you are deliberately writing introspective @@ -841,7 +846,7 @@ How do I get int literal attribute instead of SyntaxError? ---------------------------------------------------------- Trying to lookup an ``int`` literal attribute in the normal manner gives -a syntax error because the period is seen as a decimal point:: +a :exc:`SyntaxError` because the period is seen as a decimal point:: >>> 1.__class__ File "", line 1 @@ -887,7 +892,7 @@ leading '0' in a decimal number (except '0'). How do I convert a number to a string? -------------------------------------- -To convert, e.g., the number 144 to the string '144', use the built-in type +To convert, e.g., the number ``144`` to the string ``'144'``, use the built-in type constructor :func:`str`. If you want a hexadecimal or octal representation, use the built-in functions :func:`hex` or :func:`oct`. For fancy formatting, see the :ref:`f-strings` and :ref:`formatstrings` sections, @@ -1006,11 +1011,11 @@ Not as such. For simple input parsing, the easiest approach is usually to split the line into whitespace-delimited words using the :meth:`~str.split` method of string objects and then convert decimal strings to numeric values using :func:`int` or -:func:`float`. ``split()`` supports an optional "sep" parameter which is useful +:func:`float`. :meth:`!split()` supports an optional "sep" parameter which is useful if the line uses something other than whitespace as a separator. For more complicated input parsing, regular expressions are more powerful -than C's :c:func:`sscanf` and better suited for the task. +than C's ``sscanf`` and better suited for the task. What does 'UnicodeDecodeError' or 'UnicodeEncodeError' error mean? @@ -1206,15 +1211,16 @@ difference is that a Python list can contain objects of many different types. The ``array`` module also provides methods for creating arrays of fixed types with compact representations, but they are slower to index than lists. Also -note that NumPy and other third party packages define array-like structures with +note that `NumPy `_ +and other third party packages define array-like structures with various characteristics as well. -To get Lisp-style linked lists, you can emulate cons cells using tuples:: +To get Lisp-style linked lists, you can emulate *cons cells* using tuples:: lisp_list = ("like", ("this", ("example", None) ) ) If mutability is desired, you could use lists instead of tuples. Here the -analogue of lisp car is ``lisp_list[0]`` and the analogue of cdr is +analogue of a Lisp *car* is ``lisp_list[0]`` and the analogue of *cdr* is ``lisp_list[1]``. Only do this if you're sure you really need to, because it's usually a lot slower than using Python lists. @@ -1334,11 +1340,12 @@ that even though there was an error, the append worked:: ['foo', 'item'] To see why this happens, you need to know that (a) if an object implements an -``__iadd__`` magic method, it gets called when the ``+=`` augmented assignment +:meth:`~object.__iadd__` magic method, it gets called when the ``+=`` augmented +assignment is executed, and its return value is what gets used in the assignment statement; -and (b) for lists, ``__iadd__`` is equivalent to calling ``extend`` on the list +and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`~list.extend` on the list and returning the list. That's why we say that for lists, ``+=`` is a -"shorthand" for ``list.extend``:: +"shorthand" for :meth:`!list.extend`:: >>> a_list = [] >>> a_list += [1] @@ -1363,7 +1370,7 @@ Thus, in our tuple example what is happening is equivalent to:: ... TypeError: 'tuple' object does not support item assignment -The ``__iadd__`` succeeds, and thus the list is extended, but even though +The :meth:`!__iadd__` succeeds, and thus the list is extended, but even though ``result`` points to the same object that ``a_tuple[0]`` already points to, that final assignment still results in an error, because tuples are immutable. @@ -1440,7 +1447,8 @@ See also :ref:`why-self`. How do I check if an object is an instance of a given class or of a subclass of it? ----------------------------------------------------------------------------------- -Use the built-in function ``isinstance(obj, cls)``. You can check if an object +Use the built-in function :func:`isinstance(obj, cls) `. You can +check if an object is an instance of any of a number of classes by providing a tuple instead of a single class, e.g. ``isinstance(obj, (class1, class2, ...))``, and can also check whether an object is one of Python's built-in types, e.g. @@ -1537,13 +1545,13 @@ Here the ``UpperOut`` class redefines the ``write()`` method to convert the argument string to uppercase before calling the underlying ``self._outfile.write()`` method. All other methods are delegated to the underlying ``self._outfile`` object. The delegation is accomplished via the -``__getattr__`` method; consult :ref:`the language reference ` +:meth:`~object.__getattr__` method; consult :ref:`the language reference ` for more information about controlling attribute access. Note that for more general cases delegation can get trickier. When attributes -must be set as well as retrieved, the class must define a :meth:`__setattr__` +must be set as well as retrieved, the class must define a :meth:`~object.__setattr__` method too, and it must do so carefully. The basic implementation of -:meth:`__setattr__` is roughly equivalent to the following:: +:meth:`!__setattr__` is roughly equivalent to the following:: class X: ... @@ -1551,7 +1559,8 @@ method too, and it must do so carefully. The basic implementation of self.__dict__[name] = value ... -Most :meth:`__setattr__` implementations must modify ``self.__dict__`` to store +Most :meth:`!__setattr__` implementations must modify +:meth:`self.__dict__ ` to store local state for self without causing an infinite recursion. @@ -1689,17 +1698,17 @@ My class defines __del__ but it is not called when I delete the object. There are several possible reasons for this. -The del statement does not necessarily call :meth:`__del__` -- it simply +The :keyword:`del` statement does not necessarily call :meth:`~object.__del__` -- it simply decrements the object's reference count, and if this reaches zero -:meth:`__del__` is called. +:meth:`!__del__` is called. If your data structures contain circular links (e.g. a tree where each child has a parent reference and each parent has a list of children) the reference counts will never go back to zero. Once in a while Python runs an algorithm to detect such cycles, but the garbage collector might run some time after the last -reference to your data structure vanishes, so your :meth:`__del__` method may be +reference to your data structure vanishes, so your :meth:`!__del__` method may be called at an inconvenient and random time. This is inconvenient if you're trying -to reproduce a problem. Worse, the order in which object's :meth:`__del__` +to reproduce a problem. Worse, the order in which object's :meth:`!__del__` methods are executed is arbitrary. You can run :func:`gc.collect` to force a collection, but there *are* pathological cases where objects will never be collected. @@ -1707,7 +1716,7 @@ collected. Despite the cycle collector, it's still a good idea to define an explicit ``close()`` method on objects to be called whenever you're done with them. The ``close()`` method can then remove attributes that refer to subobjects. Don't -call :meth:`__del__` directly -- :meth:`__del__` should call ``close()`` and +call :meth:`!__del__` directly -- :meth:`!__del__` should call ``close()`` and ``close()`` should make sure that it can be called more than once for the same object. @@ -1724,7 +1733,7 @@ and sibling references (if they need them!). Normally, calling :func:`sys.exc_clear` will take care of this by clearing the last recorded exception. -Finally, if your :meth:`__del__` method raises an exception, a warning message +Finally, if your :meth:`!__del__` method raises an exception, a warning message is printed to :data:`sys.stderr`. @@ -1852,8 +1861,8 @@ For example, here is the implementation of How can a subclass control what data is stored in an immutable instance? ------------------------------------------------------------------------ -When subclassing an immutable type, override the :meth:`__new__` method -instead of the :meth:`__init__` method. The latter only runs *after* an +When subclassing an immutable type, override the :meth:`~object.__new__` method +instead of the :meth:`~object.__init__` method. The latter only runs *after* an instance is created, which is too late to alter data in an immutable instance. @@ -1955,8 +1964,8 @@ can't be made to work because it cannot detect changes to the attributes. To make the *lru_cache* approach work when the *station_id* is mutable, -the class needs to define the *__eq__* and *__hash__* methods so that -the cache can detect relevant attribute updates:: +the class needs to define the :meth:`~object.__eq__` and :meth:`~object.__hash__` +methods so that the cache can detect relevant attribute updates:: class Weather: "Example with a mutable station identifier" From c76db37c0d23174cbffd6fa978d39693890ef020 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Nov 2022 15:37:40 +0100 Subject: [PATCH 6/8] gh-98903: Test suite fails with exit code 4 if no tests ran (#98904) The Python test suite now fails wit exit code 4 if no tests ran. It should help detecting typos in test names and test methods. * Add "EXITCODE_" constants to Lib/test/libregrtest/main.py. * Fix a typo: "NO TEST RUN" becomes "NO TESTS RAN" --- Lib/test/libregrtest/main.py | 22 +++++--- Lib/test/test_regrtest.py | 54 ++++++++++++------- ...2-10-31-14-47-49.gh-issue-98903.7KinCV.rst | 2 + 3 files changed, 52 insertions(+), 26 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2022-10-31-14-47-49.gh-issue-98903.7KinCV.rst diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 655e4d2e56f8f0..3eeef029b22d48 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -28,6 +28,11 @@ # Must be smaller than buildbot "1200 seconds without output" limit. EXIT_TIMEOUT = 120.0 +EXITCODE_BAD_TEST = 2 +EXITCODE_INTERRUPTED = 130 +EXITCODE_ENV_CHANGED = 3 +EXITCODE_NO_TESTS_RAN = 4 + class Regrtest: """Execute a test suite. @@ -493,15 +498,18 @@ def display_header(self): print("== encodings: locale=%s, FS=%s" % (locale.getencoding(), sys.getfilesystemencoding())) + def no_tests_run(self): + return not any((self.good, self.bad, self.skipped, self.interrupted, + self.environment_changed)) + def get_tests_result(self): result = [] if self.bad: result.append("FAILURE") elif self.ns.fail_env_changed and self.environment_changed: result.append("ENV CHANGED") - elif not any((self.good, self.bad, self.skipped, self.interrupted, - self.environment_changed)): - result.append("NO TEST RUN") + elif self.no_tests_run(): + result.append("NO TESTS RAN") if self.interrupted: result.append("INTERRUPTED") @@ -750,11 +758,13 @@ def _main(self, tests, kwargs): self.save_xml_result() if self.bad: - sys.exit(2) + sys.exit(EXITCODE_BAD_TEST) if self.interrupted: - sys.exit(130) + sys.exit(EXITCODE_INTERRUPTED) if self.ns.fail_env_changed and self.environment_changed: - sys.exit(3) + sys.exit(EXITCODE_ENV_CHANGED) + if self.no_tests_run(): + sys.exit(EXITCODE_NO_TESTS_RAN) sys.exit(0) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index a36d18488a5ef7..baae4efc2ad789 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -28,6 +28,11 @@ ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?' +EXITCODE_BAD_TEST = 2 +EXITCODE_ENV_CHANGED = 3 +EXITCODE_NO_TESTS_RAN = 4 +EXITCODE_INTERRUPTED = 130 + TEST_INTERRUPTED = textwrap.dedent(""" from signal import SIGINT, raise_signal try: @@ -497,7 +502,7 @@ def list_regex(line_format, tests): result.append('INTERRUPTED') if not any((good, result, failed, interrupted, skipped, env_changed, fail_env_changed)): - result.append("NO TEST RUN") + result.append("NO TESTS RAN") elif not result: result.append('SUCCESS') result = ', '.join(result) @@ -707,7 +712,7 @@ def test_failing(self): test_failing = self.create_test('failing', code=code) tests = [test_ok, test_failing] - output = self.run_tests(*tests, exitcode=2) + output = self.run_tests(*tests, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, tests, failed=test_failing) def test_resources(self): @@ -748,13 +753,14 @@ def test_random(self): test = self.create_test('random', code) # first run to get the output with the random seed - output = self.run_tests('-r', test) + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN) randseed = self.parse_random_seed(output) match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) test_random = int(match.group(1)) # try to reproduce with the random seed - output = self.run_tests('-r', '--randseed=%s' % randseed, test) + output = self.run_tests('-r', '--randseed=%s' % randseed, test, + exitcode=EXITCODE_NO_TESTS_RAN) randseed2 = self.parse_random_seed(output) self.assertEqual(randseed2, randseed) @@ -813,7 +819,7 @@ def test_fromfile(self): def test_interrupted(self): code = TEST_INTERRUPTED test = self.create_test('sigint', code=code) - output = self.run_tests(test, exitcode=130) + output = self.run_tests(test, exitcode=EXITCODE_INTERRUPTED) self.check_executed_tests(output, test, omitted=test, interrupted=True) @@ -838,7 +844,7 @@ def test_slowest_interrupted(self): args = ("--slowest", "-j2", test) else: args = ("--slowest", test) - output = self.run_tests(*args, exitcode=130) + output = self.run_tests(*args, exitcode=EXITCODE_INTERRUPTED) self.check_executed_tests(output, test, omitted=test, interrupted=True) @@ -878,7 +884,7 @@ def test_run(self): builtins.__dict__['RUN'] = 1 """) test = self.create_test('forever', code=code) - output = self.run_tests('--forever', test, exitcode=2) + output = self.run_tests('--forever', test, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, [test]*3, failed=test) def check_leak(self, code, what): @@ -887,7 +893,7 @@ def check_leak(self, code, what): filename = 'reflog.txt' self.addCleanup(os_helper.unlink, filename) output = self.run_tests('--huntrleaks', '3:3:', test, - exitcode=2, + exitcode=EXITCODE_BAD_TEST, stderr=subprocess.STDOUT) self.check_executed_tests(output, [test], failed=test) @@ -969,7 +975,7 @@ def test_crashed(self): crash_test = self.create_test(name="crash", code=code) tests = [crash_test] - output = self.run_tests("-j2", *tests, exitcode=2) + output = self.run_tests("-j2", *tests, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, tests, failed=crash_test, randomize=True) @@ -1069,7 +1075,8 @@ def test_env_changed(self): self.check_executed_tests(output, [testname], env_changed=testname) # fail with --fail-env-changed - output = self.run_tests("--fail-env-changed", testname, exitcode=3) + output = self.run_tests("--fail-env-changed", testname, + exitcode=EXITCODE_ENV_CHANGED) self.check_executed_tests(output, [testname], env_changed=testname, fail_env_changed=True) @@ -1088,7 +1095,7 @@ def test_fail_always(self): """) testname = self.create_test(code=code) - output = self.run_tests("-w", testname, exitcode=2) + output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, [testname], failed=testname, rerun={testname: "test_fail_always"}) @@ -1123,7 +1130,8 @@ def test_bug(self): """) testname = self.create_test(code=code) - output = self.run_tests(testname, "-m", "nosuchtest", exitcode=0) + output = self.run_tests(testname, "-m", "nosuchtest", + exitcode=EXITCODE_NO_TESTS_RAN) self.check_executed_tests(output, [testname], no_test_ran=testname) def test_no_tests_ran_skip(self): @@ -1136,7 +1144,7 @@ def test_skipped(self): """) testname = self.create_test(code=code) - output = self.run_tests(testname, exitcode=0) + output = self.run_tests(testname) self.check_executed_tests(output, [testname]) def test_no_tests_ran_multiple_tests_nonexistent(self): @@ -1150,7 +1158,8 @@ def test_bug(self): testname = self.create_test(code=code) testname2 = self.create_test(code=code) - output = self.run_tests(testname, testname2, "-m", "nosuchtest", exitcode=0) + output = self.run_tests(testname, testname2, "-m", "nosuchtest", + exitcode=EXITCODE_NO_TESTS_RAN) self.check_executed_tests(output, [testname, testname2], no_test_ran=[testname, testname2]) @@ -1198,7 +1207,8 @@ def test_garbage(self): """) testname = self.create_test(code=code) - output = self.run_tests("--fail-env-changed", testname, exitcode=3) + output = self.run_tests("--fail-env-changed", testname, + exitcode=EXITCODE_ENV_CHANGED) self.check_executed_tests(output, [testname], env_changed=[testname], fail_env_changed=True) @@ -1224,7 +1234,8 @@ def test_sleep(self): """) testname = self.create_test(code=code) - output = self.run_tests("-j2", "--timeout=1.0", testname, exitcode=2) + output = self.run_tests("-j2", "--timeout=1.0", testname, + exitcode=EXITCODE_BAD_TEST) self.check_executed_tests(output, [testname], failed=testname) self.assertRegex(output, @@ -1256,7 +1267,8 @@ def test_unraisable_exc(self): """) testname = self.create_test(code=code) - output = self.run_tests("--fail-env-changed", "-v", testname, exitcode=3) + output = self.run_tests("--fail-env-changed", "-v", testname, + exitcode=EXITCODE_ENV_CHANGED) self.check_executed_tests(output, [testname], env_changed=[testname], fail_env_changed=True) @@ -1287,7 +1299,8 @@ def test_threading_excepthook(self): """) testname = self.create_test(code=code) - output = self.run_tests("--fail-env-changed", "-v", testname, exitcode=3) + output = self.run_tests("--fail-env-changed", "-v", testname, + exitcode=EXITCODE_ENV_CHANGED) self.check_executed_tests(output, [testname], env_changed=[testname], fail_env_changed=True) @@ -1328,7 +1341,7 @@ def test_print_warning(self): for option in ("-v", "-W"): with self.subTest(option=option): cmd = ["--fail-env-changed", option, testname] - output = self.run_tests(*cmd, exitcode=3) + output = self.run_tests(*cmd, exitcode=EXITCODE_ENV_CHANGED) self.check_executed_tests(output, [testname], env_changed=[testname], fail_env_changed=True) @@ -1373,7 +1386,8 @@ def test_leak_tmp_file(self): """) testnames = [self.create_test(code=code) for _ in range(3)] - output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames, exitcode=3) + output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames, + exitcode=EXITCODE_ENV_CHANGED) self.check_executed_tests(output, testnames, env_changed=testnames, fail_env_changed=True, diff --git a/Misc/NEWS.d/next/Tests/2022-10-31-14-47-49.gh-issue-98903.7KinCV.rst b/Misc/NEWS.d/next/Tests/2022-10-31-14-47-49.gh-issue-98903.7KinCV.rst new file mode 100644 index 00000000000000..65636ab0683f39 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-10-31-14-47-49.gh-issue-98903.7KinCV.rst @@ -0,0 +1,2 @@ +The Python test suite now fails wit exit code 4 if no tests ran. It should +help detecting typos in test names and test methods. From 6d683d85252df3c8dba7c33f7db87cdc1bcb0bf0 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 2 Nov 2022 15:13:07 +0000 Subject: [PATCH 7/8] gh-87092: do not allocate PyFutureFeatures dynamically (GH-98913) --- Include/internal/pycore_compile.h | 7 ++++--- Python/compile.c | 22 ++++++++++------------ Python/future.c | 16 ++++------------ Python/symtable.c | 9 ++++----- 4 files changed, 22 insertions(+), 32 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 1a628a08ca4ebf..cb490abe77a59d 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -18,10 +18,11 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( PyCompilerFlags *flags, int optimize, struct _arena *arena); -extern PyFutureFeatures* _PyFuture_FromAST( + +int _PyFuture_FromAST( struct _mod * mod, - PyObject *filename - ); + PyObject *filename, + PyFutureFeatures* futures); extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); diff --git a/Python/compile.c b/Python/compile.c index f8924789f4e954..2ab20a813d2a31 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -417,7 +417,7 @@ handled by the symbol analysis pass. struct compiler { PyObject *c_filename; struct symtable *c_st; - PyFutureFeatures *c_future; /* pointer to module's __future__ */ + PyFutureFeatures c_future; /* module's __future__ */ PyCompilerFlags *c_flags; int c_optimize; /* optimization level */ @@ -619,14 +619,14 @@ _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, Py_INCREF(filename); c.c_filename = filename; c.c_arena = arena; - c.c_future = _PyFuture_FromAST(mod, filename); - if (c.c_future == NULL) + if (!_PyFuture_FromAST(mod, filename, &c.c_future)) { goto finally; + } if (!flags) { flags = &local_flags; } - merged = c.c_future->ff_features | flags->cf_flags; - c.c_future->ff_features = merged; + merged = c.c_future.ff_features | flags->cf_flags; + c.c_future.ff_features = merged; flags->cf_flags = merged; c.c_flags = flags; c.c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; @@ -640,7 +640,7 @@ _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, goto finally; } - c.c_st = _PySymtable_Build(mod, filename, c.c_future); + c.c_st = _PySymtable_Build(mod, filename, &c.c_future); if (c.c_st == NULL) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_SystemError, "no symtable"); @@ -660,8 +660,6 @@ compiler_free(struct compiler *c) { if (c->c_st) _PySymtable_Free(c->c_st); - if (c->c_future) - PyObject_Free(c->c_future); Py_XDECREF(c->c_filename); Py_DECREF(c->c_const_cache); Py_DECREF(c->c_stack); @@ -2404,7 +2402,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id, ADDOP_LOAD_CONST(c, loc, mangled); Py_DECREF(mangled); - if (c->c_future->ff_features & CO_FUTURE_ANNOTATIONS) { + if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { VISIT(c, annexpr, annotation); } else { @@ -3927,7 +3925,7 @@ compiler_from_import(struct compiler *c, stmt_ty s) PyTuple_SET_ITEM(names, i, alias->name); } - if (location_is_after(LOC(s), c->c_future->ff_location) && + if (location_is_after(LOC(s), c->c_future.ff_location) && s->v.ImportFrom.module && _PyUnicode_EqualToASCIIString(s->v.ImportFrom.module, "__future__")) { @@ -6056,7 +6054,7 @@ check_annotation(struct compiler *c, stmt_ty s) { /* Annotations of complex targets does not produce anything under annotations future */ - if (c->c_future->ff_features & CO_FUTURE_ANNOTATIONS) { + if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { return 1; } @@ -6122,7 +6120,7 @@ compiler_annassign(struct compiler *c, stmt_ty s) if (s->v.AnnAssign.simple && (c->u->u_scope_type == COMPILER_SCOPE_MODULE || c->u->u_scope_type == COMPILER_SCOPE_CLASS)) { - if (c->c_future->ff_features & CO_FUTURE_ANNOTATIONS) { + if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { VISIT(c, annexpr, s->v.AnnAssign.annotation) } else { diff --git a/Python/future.c b/Python/future.c index 2a45d2ebeab9a1..d56f7330964684 100644 --- a/Python/future.c +++ b/Python/future.c @@ -96,22 +96,14 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename) } -PyFutureFeatures * -_PyFuture_FromAST(mod_ty mod, PyObject *filename) +int +_PyFuture_FromAST(mod_ty mod, PyObject *filename, PyFutureFeatures *ff) { - PyFutureFeatures *ff; - - ff = (PyFutureFeatures *)PyObject_Malloc(sizeof(PyFutureFeatures)); - if (ff == NULL) { - PyErr_NoMemory(); - return NULL; - } ff->ff_features = 0; ff->ff_location = (_PyCompilerSrcLocation){-1, -1, -1, -1}; if (!future_parse(ff, mod, filename)) { - PyObject_Free(ff); - return NULL; + return 0; } - return ff; + return 1; } diff --git a/Python/symtable.c b/Python/symtable.c index 342f5a080d3ddc..ea195bc155f126 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -2144,14 +2144,13 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, _PyArena_Free(arena); return NULL; } - PyFutureFeatures *future = _PyFuture_FromAST(mod, filename); - if (future == NULL) { + PyFutureFeatures future; + if (!_PyFuture_FromAST(mod, filename, &future)) { _PyArena_Free(arena); return NULL; } - future->ff_features |= flags->cf_flags; - st = _PySymtable_Build(mod, filename, future); - PyObject_Free((void *)future); + future.ff_features |= flags->cf_flags; + st = _PySymtable_Build(mod, filename, &future); _PyArena_Free(arena); return st; } From 0a806f2684779e35b0b8c78b3a9052e08a301d18 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 2 Nov 2022 21:11:11 +0530 Subject: [PATCH 8/8] GH-90699: Remove `_Py_IDENTIFIER` usage from `_curses` module (GH-98957) --- Modules/_cursesmodule.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index c10b2b302c6024..0ddd6dc7646cf4 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -103,7 +103,6 @@ static const char PyCursesVersion[] = "2.2"; #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif -#define NEEDS_PY_IDENTIFIER #define PY_SSIZE_T_CLEAN @@ -2177,12 +2176,11 @@ _curses_window_putwin(PyCursesWindowObject *self, PyObject *file) while (1) { char buf[BUFSIZ]; Py_ssize_t n = fread(buf, 1, BUFSIZ, fp); - _Py_IDENTIFIER(write); if (n <= 0) break; Py_DECREF(res); - res = _PyObject_CallMethodId(file, &PyId_write, "y#", buf, n); + res = PyObject_CallMethod(file, "write", "y#", buf, n); if (res == NULL) break; } @@ -3051,7 +3049,6 @@ _curses_getwin(PyObject *module, PyObject *file) PyObject *data; size_t datalen; WINDOW *win; - _Py_IDENTIFIER(read); PyObject *res = NULL; PyCursesInitialised; @@ -3063,7 +3060,7 @@ _curses_getwin(PyObject *module, PyObject *file) if (_Py_set_inheritable(fileno(fp), 0, NULL) < 0) goto error; - data = _PyObject_CallMethodIdNoArgs(file, &PyId_read); + data = PyObject_CallMethod(file, "read", NULL); if (data == NULL) goto error; if (!PyBytes_Check(data)) { @@ -3962,8 +3959,6 @@ update_lines_cols(void) { PyObject *o; PyObject *m = PyImport_ImportModule("curses"); - _Py_IDENTIFIER(LINES); - _Py_IDENTIFIER(COLS); if (!m) return 0; @@ -3973,13 +3968,12 @@ update_lines_cols(void) Py_DECREF(m); return 0; } - if (_PyObject_SetAttrId(m, &PyId_LINES, o)) { + if (PyObject_SetAttrString(m, "LINES", o)) { Py_DECREF(m); Py_DECREF(o); return 0; } - /* PyId_LINES.object will be initialized here. */ - if (PyDict_SetItem(ModDict, _PyUnicode_FromId(&PyId_LINES), o)) { + if (PyDict_SetItemString(ModDict, "LINES", o)) { Py_DECREF(m); Py_DECREF(o); return 0; @@ -3990,12 +3984,12 @@ update_lines_cols(void) Py_DECREF(m); return 0; } - if (_PyObject_SetAttrId(m, &PyId_COLS, o)) { + if (PyObject_SetAttrString(m, "COLS", o)) { Py_DECREF(m); Py_DECREF(o); return 0; } - if (PyDict_SetItem(ModDict, _PyUnicode_FromId(&PyId_COLS), o)) { + if (PyDict_SetItemString(ModDict, "COLS", o)) { Py_DECREF(m); Py_DECREF(o); return 0;