diff --git a/.azure-pipelines/docs-steps.yml b/.azure-pipelines/docs-steps.yml index 8e72baf2b12fe1..33d379b95aa3dc 100644 --- a/.azure-pipelines/docs-steps.yml +++ b/.azure-pipelines/docs-steps.yml @@ -12,7 +12,7 @@ steps: inputs: versionSpec: '>=3.6' -- script: python -m pip install sphinx==3.2.1 blurb python-docs-theme +- script: python -m pip install sphinx==2.2.0 blurb python-docs-theme displayName: 'Install build dependencies' - ${{ if ne(parameters.latex, 'true') }}: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6f05310e6e5862..48b5825db042f3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -63,6 +63,7 @@ jobs: # Build Python with the libpython dynamic library ./configure --with-pydebug --enable-shared make -j4 regen-all + make regen-stdlib-module-names - name: Check for changes run: | changes=$(git status --porcelain) diff --git a/.travis.yml b/.travis.yml index 547d919974957c..6a22d20455b422 100644 --- a/.travis.yml +++ b/.travis.yml @@ -172,6 +172,7 @@ before_script: - eval "$(pyenv init -)" - pyenv global 3.8 - PYTHON_FOR_REGEN=python3.8 make -j4 regen-all + - make regen-stdlib-module-names - changes=`git status --porcelain` - | # Check for changes in regenerated files diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 5736b83f211fb0..0f759732b35ea8 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1192,7 +1192,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.9 -.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame); +.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) Set the frame evaluation function. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 87425bcf1e71f2..0597ef71366571 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -92,6 +92,38 @@ for the I/O buffer escapes completely the Python memory manager. statistics of the :ref:`pymalloc memory allocator ` every time a new pymalloc object arena is created, and on shutdown. +Allocator Domains +================= + +All allocating functions belong to one of three different "domains" (see also +:c:type`PyMemAllocatorDomain`). These domains represent different allocation +strategies and are optimized for different purposes. The specific details on +how every domain allocates memory or what internal functions each domain calls +is considered an implementation detail, but for debugging purposes a simplified +table can be found at :ref:`here `. There is no hard +requirement to use the memory returned by the allocation functions belonging to +a given domain for only the purposes hinted by that domain (although this is the +recommended practice). For example, one could use the memory returned by +:c:func:`PyMem_RawMalloc` for allocating Python objects or the memory returned +by :c:func:`PyObject_Malloc` for allocating memory for buffers. + +The three allocation domains are: + +* Raw domain: intended for allocating memory for general-purpose memory + buffers where the allocation *must* go to the system allocator or where the + allocator can operate without the :term:`GIL`. The memory is requested directly + to the system. + +* "Mem" domain: intended for allocating memory for Python buffers and + general-purpose memory buffers where the allocation must be performed with + the :term:`GIL` held. The memory is taken from the Python private heap. + +* Object domain: intended for allocating memory belonging to Python objects. The + memory is taken from the Python private heap. + +When freeing memory previously allocated by the allocating functions belonging to a +given domain,the matching specific deallocating functions must be used. For example, +:c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. Raw Memory Interface ==================== @@ -272,6 +304,12 @@ The following function sets, modeled after the ANSI C standard, but specifying behavior when requesting zero bytes, are available for allocating and releasing memory from the Python heap. +.. note:: + There is no guarantee that the memory returned by these allocators can be + succesfully casted to a Python object when intercepting the allocating + functions in this domain by the methods described in + the :ref:`Customize Memory Allocators ` section. + The :ref:`default object allocator ` uses the :ref:`pymalloc memory allocator `. @@ -353,6 +391,7 @@ Legend: * ``pymalloc``: :ref:`pymalloc memory allocator ` * "+ debug": with debug hooks installed by :c:func:`PyMem_SetupDebugHooks` +.. _customize-memory-allocators: Customize Memory Allocators =========================== @@ -601,4 +640,3 @@ heap, objects in Python are allocated and released with :c:func:`PyObject_New`, These will be explained in the next chapter on defining and implementing new object types in C. - diff --git a/Doc/conf.py b/Doc/conf.py index 6b88c23a44473f..cf250981f58752 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -237,3 +237,5 @@ # bpo-40204: Disable warnings on Sphinx 2 syntax of the C domain since the # documentation is built with -W (warnings treated as errors). c_warn_on_allowed_pre_v3 = False + +strip_signature_backslash = True diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 50041829b8c388..3a3653a5ee3a3e 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1206,6 +1206,68 @@ type for ``self``, it's best to create your own converter, subclassing [clinic start generated code]*/ +Using a "defining class" converter +---------------------------------- + +Argument Clinic facilitates gaining access to the defining class of a method. +This is useful for :ref:`heap type ` methods that need to fetch +module level state. Use :c:func:`PyType_FromModuleAndSpec` to associate a new +heap type with a module. You can now use :c:func:`PyType_GetModuleState` on +the defining class to fetch the module state, for example from a module method. + +Example from ``Modules/zlibmodule.c``. First, ``defining_class`` is added to +the clinic input:: + + /*[clinic input] + zlib.Compress.compress + + cls: defining_class + data: Py_buffer + Binary data to be compressed. + / + + +After running the Argument Clinic tool, the following function signature is +generated:: + + /*[clinic start generated code]*/ + static PyObject * + zlib_Compress_compress_impl(compobject *self, PyTypeObject *cls, + Py_buffer *data) + /*[clinic end generated code: output=6731b3f0ff357ca6 input=04d00f65ab01d260]*/ + + +The following code can now use ``PyType_GetModuleState(cls)`` to fetch the +module state:: + + zlibstate *state = PyType_GetModuleState(cls); + + +Each method may only have one argument using this converter, and it must appear +after ``self``, or, if ``self`` is not used, as the first argument. The argument +will be of type ``PyTypeObject *``. The argument will not appear in the +``__text_signature__``. + +The ``defining_class`` converter is not compatible with ``__init__`` and ``__new__`` +methods, which cannot use the ``METH_METHOD`` convention. + +It is not possible to use ``defining_class`` with slot methods. In order to +fetch the module state from such methods, use ``_PyType_GetModuleByDef`` to +look up the module and then :c:func:`PyModule_GetState` to fetch the module +state. Example from the ``setattro`` slot method in +``Modules/_threadmodule.c``:: + + static int + local_setattro(localobject *self, PyObject *name, PyObject *v) + { + PyObject *module = _PyType_GetModuleByDef(Py_TYPE(self), &thread_module); + thread_module_state *state = get_thread_state(module); + ... + } + + +See also :pep:`573`. + Writing a custom converter -------------------------- diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 9b456c14351e44..ad3c7442ad56cd 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -185,7 +185,7 @@ StreamReader can be read. Use the :attr:`IncompleteReadError.partial` attribute to get the partially read data. - .. coroutinemethod:: readuntil(separator=b'\n') + .. coroutinemethod:: readuntil(separator=b'\\n') Read data from the stream until *separator* is found. diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 25b3a4ca2967cb..2f24bb63912fb6 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -199,7 +199,7 @@ The modern interface provides: .. versionadded:: 3.4 -.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v') +.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \\t\\n\\r\\v') Decode the Ascii85 encoded :term:`bytes-like object` or ASCII string *b* and return the decoded :class:`bytes`. diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 26121acaacb7a0..f55bb034b559bf 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -220,11 +220,15 @@ The module :mod:`curses` defines the following functions: multiple devices, and *x*, *y*, *z* are the event's coordinates. (*z* is currently unused.) *bstate* is an integer value whose bits will be set to indicate the type of event, and will be the bitwise OR of one or more of the - following constants, where *n* is the button number from 1 to 4: + following constants, where *n* is the button number from 1 to 5: :const:`BUTTONn_PRESSED`, :const:`BUTTONn_RELEASED`, :const:`BUTTONn_CLICKED`, :const:`BUTTONn_DOUBLE_CLICKED`, :const:`BUTTONn_TRIPLE_CLICKED`, :const:`BUTTON_SHIFT`, :const:`BUTTON_CTRL`, :const:`BUTTON_ALT`. + .. versionchanged:: 3.10 + The ``BUTTON5_*`` constants are now exposed if they are provided by the + underlying curses library. + .. function:: getsyx() diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index a5ee0fb5389793..aa08988c8b36f7 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -149,7 +149,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. contains a good example of its use. -.. function:: context_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n') +.. function:: context_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\\n') Compare *a* and *b* (lists of strings); return a delta (a :term:`generator` generating the delta lines) in context diff format. @@ -279,7 +279,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. emu -.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n') +.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\\n') Compare *a* and *b* (lists of strings); return a delta (a :term:`generator` generating the delta lines) in unified diff format. @@ -321,7 +321,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. See :ref:`difflib-interface` for a more detailed example. -.. function:: diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\n') +.. function:: diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\\n') Compare *a* and *b* (lists of bytes objects) using *dfunc*; yield a sequence of delta lines (also bytes) in the format returned by *dfunc*. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index c09d8338d2c3b3..f3b25383c53d1e 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -742,7 +742,7 @@ iterations of the loop. This opcode performs several operations before a with block starts. First, it loads :meth:`~object.__exit__` from the context manager and pushes it onto - the stack for later use by :opcode:`WITH_CLEANUP_START`. Then, + the stack for later use by :opcode:`WITH_EXCEPT_START`. Then, :meth:`~object.__enter__` is called, and a finally block pointing to *delta* is pushed. Finally, the result of calling the ``__enter__()`` method is pushed onto the stack. The next opcode will either ignore it (:opcode:`POP_TOP`), or diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 42ad0c9f06e237..a77322f83acbde 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -719,51 +719,36 @@ above. An example's doctest directives modify doctest's behavior for that single example. Use ``+`` to enable the named behavior, or ``-`` to disable it. -For example, this test passes: +For example, this test passes:: -.. doctest:: - :no-trim-doctest-flags: - - >>> print(list(range(20))) # doctest: +NORMALIZE_WHITESPACE + >>> print(list(range(20))) # doctest: +NORMALIZE_WHITESPACE [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19] Without the directive it would fail, both because the actual output doesn't have two blanks before the single-digit list elements, and because the actual output is on a single line. This test also passes, and also requires a directive to do -so: - -.. doctest:: - :no-trim-doctest-flags: +so:: - >>> print(list(range(20))) # doctest: +ELLIPSIS + >>> print(list(range(20))) # doctest: +ELLIPSIS [0, 1, ..., 18, 19] Multiple directives can be used on a single physical line, separated by -commas: +commas:: -.. doctest:: - :no-trim-doctest-flags: - - >>> print(list(range(20))) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + >>> print(list(range(20))) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE [0, 1, ..., 18, 19] If multiple directive comments are used for a single example, then they are -combined: - -.. doctest:: - :no-trim-doctest-flags: +combined:: - >>> print(list(range(20))) # doctest: +ELLIPSIS - ... # doctest: +NORMALIZE_WHITESPACE + >>> print(list(range(20))) # doctest: +ELLIPSIS + ... # doctest: +NORMALIZE_WHITESPACE [0, 1, ..., 18, 19] As the previous example shows, you can add ``...`` lines to your example containing only directives. This can be useful when an example is too long for -a directive to comfortably fit on the same line: - -.. doctest:: - :no-trim-doctest-flags: +a directive to comfortably fit on the same line:: >>> print(list(range(5)) + list(range(10, 20)) + list(range(30, 40))) ... # doctest: +ELLIPSIS @@ -808,23 +793,18 @@ instead. Another is to do :: There are others, but you get the idea. -Another bad idea is to print things that embed an object address, like - -.. doctest:: +Another bad idea is to print things that embed an object address, like :: - >>> id(1.0) # certain to fail some of the time # doctest: +SKIP + >>> id(1.0) # certain to fail some of the time 7948648 >>> class C: pass - >>> C() # the default repr() for instances embeds an address # doctest: +SKIP - - -The :const:`ELLIPSIS` directive gives a nice approach for the last example: + >>> C() # the default repr() for instances embeds an address + <__main__.C instance at 0x00AC18F0> -.. doctest:: - :no-trim-doctest-flags: +The :const:`ELLIPSIS` directive gives a nice approach for the last example:: - >>> C() # doctest: +ELLIPSIS - + >>> C() #doctest: +ELLIPSIS + <__main__.C instance at 0x...> Floating-point numbers are also subject to small output variations across platforms, because Python defers to the platform C library for float formatting, diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index e093f138936b36..07152c224f2ff0 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -116,7 +116,7 @@ Here is the :class:`Header` class description: if *s* is a byte string. - .. method:: encode(splitchars=';, \t', maxlinelen=None, linesep='\n') + .. method:: encode(splitchars=';, \\t', maxlinelen=None, linesep='\\n') Encode a message header into an RFC-compliant format, possibly wrapping long lines and encapsulating non-ASCII parts in base64 or quoted-printable diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index a5221250c40486..fa1b42cf484094 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -48,7 +48,7 @@ The simplest possible invocation is:: This invocation will install ``pip`` if it is not already installed, but otherwise does nothing. To ensure the installed version of ``pip`` -is at least as recent as the one bundled with ``ensurepip``, pass the +is at least as recent as the one available in ``ensurepip``, pass the ``--upgrade`` option:: python -m ensurepip --upgrade @@ -86,7 +86,7 @@ Module API .. function:: version() - Returns a string specifying the bundled version of pip that will be + Returns a string specifying the available version of pip that will be installed when bootstrapping an environment. .. function:: bootstrap(root=None, upgrade=False, user=False, \ @@ -100,7 +100,7 @@ Module API for the current environment. *upgrade* indicates whether or not to upgrade an existing installation - of an earlier version of ``pip`` to the bundled version. + of an earlier version of ``pip`` to the available version. *user* indicates whether to use the user scheme rather than installing globally. diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index c532e2caec466c..4d6f2c36336138 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -197,7 +197,7 @@ Having two enum members with the same name is invalid:: ... Traceback (most recent call last): ... - TypeError: Attempted to reuse key: 'SQUARE' + TypeError: 'SQUARE' already defined as: 2 However, two enum members are allowed to have the same value. Given two members A and B with the same value (and A defined first), B is an alias to A. By-value @@ -422,7 +422,7 @@ any members. So this is forbidden:: ... Traceback (most recent call last): ... - TypeError: Cannot extend enumerations + TypeError: MoreColor: cannot extend enumeration 'Color' But this is allowed:: @@ -617,6 +617,7 @@ by extension, string enumerations of different types can also be compared to each other. :class:`StrEnum` exists to help avoid the problem of getting an incorrect member:: + >>> from enum import StrEnum >>> class Directions(StrEnum): ... NORTH = 'north', # notice the trailing comma ... SOUTH = 'south' @@ -638,12 +639,22 @@ IntFlag The next variation of :class:`Enum` provided, :class:`IntFlag`, is also based on :class:`int`. The difference being :class:`IntFlag` members can be combined using the bitwise operators (&, \|, ^, ~) and the result is still an -:class:`IntFlag` member. However, as the name implies, :class:`IntFlag` +:class:`IntFlag` member, if possible. However, as the name implies, :class:`IntFlag` members also subclass :class:`int` and can be used wherever an :class:`int` is -used. Any operation on an :class:`IntFlag` member besides the bit-wise -operations will lose the :class:`IntFlag` membership. +used. + +.. note:: + + Any operation on an :class:`IntFlag` member besides the bit-wise operations will + lose the :class:`IntFlag` membership. + +.. note:: + + Bit-wise operations that result in invalid :class:`IntFlag` values will lose the + :class:`IntFlag` membership. .. versionadded:: 3.6 +.. versionchanged:: 3.10 Sample :class:`IntFlag` class:: @@ -671,21 +682,41 @@ It is also possible to name the combinations:: >>> Perm.RWX >>> ~Perm.RWX - + + >>> Perm(7) + + +.. note:: + + Named combinations are considered aliases. Aliases do not show up during + iteration, but can be returned from by-value lookups. + +.. versionchanged:: 3.10 Another important difference between :class:`IntFlag` and :class:`Enum` is that if no flags are set (the value is 0), its boolean evaluation is :data:`False`:: >>> Perm.R & Perm.X - + >>> bool(Perm.R & Perm.X) False Because :class:`IntFlag` members are also subclasses of :class:`int` they can -be combined with them:: +be combined with them (but may lose :class:`IntFlag` membership:: + + >>> Perm.X | 4 + >>> Perm.X | 8 - + 9 + +.. note:: + + The negation operator, ``~``, always returns an :class:`IntFlag` member with a + positive value:: + + >>> (~Perm.X).value == (Perm.R|Perm.W).value == 6 + True :class:`IntFlag` members can also be iterated over:: @@ -717,7 +748,7 @@ flags being set, the boolean evaluation is :data:`False`:: ... GREEN = auto() ... >>> Color.RED & Color.GREEN - + >>> bool(Color.RED & Color.GREEN) False @@ -751,7 +782,7 @@ value:: >>> purple = Color.RED | Color.BLUE >>> list(purple) - [, ] + [, ] .. versionadded:: 3.10 @@ -953,7 +984,7 @@ to handle any extra arguments:: ... BLEACHED_CORAL = () # New color, no Pantone code yet! ... >>> Swatch.SEA_GREEN - + >>> Swatch.SEA_GREEN.pantone '1246' >>> Swatch.BLEACHED_CORAL.pantone @@ -1144,6 +1175,14 @@ Supported ``_sunder_`` names :class:`auto` to get an appropriate value for an enum member; may be overridden +.. note:: + + For standard :class:`Enum` classes the next value chosen is the last value seen + incremented by one. + + For :class:`Flag`-type classes the next value chosen will be the next highest + power-of-two, regardless of the last value seen. + .. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` .. versionadded:: 3.7 ``_ignore_`` @@ -1159,7 +1198,9 @@ and raise an error if the two do not match:: ... Traceback (most recent call last): ... - TypeError: member order does not match _order_ + TypeError: member order does not match _order_: + ['RED', 'BLUE', 'GREEN'] + ['RED', 'GREEN', 'BLUE'] .. note:: @@ -1179,11 +1220,9 @@ Private names are not converted to Enum members, but remain normal attributes. """""""""""""""""""" :class:`Enum` members are instances of their :class:`Enum` class, and are -normally accessed as ``EnumClass.member``. Under certain circumstances they -can also be accessed as ``EnumClass.member.member``, but you should never do -this as that lookup may fail or, worse, return something besides the -:class:`Enum` member you are looking for (this is another good reason to use -all-uppercase names for members):: +normally accessed as ``EnumClass.member``. In Python versions ``3.5`` to +``3.9`` you could access members from other members -- this practice was +discouraged, and in ``3.10`` :class:`Enum` has returned to not allowing it:: >>> class FieldTypes(Enum): ... name = 0 @@ -1191,11 +1230,12 @@ all-uppercase names for members):: ... size = 2 ... >>> FieldTypes.value.size - - >>> FieldTypes.size.value - 2 + Traceback (most recent call last): + ... + AttributeError: FieldTypes: no attribute 'size' .. versionchanged:: 3.5 +.. versionchanged:: 3.10 Creating members that are mixed with other data types @@ -1237,14 +1277,14 @@ but not of the class:: >>> dir(Planet) ['EARTH', 'JUPITER', 'MARS', 'MERCURY', 'NEPTUNE', 'SATURN', 'URANUS', 'VENUS', '__class__', '__doc__', '__members__', '__module__'] >>> dir(Planet.EARTH) - ['__class__', '__doc__', '__module__', 'name', 'surface_gravity', 'value'] + ['__class__', '__doc__', '__module__', 'mass', 'name', 'radius', 'surface_gravity', 'value'] Combining members of ``Flag`` """"""""""""""""""""""""""""" -If a combination of Flag members is not named, the :func:`repr` will include -all named flags and all named combinations of flags that are in the value:: +Iterating over a combination of Flag members will only return the members that +are comprised of a single bit:: >>> class Color(Flag): ... RED = auto() @@ -1254,10 +1294,10 @@ all named flags and all named combinations of flags that are in the value:: ... YELLOW = RED | GREEN ... CYAN = GREEN | BLUE ... - >>> Color(3) # named combination + >>> Color(3) - >>> Color(7) # not named combination - + >>> Color(7) + ``StrEnum`` and :meth:`str.__str__` """"""""""""""""""""""""""""""""""" @@ -1269,3 +1309,71 @@ parts of Python will read the string data directly, while others will call :meth:`StrEnum.__str__` will be the same as :meth:`str.__str__` so that ``str(StrEnum.member) == StrEnum.member`` is true. +``Flag`` and ``IntFlag`` minutia +"""""""""""""""""""""""""""""""" + +The code sample:: + + >>> class Color(IntFlag): + ... BLACK = 0 + ... RED = 1 + ... GREEN = 2 + ... BLUE = 4 + ... PURPLE = RED | BLUE + ... WHITE = RED | GREEN | BLUE + ... + +- single-bit flags are canonical +- multi-bit and zero-bit flags are aliases +- only canonical flags are returned during iteration:: + + >>> list(Color.WHITE) + [, , ] + +- negating a flag or flag set returns a new flag/flag set with the + corresponding positive integer value:: + + >>> Color.GREEN + + + >>> ~Color.GREEN + + +- names of pseudo-flags are constructed from their members' names:: + + >>> (Color.RED | Color.GREEN).name + 'RED|GREEN' + +- multi-bit flags, aka aliases, can be returned from operations:: + + >>> Color.RED | Color.BLUE + + + >>> Color(7) # or Color(-1) + + +- membership / containment checking has changed slightly -- zero valued flags + are never considered to be contained:: + + >>> Color.BLACK in Color.WHITE + False + + otherwise, if all bits of one flag are in the other flag, True is returned:: + + >>> Color.PURPLE in Color.WHITE + True + +There is a new boundary mechanism that controls how out-of-range / invalid +bits are handled: ``STRICT``, ``CONFORM``, ``EJECT``, and ``KEEP``: + + * STRICT --> raises an exception when presented with invalid values + * CONFORM --> discards any invalid bits + * EJECT --> lose Flag status and become a normal int with the given value + * KEEP --> keep the extra bits + - keeps Flag status and extra bits + - extra bits do not show up in iteration + - extra bits do show up in repr() and str() + +The default for Flag is ``STRICT``, the default for ``IntFlag`` is ``DISCARD``, +and the default for ``_convert_`` is ``KEEP`` (see ``ssl.Options`` for an +example of when ``KEEP`` is needed). diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 0598a6ce9415e2..f84353ce391d14 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1334,7 +1334,7 @@ are always available. They are listed here in alphabetical order. supported. -.. function:: print(*objects, sep=' ', end='\n', file=sys.stdout, flush=False) +.. function:: print(*objects, sep=' ', end='\\n', file=sys.stdout, flush=False) Print *objects* to the text stream *file*, separated by *sep* and followed by *end*. *sep*, *end*, *file* and *flush*, if present, must be given as keyword @@ -1703,18 +1703,19 @@ are always available. They are listed here in alphabetical order. With three arguments, return a new type object. This is essentially a - dynamic form of the :keyword:`class` statement. The *name* string is the - class name and becomes the :attr:`~definition.__name__` attribute; the *bases* - tuple itemizes the base classes and becomes the :attr:`~class.__bases__` - attribute; and the *dict* dictionary is the namespace containing definitions - for class body and is copied to a standard dictionary to become the - :attr:`~object.__dict__` attribute. For example, the following two - statements create identical :class:`type` objects: + dynamic form of the :keyword:`class` statement. The *name* string is + the class name and becomes the :attr:`~definition.__name__` attribute. + The *bases* tuple contains the base classes and becomes the + :attr:`~class.__bases__` attribute; if empty, :class:`object`, the + ultimate base of all classes, is added. The *dict* dictionary contains + attribute and method definitions for the class body; it may be copied + or wrapped before becoming the :attr:`~object.__dict__` attribute. + The following two statements create identical :class:`type` objects: >>> class X: ... a = 1 ... - >>> X = type('X', (object,), dict(a=1)) + >>> X = type('X', (), dict(a=1)) See also :ref:`bltin-type-objects`. diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index d644974e660984..7bb030bd08f9d7 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -746,5 +746,5 @@ Domain Dedication 1.0 Universal: Wikipedia article with information on which algorithms have known issues and what that means regarding their use. - https://www.ietf.org/rfc/rfc2898.txt - PKCS #5: Password-Based Cryptography Specification Version 2.0 + https://www.ietf.org/rfc/rfc8018.txt + PKCS #5: Password-Based Cryptography Specification Version 2.1 diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index a2c1eb00d8b33d..17792b200599bd 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -93,7 +93,7 @@ Cookie Objects :meth:`value_decode` are inverses on the range of *value_decode*. -.. method:: BaseCookie.output(attrs=None, header='Set-Cookie:', sep='\r\n') +.. method:: BaseCookie.output(attrs=None, header='Set-Cookie:', sep='\\r\\n') Return a string representation suitable to be sent as HTTP headers. *attrs* and *header* are sent to each :class:`Morsel`'s :meth:`output` method. *sep* is used diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index a59a5d3a465703..e7eaabd8bfa25a 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -250,7 +250,7 @@ View Last Restart Scroll the shell window to the last Shell restart. Restart Shell - Restart the shell to clean the environment. + Restart the shell to clean the environment and reset display and exception handling. Previous History Cycle through earlier commands in history which match the current entry. diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 048cb2a7ff6924..aecbec56866d73 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -964,7 +964,7 @@ Text I/O .. versionadded:: 3.7 -.. class:: StringIO(initial_value='', newline='\n') +.. class:: StringIO(initial_value='', newline='\\n') A text stream using an in-memory text buffer. It inherits :class:`TextIOBase`. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 2bbf3aad619884..ac96de334b3290 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1195,14 +1195,15 @@ Below is a table mapping various :mod:`os` functions to their corresponding .. note:: - Although :func:`os.path.relpath` and :meth:`PurePath.relative_to` have some - overlapping use-cases, their semantics differ enough to warrant not - considering them equivalent. + Not all pairs of functions/methods below are equivalent. Some of them, + despite having some overlapping use-cases, have different semantics. They + include :func:`os.path.abspath` and :meth:`Path.resolve`, + :func:`os.path.relpath` and :meth:`PurePath.relative_to`. ==================================== ============================== -os and os.path pathlib +:mod:`os` and :mod:`os.path` :mod:`pathlib` ==================================== ============================== -:func:`os.path.abspath` :meth:`Path.resolve` +:func:`os.path.abspath` :meth:`Path.resolve` [#]_ :func:`os.chmod` :meth:`Path.chmod` :func:`os.mkdir` :meth:`Path.mkdir` :func:`os.makedirs` :meth:`Path.mkdir` @@ -1221,6 +1222,7 @@ os and os.path pathlib :func:`os.link` :meth:`Path.link_to` :func:`os.symlink` :meth:`Path.symlink_to` :func:`os.readlink` :meth:`Path.readlink` +:func:`os.path.relpath` :meth:`Path.relative_to` [#]_ :func:`os.stat` :meth:`Path.stat`, :meth:`Path.owner`, :meth:`Path.group` @@ -1231,3 +1233,8 @@ os and os.path pathlib :func:`os.path.samefile` :meth:`Path.samefile` :func:`os.path.splitext` :data:`PurePath.suffix` ==================================== ============================== + +.. rubric:: Footnotes + +.. [#] :func:`os.path.abspath` does not resolve symbolic links while :meth:`Path.resolve` does. +.. [#] :meth:`Path.relative_to` requires ``self`` to be the subpath of the argument, but :func:`os.path.relpath` does not. diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 07ee0115214296..9d85c2b9958eb2 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -141,8 +141,9 @@ Functions for integers ``randrange(10)``. In the future, this will raise a :exc:`TypeError`. .. deprecated:: 3.10 - The exception raised for non-integral values such as ``range(10.5)`` - will be changed from :exc:`ValueError` to :exc:`TypeError`. + The exception raised for non-integral values such as ``randrange(10.5)`` + or ``randrange('10')`` will be changed from :exc:`ValueError` to + :exc:`TypeError`. .. function:: randint(a, b) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 54786d0c2ab0df..1bfd518349b389 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -347,8 +347,8 @@ The meaning of the various alignment options is as follows: | ``'='`` | Forces the padding to be placed after the sign (if any) | | | but before the digits. This is used for printing fields | | | in the form '+000000120'. This alignment option is only | - | | valid for numeric types. It becomes the default when '0'| - | | immediately precedes the field width. | + | | valid for numeric types. It becomes the default for | + | | numbers when '0' immediately precedes the field width. | +---------+----------------------------------------------------------+ | ``'^'`` | Forces the field to be centered within the available | | | space. | @@ -424,6 +424,10 @@ When no explicit alignment is given, preceding the *width* field by a zero sign-aware zero-padding for numeric types. This is equivalent to a *fill* character of ``'0'`` with an *alignment* type of ``'='``. +.. versionchanged:: 3.10 + Preceding the *width* field by ``'0'`` no longer affects the default + alignment for strings. + The *precision* is a decimal number indicating how many digits should be displayed after the decimal point for a floating point value formatted with ``'f'`` and ``'F'``, or before and after the decimal point for a floating point diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 0f13adcf0e5b2f..80b30d01f91aac 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -153,10 +153,12 @@ always available. .. data:: builtin_module_names - A tuple of strings giving the names of all modules that are compiled into this + A tuple of strings containing the names of all modules that are compiled into this Python interpreter. (This information is not available in any other way --- ``modules.keys()`` only lists the imported modules.) + See also the :attr:`sys.stdlib_module_names` list. + .. function:: call_tracing(func, args) @@ -1564,6 +1566,25 @@ always available. to a console and Python apps started with :program:`pythonw`. +.. data:: stdlib_module_names + + A frozenset of strings containing the names of standard library modules. + + It is the same on all platforms. Modules which are not available on + some platforms and modules disabled at Python build are also listed. + All module kinds are listed: pure Python, built-in, frozen and extension + modules. Test modules are excluded. + + For packages, only the main package is listed: sub-packages and sub-modules + are not listed. For example, the ``email`` package is listed, but the + ``email.mime`` sub-package and the ``email.message`` sub-module are not + listed. + + See also the :attr:`sys.builtin_module_names` list. + + .. versionadded:: 3.10 + + .. data:: thread_info A :term:`named tuple` holding information about the thread diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index c233f18d30a29f..e938dd58b05312 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -212,11 +212,16 @@ The module also defines the following classes: :class:`TracebackException` objects are created from actual exceptions to capture data for later printing in a lightweight fashion. -.. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False) +.. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False) Capture an exception for later rendering. *limit*, *lookup_lines* and *capture_locals* are as for the :class:`StackSummary` class. + If *compact* is true, only data that is required by :class:`TracebackException`'s + ``format`` method is saved in the class attributes. In particular, the + ``__context__`` field is calculated only if ``__cause__`` is ``None`` and + ``__suppress_context__`` is false. + Note that when locals are captured, they are also shown in the traceback. .. attribute:: __cause__ @@ -294,6 +299,9 @@ capture data for later printing in a lightweight fashion. The message indicating which exception occurred is always the last string in the output. + .. versionchanged:: 3.10 + Added the *compact* parameter. + :class:`StackSummary` Objects ----------------------------- diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 0a0993518efddc..bb229f0bf9ce0e 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1498,11 +1498,11 @@ Test cases after :meth:`setUpClass` if :meth:`setUpClass` raises an exception. It is responsible for calling all the cleanup functions added by - :meth:`addCleanupClass`. If you need cleanup functions to be called + :meth:`addClassCleanup`. If you need cleanup functions to be called *prior* to :meth:`tearDownClass` then you can call - :meth:`doCleanupsClass` yourself. + :meth:`doClassCleanups` yourself. - :meth:`doCleanupsClass` pops methods off the stack of cleanup + :meth:`doClassCleanups` pops methods off the stack of cleanup functions one at a time, so it can be called at any time. .. versionadded:: 3.8 diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index e1cc96794221ad..bf72c46561b7c7 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -174,7 +174,7 @@ module documentation. This section lists the differences between the API and The :meth:`toxml` method now preserves the attribute order specified by the user. -.. method:: Node.toprettyxml(indent="\t", newl="\n", encoding=None, \ +.. method:: Node.toprettyxml(indent="\\t", newl="\\n", encoding=None, \ standalone=None) Return a pretty-printed version of the document. *indent* specifies the diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 5bba3eea6f6c04..f22af8b44a1127 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -411,7 +411,8 @@ This allows common :keyword:`try`...\ :keyword:`except`...\ :keyword:`finally` usage patterns to be encapsulated for convenient reuse. .. productionlist:: python-grammar - with_stmt: "with" `with_item` ("," `with_item`)* ":" `suite` + with_stmt: "with" ( "(" `with_stmt_contents` ","? ")" | `with_stmt_contents` ) ":" `suite` + with_stmt_contents: `with_item` ("," `with_item`)* with_item: `expression` ["as" `target`] The execution of the :keyword:`with` statement with one "item" proceeds as follows: @@ -488,9 +489,21 @@ is semantically equivalent to:: with B() as b: SUITE +You can also write multi-item context managers in multiple lines if +the items are surrounded by parentheses. For example:: + + with ( + A() as a, + B() as b, + ): + SUITE + .. versionchanged:: 3.1 Support for multiple context expressions. +.. versionchanged:: 3.10 + Support for using grouping parentheses to break the statement in multiple lines. + .. seealso:: :pep:`343` - The "with" statement diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 7a51a9dbfb83ad..16bb8fb28178ab 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -72,6 +72,59 @@ New Features .. _whatsnew310-pep563: +Parenthesized context managers +------------------------------ + +Using enclosing parentheses for continuation across multiple lines +in context managers is now supported. This allows formatting a long +collection of context managers in multiple lines in a similar way +as it was previously possible with import statements. For instance, +all these examples are now valid: + +.. code-block:: python + + with (CtxManager() as example): + ... + + with ( + CtxManager1(), + CtxManager2() + ): + ... + + with (CtxManager1() as example, + CtxManager2()): + ... + + with (CtxManager1(), + CtxManager2() as example): + ... + + with ( + CtxManager1() as example1, + CtxManager2() as example2 + ): + ... + +it is also possible to use a trailing comma at the end of the +enclosed group: + +.. code-block:: python + + with ( + CtxManager1() as example1, + CtxManager2() as example2, + CtxManager3() as example3, + ): + ... + +This new syntax uses the non LL(1) capacities of the new parser. +Check :pep:`617` for more details. + +(Contributed by Guido van Rossum, Pablo Galindo and Lysandros Nikolaou +in :issue:`12782` and :issue:`40334`.) + + PEP 563: Postponed Evaluation of Annotations Becomes Default ------------------------------------------------------------ @@ -166,6 +219,48 @@ See :class:`typing.Callable`, :class:`typing.ParamSpec`, (Contributed by Ken Jin in :issue:`41559`.) +Better error messages in the parser +----------------------------------- + +When parsing code that contains unclosed parentheses or brackets the interpreter +now includes the location of the unclosed bracket of parentheses instead of displaying +*SyntaxError: unexpected EOF while parsing* or pointing to some incorrect location. +For instance, consider the following code (notice the unclosed '{'): + +.. code-block:: python + + expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, + 38: 4, 39: 4, 45: 5, 46: 5, 47: 5, 48: 5, 49: 5, 54: 6, + some_other_code = foo() + +previous versions of the interpreter reported confusing places as the location of +the syntax error: + +.. code-block:: text + + File "example.py", line 3 + some_other_code = foo() + ^ + SyntaxError: invalid syntax + +but in Python3.10 a more informative error is emitted: + +.. code-block:: text + + File "example.py", line 1 + expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, + ^ + SyntaxError: '{' was never closed + + +In a similar way, errors involving unclosed string literals (single and triple +quoted) now point to the start of the string instead of reporting EOF/EOL. + +These improvements are inspired by previous work in the PyPy interpreter. + +(Contributed by Pablo Galindo in :issue:`42864` and Batuhan Taskaya in +:issue:`40176`.) + Other Language Changes ====================== @@ -242,11 +337,15 @@ by :func:`curses.color_content`, :func:`curses.init_color`, support is provided by the underlying ncurses library. (Contributed by Jeffrey Kintscher and Hans Petter Jansson in :issue:`36982`.) +The ``BUTTON5_*`` constants are now exposed in the :mod:`curses` module if +they are provided by the underlying curses library. +(Contributed by Zackery Spytz in :issue:`39273`.) + distutils --------- The ``bdist_wininst`` command deprecated in Python 3.8 has been removed. -The ``bidst_wheel`` command is now recommended to distribute binary packages +The ``bdist_wheel`` command is now recommended to distribute binary packages on Windows. (Contributed by Victor Stinner in :issue:`42802`.) @@ -350,6 +449,10 @@ Add :data:`sys.orig_argv` attribute: the list of the original command line arguments passed to the Python executable. (Contributed by Victor Stinner in :issue:`23427`.) +Add :data:`sys.stdlib_module_names`, containing the list of the standard library +module names. +(Contributed by Victor Stinner in :issue:`42955`.) + threading --------- @@ -480,6 +583,10 @@ Deprecated as appropriate to help identify code which needs updating during this transition. +* Non-integer arguments to :func:`random.randrange` are deprecated. + The :exc:`ValueError` is deprecated in favor of a :exc:`TypeError`. + (Contributed by Serhiy Storchaka and Raymond Hettinger in :issue:`37319`.) + * The various ``load_module()`` methods of :mod:`importlib` have been documented as deprecated since Python 3.6, but will now also trigger a :exc:`DeprecationWarning`. Use @@ -554,6 +661,23 @@ Removed the :mod:`collections` module. (Contributed by Victor Stinner in :issue:`37324`.) +* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's + :doc:`high-level API <../library/asyncio-api-index>` following deprecation + in Python 3.8. The motivation behind this change is multifold: + + 1. This simplifies the high-level API. + 2. The functions in the high-level API have been implicitly getting the + current thread's running event loop since Python 3.7. There isn't a need to + pass the event loop to the API in most normal use cases. + 3. Event loop passing is error-prone especially when dealing with loops + running in different threads. + + Note that the low-level API will still accept ``loop``. + See `Changes in the Python API`_ for examples of how to replace existing code. + + (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley + in :issue:`42392`.) + Porting to Python 3.10 ====================== @@ -592,6 +716,26 @@ Changes in the Python API a 16-bit unsigned integer. (Contributed by Erlend E. Aasland in :issue:`42393`.) +* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's + :doc:`high-level API <../library/asyncio-api-index>` following deprecation + in Python 3.8. + + A coroutine that currently look like this:: + + async def foo(loop): + await asyncio.sleep(1, loop=loop) + + Should be replaced with this:: + + async def foo(): + await asyncio.sleep(1) + + If ``foo()`` was specifically designed *not* to run in the current thread's + running event loop (e.g. running in another thread's event loop), consider + using :func:`asyncio.run_coroutine_threadsafe` instead. + + (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley + in :issue:`42392`.) CPython bytecode changes ======================== @@ -617,6 +761,18 @@ Build Changes don't build nor install test modules. (Contributed by Xavier de Gaye, Thomas Petazzoni and Peixing Xin in :issue:`27640`.) +* Add ``--with-wheel-pkg-dir=PATH`` option to the ``./configure`` script. If + specified, the :mod:`ensurepip` module looks for ``setuptools`` and ``pip`` + wheel packages in this directory: if both are present, these wheel packages + are used instead of ensurepip bundled wheel packages. + + Some Linux distribution packaging policies recommend against bundling + dependencies. For example, Fedora installs wheel packages in the + ``/usr/share/python-wheels/`` directory and don't install the + ``ensurepip._bundled`` package. + + (Contributed by Victor Stinner in :issue:`42856`.) + C API Changes ============= diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index 1083942c14929c..7a4219c8b338b4 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -376,4 +376,4 @@ PyAPI_FUNC(void) _Py_add_one_to_index_C(int nd, Py_ssize_t *index, PyAPI_FUNC(int) _Py_convert_optional_to_ssize_t(PyObject *, void *); /* Same as PyNumber_Index but can return an instance of a subclass of int. */ -PyAPI_FUNC(PyObject *) _PyNumber_Index(PyObject *o); \ No newline at end of file +PyAPI_FUNC(PyObject *) _PyNumber_Index(PyObject *o); diff --git a/Include/errcode.h b/Include/errcode.h index 790518b8b7730e..f2671d6c9b30b4 100644 --- a/Include/errcode.h +++ b/Include/errcode.h @@ -26,8 +26,6 @@ extern "C" { #define E_TOODEEP 20 /* Too many indentation levels */ #define E_DEDENT 21 /* No matching outer block for dedent */ #define E_DECODE 22 /* Error in decoding into Unicode */ -#define E_EOFS 23 /* EOF in triple-quoted string */ -#define E_EOLS 24 /* EOL in single-quoted string */ #define E_LINECONT 25 /* Unexpected characters after a line continuation */ #define E_BADSINGLE 27 /* Ill-formed single statement input */ diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 3975765a46cc4d..3cd27b035c2c78 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -168,6 +168,12 @@ _PyObject_IS_GC(PyObject *obj) // Fast inlined version of PyType_IS_GC() #define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) +// Usage: assert(_Py_CheckSlotResult(obj, "__getitem__", result != NULL))); +extern int _Py_CheckSlotResult( + PyObject *obj, + const char *slot_name, + int success); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 2cf1160afc0149..9dd66aec9c3d78 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -84,6 +84,8 @@ PyAPI_FUNC(PyObject *) _PyErr_FormatFromCauseTstate( PyAPI_FUNC(int) _PyErr_CheckSignalsTstate(PyThreadState *tstate); +PyAPI_FUNC(void) _Py_DumpExtensionModules(int fd, PyInterpreterState *interp); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index 1f092411a72ba5..4d282308769dc8 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -74,17 +74,13 @@ PyAPI_FUNC(void) _Py_DumpASCII(int fd, PyObject *text); This function is signal safe. */ PyAPI_FUNC(void) _Py_DumpDecimal( int fd, - unsigned long value); + size_t value); -/* Format an integer as hexadecimal into the file descriptor fd with at least - width digits. - - The maximum width is sizeof(unsigned long)*2 digits. - - This function is signal safe. */ +/* Format an integer as hexadecimal with width digits into fd file descriptor. + The function is signal safe. */ PyAPI_FUNC(void) _Py_DumpHexadecimal( int fd, - unsigned long value, + uintptr_t value, Py_ssize_t width); PyAPI_FUNC(PyObject*) _PyTraceBack_FromFrame( diff --git a/Include/moduleobject.h b/Include/moduleobject.h index cf9ad40c0a17a0..49b116ca1c3587 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -84,6 +84,12 @@ typedef struct PyModuleDef{ freefunc m_free; } PyModuleDef; + +// Internal C API +#ifdef Py_BUILD_CORE +extern int _PyModule_IsExtension(PyObject *obj); +#endif + #ifdef __cplusplus } #endif diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 59b4699feb5062..22a7d0aade855f 100755 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -175,7 +175,12 @@ def main(): '__package__': None, '__cached__': None, } - runctx(code, globs, None, options.outfile, options.sort) + try: + runctx(code, globs, None, options.outfile, options.sort) + except BrokenPipeError as exc: + # Prevent "Exception ignored" during interpreter shutdown. + sys.stdout = None + sys.exit(exc.errno) else: parser.print_usage() return parser diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/ctypes/test/test_parameters.py index e4c25fd880cefb..38af7ac13d756c 100644 --- a/Lib/ctypes/test/test_parameters.py +++ b/Lib/ctypes/test/test_parameters.py @@ -201,6 +201,49 @@ def __dict__(self): with self.assertRaises(ZeroDivisionError): WorseStruct().__setstate__({}, b'foo') + def test_parameter_repr(self): + from ctypes import ( + c_bool, + c_char, + c_wchar, + c_byte, + c_ubyte, + c_short, + c_ushort, + c_int, + c_uint, + c_long, + c_ulong, + c_longlong, + c_ulonglong, + c_float, + c_double, + c_longdouble, + c_char_p, + c_wchar_p, + c_void_p, + ) + self.assertRegex(repr(c_bool.from_param(True)), r"^$") + self.assertEqual(repr(c_char.from_param(97)), "") + self.assertRegex(repr(c_wchar.from_param('a')), r"^$") + self.assertEqual(repr(c_byte.from_param(98)), "") + self.assertEqual(repr(c_ubyte.from_param(98)), "") + self.assertEqual(repr(c_short.from_param(511)), "") + self.assertEqual(repr(c_ushort.from_param(511)), "") + self.assertRegex(repr(c_int.from_param(20000)), r"^$") + self.assertRegex(repr(c_uint.from_param(20000)), r"^$") + self.assertRegex(repr(c_long.from_param(20000)), r"^$") + self.assertRegex(repr(c_ulong.from_param(20000)), r"^$") + self.assertRegex(repr(c_longlong.from_param(20000)), r"^$") + self.assertRegex(repr(c_ulonglong.from_param(20000)), r"^$") + self.assertEqual(repr(c_float.from_param(1.5)), "") + self.assertEqual(repr(c_double.from_param(1.5)), "") + self.assertEqual(repr(c_double.from_param(1e300)), "") + self.assertRegex(repr(c_longdouble.from_param(1.5)), r"^$") + self.assertRegex(repr(c_char_p.from_param(b'hihi')), r"^$") + self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^$") + self.assertRegex(repr(c_void_p.from_param(0x12)), r"^$") + ################################################################ if __name__ == '__main__': diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index cb2882e3360fcf..2276fd7fd8f3b2 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -1,27 +1,82 @@ +import collections import os import os.path +import subprocess import sys -import runpy +import sysconfig import tempfile -import subprocess from importlib import resources -from . import _bundled - __all__ = ["version", "bootstrap"] - - +_PACKAGE_NAMES = ('setuptools', 'pip') _SETUPTOOLS_VERSION = "47.1.0" - _PIP_VERSION = "20.2.3" - _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION, "py3"), ("pip", _PIP_VERSION, "py2.py3"), ] +# Packages bundled in ensurepip._bundled have wheel_name set. +# Packages from WHEEL_PKG_DIR have wheel_path set. +_Package = collections.namedtuple('Package', + ('version', 'wheel_name', 'wheel_path')) + +# Directory of system wheel packages. Some Linux distribution packaging +# policies recommend against bundling dependencies. For example, Fedora +# installs wheel packages in the /usr/share/python-wheels/ directory and don't +# install the ensurepip._bundled package. +_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR') + + +def _find_packages(path): + packages = {} + try: + filenames = os.listdir(path) + except OSError: + # Ignore: path doesn't exist or permission error + filenames = () + # Make the code deterministic if a directory contains multiple wheel files + # of the same package, but don't attempt to implement correct version + # comparison since this case should not happen. + filenames = sorted(filenames) + for filename in filenames: + # filename is like 'pip-20.2.3-py2.py3-none-any.whl' + if not filename.endswith(".whl"): + continue + for name in _PACKAGE_NAMES: + prefix = name + '-' + if filename.startswith(prefix): + break + else: + continue + + # Extract '20.2.2' from 'pip-20.2.2-py2.py3-none-any.whl' + version = filename.removeprefix(prefix).partition('-')[0] + wheel_path = os.path.join(path, filename) + packages[name] = _Package(version, None, wheel_path) + return packages + + +def _get_packages(): + global _PACKAGES, _WHEEL_PKG_DIR + if _PACKAGES is not None: + return _PACKAGES + + packages = {} + for name, version, py_tag in _PROJECTS: + wheel_name = f"{name}-{version}-{py_tag}-none-any.whl" + packages[name] = _Package(version, wheel_name, None) + if _WHEEL_PKG_DIR: + dir_packages = _find_packages(_WHEEL_PKG_DIR) + # only used the wheel package directory if all packages are found there + if all(name in dir_packages for name in _PACKAGE_NAMES): + packages = dir_packages + _PACKAGES = packages + return packages +_PACKAGES = None + def _run_pip(args, additional_paths=None): # Run the bootstraping in a subprocess to avoid leaking any state that happens @@ -42,7 +97,8 @@ def version(): """ Returns a string specifying the bundled version of pip. """ - return _PIP_VERSION + return _get_packages()['pip'].version + def _disable_pip_configuration_settings(): # We deliberately ignore all pip environment variables @@ -104,16 +160,23 @@ def _bootstrap(*, root=None, upgrade=False, user=False, # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path additional_paths = [] - for project, version, py_tag in _PROJECTS: - wheel_name = "{}-{}-{}-none-any.whl".format(project, version, py_tag) - whl = resources.read_binary( - _bundled, - wheel_name, - ) - with open(os.path.join(tmpdir, wheel_name), "wb") as fp: + for name, package in _get_packages().items(): + if package.wheel_name: + # Use bundled wheel package + from ensurepip import _bundled + wheel_name = package.wheel_name + whl = resources.read_binary(_bundled, wheel_name) + else: + # Use the wheel package directory + with open(package.wheel_path, "rb") as fp: + whl = fp.read() + wheel_name = os.path.basename(package.wheel_path) + + filename = os.path.join(tmpdir, wheel_name) + with open(filename, "wb") as fp: fp.write(whl) - additional_paths.append(os.path.join(tmpdir, wheel_name)) + additional_paths.append(filename) # Construct the arguments to be passed to the pip command args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] @@ -126,7 +189,7 @@ def _bootstrap(*, root=None, upgrade=False, user=False, if verbosity: args += ["-" + "v" * verbosity] - return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) + return _run_pip([*args, *_PACKAGE_NAMES], additional_paths) def _uninstall_helper(*, verbosity=0): """Helper to support a clean default uninstall process on Windows @@ -139,11 +202,14 @@ def _uninstall_helper(*, verbosity=0): except ImportError: return - # If the pip version doesn't match the bundled one, leave it alone - if pip.__version__ != _PIP_VERSION: - msg = ("ensurepip will only uninstall a matching version " - "({!r} installed, {!r} bundled)") - print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) + # If the installed pip version doesn't match the available one, + # leave it alone + available_version = version() + if pip.__version__ != available_version: + print(f"ensurepip will only uninstall a matching version " + f"({pip.__version__!r} installed, " + f"{available_version!r} available)", + file=sys.stderr) return _disable_pip_configuration_settings() @@ -153,7 +219,7 @@ def _uninstall_helper(*, verbosity=0): if verbosity: args += ["-" + "v" * verbosity] - return _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) + return _run_pip([*args, *reversed(_PACKAGE_NAMES)]) def _main(argv=None): diff --git a/Lib/enum.py b/Lib/enum.py index 8ca385420da029..d4b11521ab27f3 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1,6 +1,6 @@ import sys from types import MappingProxyType, DynamicClassAttribute -from builtins import property as _bltin_property +from builtins import property as _bltin_property, bin as _bltin_bin __all__ = [ @@ -8,9 +8,15 @@ 'Enum', 'IntEnum', 'StrEnum', 'Flag', 'IntFlag', 'auto', 'unique', 'property', + 'FlagBoundary', 'STRICT', 'CONFORM', 'EJECT', 'KEEP', ] +# Dummy value for Enum and Flag as there are explicit checks for them +# before they have been created. +# This is also why there are checks in EnumMeta like `if Enum is not None` +Enum = Flag = EJECT = None + def _is_descriptor(obj): """ Returns True if obj is a descriptor, False otherwise. @@ -56,6 +62,15 @@ def _is_private(cls_name, name): else: return False +def _is_single_bit(num): + """ + True if only one bit set in num (should be an int) + """ + if num == 0: + return False + num &= num - 1 + return num == 0 + def _make_class_unpicklable(obj): """ Make the given obj un-picklable. @@ -71,6 +86,37 @@ def _break_on_call_reduce(self, proto): setattr(obj, '__reduce_ex__', _break_on_call_reduce) setattr(obj, '__module__', '') +def _iter_bits_lsb(num): + while num: + b = num & (~num + 1) + yield b + num ^= b + +def bin(num, max_bits=None): + """ + Like built-in bin(), except negative values are represented in + twos-compliment, and the leading bit always indicates sign + (0=positive, 1=negative). + + >>> bin(10) + '0b0 1010' + >>> bin(~10) # ~10 is -11 + '0b1 0101' + """ + + ceiling = 2 ** (num).bit_length() + if num >= 0: + s = _bltin_bin(num + ceiling).replace('1', '0', 1) + else: + s = _bltin_bin(~num ^ (ceiling - 1) + ceiling) + sign = s[:3] + digits = s[3:] + if max_bits is not None: + if len(digits) < max_bits: + digits = (sign[-1] * max_bits + digits)[-max_bits:] + return "%s %s" % (sign, digits) + + _auto_null = object() class auto: """ @@ -92,22 +138,30 @@ def __get__(self, instance, ownerclass=None): try: return ownerclass._member_map_[self.name] except KeyError: - raise AttributeError('%r not found in %r' % (self.name, ownerclass.__name__)) + raise AttributeError( + '%s: no attribute %r' % (ownerclass.__name__, self.name) + ) else: if self.fget is None: - raise AttributeError('%s: cannot read attribute %r' % (ownerclass.__name__, self.name)) + raise AttributeError( + '%s: no attribute %r' % (ownerclass.__name__, self.name) + ) else: return self.fget(instance) def __set__(self, instance, value): if self.fset is None: - raise AttributeError("%s: cannot set attribute %r" % (self.clsname, self.name)) + raise AttributeError( + "%s: cannot set attribute %r" % (self.clsname, self.name) + ) else: return self.fset(instance, value) def __delete__(self, instance): if self.fdel is None: - raise AttributeError("%s: cannot delete attribute %r" % (self.clsname, self.name)) + raise AttributeError( + "%s: cannot delete attribute %r" % (self.clsname, self.name) + ) else: return self.fdel(instance) @@ -148,11 +202,17 @@ def __set_name__(self, enum_class, member_name): if enum_class._member_type_ is object: enum_member._value_ = value else: - enum_member._value_ = enum_class._member_type_(*args) + try: + enum_member._value_ = enum_class._member_type_(*args) + except Exception as exc: + raise TypeError( + '_value_ not set in __new__, unable to create it' + ) from None value = enum_member._value_ enum_member._name_ = member_name enum_member.__objclass__ = enum_class enum_member.__init__(*args) + enum_member._sort_order_ = len(enum_class._member_names_) # If another member with the same value was already defined, the # new member becomes an alias to the existing one. for name, canonical_member in enum_class._member_map_.items(): @@ -160,8 +220,21 @@ def __set_name__(self, enum_class, member_name): enum_member = canonical_member break else: - # no other instances found, record this member in _member_names_ - enum_class._member_names_.append(member_name) + # this could still be an alias if the value is multi-bit and the + # class is a flag class + if ( + Flag is None + or not issubclass(enum_class, Flag) + ): + # no other instances found, record this member in _member_names_ + enum_class._member_names_.append(member_name) + elif ( + Flag is not None + and issubclass(enum_class, Flag) + and _is_single_bit(value) + ): + # no other instances found, record this member in _member_names_ + enum_class._member_names_.append(member_name) # get redirect in place before adding to _member_map_ # but check for other instances in parent classes first need_override = False @@ -193,7 +266,7 @@ def __set_name__(self, enum_class, member_name): # This may fail if value is not hashable. We can't add the value # to the map, and by-value lookups for this value will be # linear. - enum_class._value2member_map_[value] = enum_member + enum_class._value2member_map_.setdefault(value, enum_member) except TypeError: pass @@ -228,6 +301,7 @@ def __setitem__(self, key, value): if key not in ( '_order_', '_create_pseudo_member_', '_generate_next_value_', '_missing_', '_ignore_', + '_iter_member_', '_iter_member_by_value_', '_iter_member_by_def_', ): raise ValueError( '_sunder_ names, such as %r, are reserved for future Enum use' @@ -265,10 +339,7 @@ def __setitem__(self, key, value): if isinstance(value, auto): if value.value == _auto_null: value.value = self._generate_next_value( - key, - 1, - len(self._member_names), - self._last_values[:], + key, 1, len(self._member_names), self._last_values[:], ) self._auto_called = True value = value.value @@ -287,15 +358,11 @@ def update(self, members, **more_members): self[name] = value -# Dummy value for Enum as EnumMeta explicitly checks for it, but of course -# until EnumMeta finishes running the first time the Enum class doesn't exist. -# This is also why there are checks in EnumMeta like `if Enum is not None` -Enum = None - class EnumMeta(type): """ Metaclass for Enum """ + @classmethod def __prepare__(metacls, cls, bases, **kwds): # check that previous enum members do not exist @@ -311,7 +378,7 @@ def __prepare__(metacls, cls, bases, **kwds): ) return enum_dict - def __new__(metacls, cls, bases, classdict, **kwds): + def __new__(metacls, cls, bases, classdict, boundary=None, **kwds): # an Enum class is final once enumeration items have been defined; it # cannot be mixed with other types (int, float, etc.) if it has an # inherited __new__ unless a new __new__ is defined (or the resulting @@ -346,15 +413,29 @@ def __new__(metacls, cls, bases, classdict, **kwds): classdict['_use_args_'] = use_args # # convert future enum members into temporary _proto_members + # and record integer values in case this will be a Flag + flag_mask = 0 for name in member_names: - classdict[name] = _proto_member(classdict[name]) + value = classdict[name] + if isinstance(value, int): + flag_mask |= value + classdict[name] = _proto_member(value) # - # house keeping structures + # house-keeping structures classdict['_member_names_'] = [] classdict['_member_map_'] = {} classdict['_value2member_map_'] = {} classdict['_member_type_'] = member_type # + # Flag structures (will be removed if final class is not a Flag + classdict['_boundary_'] = ( + boundary + or getattr(first_enum, '_boundary_', None) + ) + classdict['_flag_mask_'] = flag_mask + classdict['_all_bits_'] = 2 ** ((flag_mask).bit_length()) - 1 + classdict['_inverted_'] = None + # # If a custom type is mixed into the Enum, and it does not know how # to pickle itself, pickle.dumps will succeed but pickle.loads will # fail. Rather than have the error show up later and possibly far @@ -408,11 +489,75 @@ def __new__(metacls, cls, bases, classdict, **kwds): enum_class.__new__ = Enum.__new__ # # py3 support for definition order (helps keep py2/py3 code in sync) + # + # _order_ checking is spread out into three/four steps + # - if enum_class is a Flag: + # - remove any non-single-bit flags from _order_ + # - remove any aliases from _order_ + # - check that _order_ and _member_names_ match + # + # step 1: ensure we have a list if _order_ is not None: if isinstance(_order_, str): _order_ = _order_.replace(',', ' ').split() + # + # remove Flag structures if final class is not a Flag + if ( + Flag is None and cls != 'Flag' + or Flag is not None and not issubclass(enum_class, Flag) + ): + delattr(enum_class, '_boundary_') + delattr(enum_class, '_flag_mask_') + delattr(enum_class, '_all_bits_') + delattr(enum_class, '_inverted_') + elif Flag is not None and issubclass(enum_class, Flag): + # ensure _all_bits_ is correct and there are no missing flags + single_bit_total = 0 + multi_bit_total = 0 + for flag in enum_class._member_map_.values(): + flag_value = flag._value_ + if _is_single_bit(flag_value): + single_bit_total |= flag_value + else: + # multi-bit flags are considered aliases + multi_bit_total |= flag_value + if enum_class._boundary_ is not KEEP: + missed = list(_iter_bits_lsb(multi_bit_total & ~single_bit_total)) + if missed: + raise TypeError( + 'invalid Flag %r -- missing values: %s' + % (cls, ', '.join((str(i) for i in missed))) + ) + enum_class._flag_mask_ = single_bit_total + # + # set correct __iter__ + member_list = [m._value_ for m in enum_class] + if member_list != sorted(member_list): + enum_class._iter_member_ = enum_class._iter_member_by_def_ + if _order_: + # _order_ step 2: remove any items from _order_ that are not single-bit + _order_ = [ + o + for o in _order_ + if o not in enum_class._member_map_ or _is_single_bit(enum_class[o]._value_) + ] + # + if _order_: + # _order_ step 3: remove aliases from _order_ + _order_ = [ + o + for o in _order_ + if ( + o not in enum_class._member_map_ + or + (o in enum_class._member_map_ and o in enum_class._member_names_) + )] + # _order_ step 4: verify that _order_ and _member_names_ match if _order_ != enum_class._member_names_: - raise TypeError('member order does not match _order_') + raise TypeError( + 'member order does not match _order_:\n%r\n%r' + % (enum_class._member_names_, _order_) + ) # return enum_class @@ -422,7 +567,7 @@ def __bool__(self): """ return True - def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1): + def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None): """ Either returns an existing member, or creates a new enum class. @@ -457,6 +602,7 @@ def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, s qualname=qualname, type=type, start=start, + boundary=boundary, ) def __contains__(cls, member): @@ -539,7 +685,7 @@ def __setattr__(cls, name, value): raise AttributeError('Cannot reassign members.') super().__setattr__(name, value) - def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, start=1): + def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, start=1, boundary=None): """ Convenience method to create a new Enum class. @@ -589,9 +735,9 @@ def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, s if qualname is not None: classdict['__qualname__'] = qualname - return metacls.__new__(metacls, class_name, bases, classdict) + return metacls.__new__(metacls, class_name, bases, classdict, boundary=boundary) - def _convert_(cls, name, module, filter, source=None): + def _convert_(cls, name, module, filter, source=None, boundary=None): """ Create a new Enum subclass that replaces a collection of global constants """ @@ -618,7 +764,7 @@ def _convert_(cls, name, module, filter, source=None): except TypeError: # unless some values aren't comparable, in which case sort by name members.sort(key=lambda t: t[0]) - cls = cls(name, members, module=module) + cls = cls(name, members, module=module, boundary=boundary or KEEP) cls.__reduce_ex__ = _reduce_ex_by_name module_globals.update(cls.__members__) module_globals[name] = cls @@ -733,6 +879,7 @@ class Enum(metaclass=EnumMeta): Derive from this class to define new enumerations. """ + def __new__(cls, value): # all enum instances are actually created during class construction # without calling this method; this method is called by the metaclass' @@ -761,6 +908,11 @@ def __new__(cls, value): result = None if isinstance(result, cls): return result + elif ( + Flag is not None and issubclass(cls, Flag) + and cls._boundary_ is EJECT and isinstance(result, int) + ): + return result else: ve_exc = ValueError("%r is not a valid %s" % (value, cls.__qualname__)) if result is None and exc is None: @@ -770,7 +922,8 @@ def __new__(cls, value): 'error in %s._missing_: returned %r instead of None or a valid member' % (cls.__name__, result) ) - exc.__context__ = ve_exc + if not isinstance(exc, ValueError): + exc.__context__ = ve_exc raise exc def _generate_next_value_(name, start, count, last_values): @@ -875,14 +1028,14 @@ def __new__(cls, *values): # it must be a string if not isinstance(values[0], str): raise TypeError('%r is not a string' % (values[0], )) - if len(values) > 1: + if len(values) >= 2: # check that encoding argument is a string if not isinstance(values[1], str): raise TypeError('encoding must be a string, not %r' % (values[1], )) - if len(values) > 2: - # check that errors argument is a string - if not isinstance(values[2], str): - raise TypeError('errors must be a string, not %r' % (values[2], )) + if len(values) == 3: + # check that errors argument is a string + if not isinstance(values[2], str): + raise TypeError('errors must be a string, not %r' % (values[2])) value = str(*values) member = str.__new__(cls, value) member._value_ = value @@ -900,7 +1053,22 @@ def _generate_next_value_(name, start, count, last_values): def _reduce_ex_by_name(self, proto): return self.name -class Flag(Enum): +class FlagBoundary(StrEnum): + """ + control how out of range values are handled + "strict" -> error is raised [default for Flag] + "conform" -> extra bits are discarded + "eject" -> lose flag status [default for IntFlag] + "keep" -> keep flag status and all bits + """ + STRICT = auto() + CONFORM = auto() + EJECT = auto() + KEEP = auto() +STRICT, CONFORM, EJECT, KEEP = FlagBoundary + + +class Flag(Enum, boundary=STRICT): """ Support for flags """ @@ -916,45 +1084,108 @@ def _generate_next_value_(name, start, count, last_values): """ if not count: return start if start is not None else 1 - for last_value in reversed(last_values): - try: - high_bit = _high_bit(last_value) - break - except Exception: - raise TypeError('Invalid Flag value: %r' % last_value) from None + last_value = max(last_values) + try: + high_bit = _high_bit(last_value) + except Exception: + raise TypeError('Invalid Flag value: %r' % last_value) from None return 2 ** (high_bit+1) @classmethod - def _missing_(cls, value): + def _iter_member_by_value_(cls, value): """ - Returns member (possibly creating it) if one can be found for value. + Extract all members from the value in definition (i.e. increasing value) order. """ - original_value = value - if value < 0: - value = ~value - possible_member = cls._create_pseudo_member_(value) - if original_value < 0: - possible_member = ~possible_member - return possible_member + for val in _iter_bits_lsb(value & cls._flag_mask_): + yield cls._value2member_map_.get(val) + + _iter_member_ = _iter_member_by_value_ @classmethod - def _create_pseudo_member_(cls, value): + def _iter_member_by_def_(cls, value): + """ + Extract all members from the value in definition order. + """ + yield from sorted( + cls._iter_member_by_value_(value), + key=lambda m: m._sort_order_, + ) + + @classmethod + def _missing_(cls, value): """ Create a composite member iff value contains only members. """ - pseudo_member = cls._value2member_map_.get(value, None) - if pseudo_member is None: - # verify all bits are accounted for - _, extra_flags = _decompose(cls, value) - if extra_flags: - raise ValueError("%r is not a valid %s" % (value, cls.__qualname__)) + if not isinstance(value, int): + raise ValueError( + "%r is not a valid %s" % (value, cls.__qualname__) + ) + # check boundaries + # - value must be in range (e.g. -16 <-> +15, i.e. ~15 <-> 15) + # - value must not include any skipped flags (e.g. if bit 2 is not + # defined, then 0d10 is invalid) + flag_mask = cls._flag_mask_ + all_bits = cls._all_bits_ + neg_value = None + if ( + not ~all_bits <= value <= all_bits + or value & (all_bits ^ flag_mask) + ): + if cls._boundary_ is STRICT: + max_bits = max(value.bit_length(), flag_mask.bit_length()) + raise ValueError( + "%s: invalid value: %r\n given %s\n allowed %s" % ( + cls.__name__, value, bin(value, max_bits), bin(flag_mask, max_bits), + )) + elif cls._boundary_ is CONFORM: + value = value & flag_mask + elif cls._boundary_ is EJECT: + return value + elif cls._boundary_ is KEEP: + if value < 0: + value = ( + max(all_bits+1, 2**(value.bit_length())) + + value + ) + else: + raise ValueError( + 'unknown flag boundary: %r' % (cls._boundary_, ) + ) + if value < 0: + neg_value = value + value = all_bits + 1 + value + # get members and unknown + unknown = value & ~flag_mask + member_value = value & flag_mask + if unknown and cls._boundary_ is not KEEP: + raise ValueError( + '%s(%r) --> unknown values %r [%s]' + % (cls.__name__, value, unknown, bin(unknown)) + ) + # normal Flag? + __new__ = getattr(cls, '__new_member__', None) + if cls._member_type_ is object and not __new__: # construct a singleton enum pseudo-member pseudo_member = object.__new__(cls) - pseudo_member._name_ = None + else: + pseudo_member = (__new__ or cls._member_type_.__new__)(cls, value) + if not hasattr(pseudo_member, 'value'): pseudo_member._value_ = value - # use setdefault in case another thread already created a composite - # with this value + if member_value: + pseudo_member._name_ = '|'.join([ + m._name_ for m in cls._iter_member_(member_value) + ]) + if unknown: + pseudo_member._name_ += '|0x%x' % unknown + else: + pseudo_member._name_ = None + # use setdefault in case another thread already created a composite + # with this value, but only if all members are known + # note: zero is a special case -- add it + if not unknown: pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member) + if neg_value is not None: + cls._value2member_map_[neg_value] = pseudo_member return pseudo_member def __contains__(self, other): @@ -965,38 +1196,33 @@ def __contains__(self, other): raise TypeError( "unsupported operand type(s) for 'in': '%s' and '%s'" % ( type(other).__qualname__, self.__class__.__qualname__)) + if other._value_ == 0 or self._value_ == 0: + return False return other._value_ & self._value_ == other._value_ def __iter__(self): """ - Returns flags in decreasing value order. + Returns flags in definition order. """ - members, extra_flags = _decompose(self.__class__, self.value) - return (m for m in members if m._value_ != 0) + yield from self._iter_member_(self._value_) + + def __len__(self): + return self._value_.bit_count() def __repr__(self): cls = self.__class__ if self._name_ is not None: return '<%s.%s: %r>' % (cls.__name__, self._name_, self._value_) - members, uncovered = _decompose(cls, self._value_) - return '<%s.%s: %r>' % ( - cls.__name__, - '|'.join([str(m._name_ or m._value_) for m in members]), - self._value_, - ) + else: + # only zero is unnamed by default + return '<%s: %r>' % (cls.__name__, self._value_) def __str__(self): cls = self.__class__ if self._name_ is not None: return '%s.%s' % (cls.__name__, self._name_) - members, uncovered = _decompose(cls, self._value_) - if len(members) == 1 and members[0]._name_ is None: - return '%s.%r' % (cls.__name__, members[0]._value_) else: - return '%s.%s' % ( - cls.__name__, - '|'.join([str(m._name_ or m._value_) for m in members]), - ) + return '%s(%s)' % (cls.__name__, self._value_) def __bool__(self): return bool(self._value_) @@ -1017,86 +1243,56 @@ def __xor__(self, other): return self.__class__(self._value_ ^ other._value_) def __invert__(self): - members, uncovered = _decompose(self.__class__, self._value_) - inverted = self.__class__(0) - for m in self.__class__: - if m not in members and not (m._value_ & self._value_): - inverted = inverted | m - return self.__class__(inverted) + if self._inverted_ is None: + if self._boundary_ is KEEP: + # use all bits + self._inverted_ = self.__class__(~self._value_) + else: + # calculate flags not in this member + self._inverted_ = self.__class__(self._flag_mask_ ^ self._value_) + self._inverted_._inverted_ = self + return self._inverted_ -class IntFlag(int, Flag): +class IntFlag(int, Flag, boundary=EJECT): """ Support for integer-based Flags """ - @classmethod - def _missing_(cls, value): - """ - Returns member (possibly creating it) if one can be found for value. - """ - if not isinstance(value, int): - raise ValueError("%r is not a valid %s" % (value, cls.__qualname__)) - new_member = cls._create_pseudo_member_(value) - return new_member - - @classmethod - def _create_pseudo_member_(cls, value): - """ - Create a composite member iff value contains only members. - """ - pseudo_member = cls._value2member_map_.get(value, None) - if pseudo_member is None: - need_to_create = [value] - # get unaccounted for bits - _, extra_flags = _decompose(cls, value) - # timer = 10 - while extra_flags: - # timer -= 1 - bit = _high_bit(extra_flags) - flag_value = 2 ** bit - if (flag_value not in cls._value2member_map_ and - flag_value not in need_to_create - ): - need_to_create.append(flag_value) - if extra_flags == -flag_value: - extra_flags = 0 - else: - extra_flags ^= flag_value - for value in reversed(need_to_create): - # construct singleton pseudo-members - pseudo_member = int.__new__(cls, value) - pseudo_member._name_ = None - pseudo_member._value_ = value - # use setdefault in case another thread already created a composite - # with this value - pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member) - return pseudo_member - def __or__(self, other): - if not isinstance(other, (self.__class__, int)): + if isinstance(other, self.__class__): + other = other._value_ + elif isinstance(other, int): + other = other + else: return NotImplemented - result = self.__class__(self._value_ | self.__class__(other)._value_) - return result + value = self._value_ + return self.__class__(value | other) def __and__(self, other): - if not isinstance(other, (self.__class__, int)): + if isinstance(other, self.__class__): + other = other._value_ + elif isinstance(other, int): + other = other + else: return NotImplemented - return self.__class__(self._value_ & self.__class__(other)._value_) + value = self._value_ + return self.__class__(value & other) def __xor__(self, other): - if not isinstance(other, (self.__class__, int)): + if isinstance(other, self.__class__): + other = other._value_ + elif isinstance(other, int): + other = other + else: return NotImplemented - return self.__class__(self._value_ ^ self.__class__(other)._value_) + value = self._value_ + return self.__class__(value ^ other) __ror__ = __or__ __rand__ = __and__ __rxor__ = __xor__ - - def __invert__(self): - result = self.__class__(~self._value_) - return result - + __invert__ = Flag.__invert__ def _high_bit(value): """ @@ -1119,31 +1315,7 @@ def unique(enumeration): (enumeration, alias_details)) return enumeration -def _decompose(flag, value): - """ - Extract all members from the value. - """ - # _decompose is only called if the value is not named - not_covered = value - negative = value < 0 - members = [] - for member in flag: - member_value = member.value - if member_value and member_value & value == member_value: - members.append(member) - not_covered &= ~member_value - if not negative: - tmp = not_covered - while tmp: - flag_value = 2 ** _high_bit(tmp) - if flag_value in flag._value2member_map_: - members.append(flag._value2member_map_[flag_value]) - not_covered &= ~flag_value - tmp &= ~flag_value - if not members and value in flag._value2member_map_: - members.append(flag._value2member_map_[value]) - members.sort(key=lambda m: m._value_, reverse=True) - if len(members) > 1 and members[0].value == value: - # we have the breakdown, don't need the value member itself - members.pop(0) - return members, not_covered +def _power_of_two(value): + if value < 1: + return False + return value == 2 ** _high_bit(value) diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index 26200981eb8d93..f1abb38eee2e54 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -3,6 +3,9 @@ Released on 2021-10-04? ====================================== +bpo-43008: Make IDLE invoke :func:`sys.excepthook` in normal, +2-process mode. + bpo-33065: Fix problem debugging user classes with __repr__ method. bpo-32631: Finish zzdummy example extension module: make menu entries diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 73e64852c69dfd..c52a04b503adb4 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -18,8 +18,8 @@ HORIZONTAL, VERTICAL, ANCHOR, ACTIVE, END) from tkinter.ttk import (Frame, LabelFrame, Button, Checkbutton, Entry, Label, OptionMenu, Notebook, Radiobutton, Scrollbar, Style) -import tkinter.colorchooser as tkColorChooser -import tkinter.font as tkFont +from tkinter import colorchooser +import tkinter.font as tkfont from tkinter import messagebox from idlelib.config import idleConf, ConfigChanges @@ -609,7 +609,7 @@ def load_font_cfg(self): font_bold = configured_font[2]=='bold' # Set sorted no-duplicate editor font selection list and font_name. - fonts = sorted(set(tkFont.families(self))) + fonts = sorted(set(tkfont.families(self))) for font in fonts: self.fontlist.insert(END, font) self.font_name.set(font_name) @@ -663,7 +663,7 @@ def set_samples(self, event=None): Updates font_sample and highlight page highlight_sample. """ font_name = self.font_name.get() - font_weight = tkFont.BOLD if self.font_bold.get() else tkFont.NORMAL + font_weight = tkfont.BOLD if self.font_bold.get() else tkfont.NORMAL new_font = (font_name, self.font_size.get(), font_weight) self.font_sample['font'] = new_font self.highlight_sample['font'] = new_font @@ -1100,7 +1100,7 @@ def get_color(self): target = self.highlight_target.get() prev_color = self.style.lookup(self.frame_color_set['style'], 'background') - rgbTuplet, color_string = tkColorChooser.askcolor( + rgbTuplet, color_string = colorchooser.askcolor( parent=self, title='Pick new color for : '+target, initialcolor=prev_color) if color_string and (color_string != prev_color): diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index a178eaf93c013a..66e9da5a9dccf9 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -12,8 +12,8 @@ from tkinter import * from tkinter.font import Font from tkinter.ttk import Scrollbar -import tkinter.simpledialog as tkSimpleDialog -import tkinter.messagebox as tkMessageBox +from tkinter import simpledialog +from tkinter import messagebox from idlelib.config import idleConf from idlelib import configdialog @@ -46,7 +46,7 @@ def _sphinx_version(): return release -class EditorWindow(object): +class EditorWindow: from idlelib.percolator import Percolator from idlelib.colorizer import ColorDelegator, color_config from idlelib.undo import UndoDelegator @@ -295,9 +295,9 @@ def __init__(self, flist=None, filename=None, key=None, root=None): window.register_callback(self.postwindowsmenu) # Some abstractions so IDLE extensions are cross-IDE - self.askyesno = tkMessageBox.askyesno - self.askinteger = tkSimpleDialog.askinteger - self.showerror = tkMessageBox.showerror + self.askinteger = simpledialog.askinteger + self.askyesno = messagebox.askyesno + self.showerror = messagebox.showerror # Add pseudoevents for former extension fixed keys. # (This probably needs to be done once in the process.) @@ -596,7 +596,7 @@ def python_docs(self, event=None): try: os.startfile(self.help_url) except OSError as why: - tkMessageBox.showerror(title='Document Start Failure', + messagebox.showerror(title='Document Start Failure', message=str(why), parent=self.text) else: webbrowser.open(self.help_url) @@ -927,7 +927,7 @@ def display_extra_help(helpfile=helpfile): try: os.startfile(helpfile) except OSError as why: - tkMessageBox.showerror(title='Document Start Failure', + messagebox.showerror(title='Document Start Failure', message=str(why), parent=self.text) else: webbrowser.open(helpfile) @@ -963,7 +963,7 @@ def update_recent_files_list(self, new_file=None): except OSError as err: if not getattr(self.root, "recentfiles_message", False): self.root.recentfiles_message = True - tkMessageBox.showwarning(title='IDLE Warning', + messagebox.showwarning(title='IDLE Warning', message="Cannot save Recent Files list to disk.\n" f" {err}\n" "Select OK to continue.", @@ -1546,7 +1546,7 @@ def get_line_indent(line, tabwidth): return m.end(), len(m.group().expandtabs(tabwidth)) -class IndentSearcher(object): +class IndentSearcher: # .run() chews over the Text widget, looking for a block opener # and the stmt following it. Returns a pair, diff --git a/Lib/idlelib/filelist.py b/Lib/idlelib/filelist.py index 0d200854ef0007..254f5caf6b81b0 100644 --- a/Lib/idlelib/filelist.py +++ b/Lib/idlelib/filelist.py @@ -1,7 +1,7 @@ "idlelib.filelist" import os -from tkinter import messagebox as tkMessageBox +from tkinter import messagebox class FileList: @@ -20,7 +20,7 @@ def open(self, filename, action=None): filename = self.canonize(filename) if os.path.isdir(filename): # This can happen when bad filename is passed on command line: - tkMessageBox.showerror( + messagebox.showerror( "File Error", "%r is a directory." % (filename,), master=self.root) @@ -88,7 +88,7 @@ def filename_changed_edit(self, edit): if newkey in self.dict: conflict = self.dict[newkey] self.inversedict[conflict] = None - tkMessageBox.showerror( + messagebox.showerror( "Name Conflict", "You now have multiple edit windows open for %r" % (filename,), master=self.root) diff --git a/Lib/idlelib/idle_test/mock_tk.py b/Lib/idlelib/idle_test/mock_tk.py index b736bd001da87f..db583553838fb3 100644 --- a/Lib/idlelib/idle_test/mock_tk.py +++ b/Lib/idlelib/idle_test/mock_tk.py @@ -59,27 +59,26 @@ def __call__(self, title, message, *args, **kwds): class Mbox: """Mock for tkinter.messagebox with an Mbox_func for each function. - This module was 'tkMessageBox' in 2.x; hence the 'import as' in 3.x. Example usage in test_module.py for testing functions in module.py: --- from idlelib.idle_test.mock_tk import Mbox import module -orig_mbox = module.tkMessageBox +orig_mbox = module.messagebox showerror = Mbox.showerror # example, for attribute access in test methods class Test(unittest.TestCase): @classmethod def setUpClass(cls): - module.tkMessageBox = Mbox + module.messagebox = Mbox @classmethod def tearDownClass(cls): - module.tkMessageBox = orig_mbox + module.messagebox = orig_mbox --- For 'ask' functions, set func.result return value before calling the method - that uses the message function. When tkMessageBox functions are the + that uses the message function. When messagebox functions are the only gui alls in a method, this replacement makes the method gui-free, """ askokcancel = Mbox_func() # True or False diff --git a/Lib/idlelib/idle_test/test_autocomplete.py b/Lib/idlelib/idle_test/test_autocomplete.py index 9c113bd893f137..642bb5db64dc34 100644 --- a/Lib/idlelib/idle_test/test_autocomplete.py +++ b/Lib/idlelib/idle_test/test_autocomplete.py @@ -195,7 +195,7 @@ def test_open_completions_none(self): self.assertFalse(acp.open_completions(ac.TAB)) self.text.delete('1.0', 'end') - class dummy_acw(): + class dummy_acw: __init__ = Func() show_window = Func(result=False) hide_window = Func() diff --git a/Lib/idlelib/idle_test/test_calltip.py b/Lib/idlelib/idle_test/test_calltip.py index a76829f3656c80..b23915c5ab7849 100644 --- a/Lib/idlelib/idle_test/test_calltip.py +++ b/Lib/idlelib/idle_test/test_calltip.py @@ -10,7 +10,7 @@ # Test Class TC is used in multiple get_argspec test methods -class TC(): +class TC: 'doc' tip = "(ai=None, *b)" def __init__(self, ai=None, *b): 'doc' @@ -268,7 +268,7 @@ def test_good_entity(self): # open_calltip is about half the code; the others are fairly trivial. # The default mocks are what are needed for open_calltip. -class mock_Shell(): +class mock_Shell: "Return mock sufficient to pass to hyperparser." def __init__(self, text): text.tag_prevrange = Mock(return_value=None) diff --git a/Lib/idlelib/idle_test/test_codecontext.py b/Lib/idlelib/idle_test/test_codecontext.py index 9578cc731a6f93..6969ad73b01a81 100644 --- a/Lib/idlelib/idle_test/test_codecontext.py +++ b/Lib/idlelib/idle_test/test_codecontext.py @@ -20,7 +20,7 @@ } code_sample = """\ -class C1(): +class C1: # Class comment. def __init__(self, a, b): self.a = a @@ -178,29 +178,29 @@ def test_get_context(self): with self.assertRaises(AssertionError): gc(1, stopline=0) - eq(gc(3), ([(2, 0, 'class C1():', 'class')], 0)) + eq(gc(3), ([(2, 0, 'class C1:', 'class')], 0)) # Don't return comment. - eq(gc(4), ([(2, 0, 'class C1():', 'class')], 0)) + eq(gc(4), ([(2, 0, 'class C1:', 'class')], 0)) # Two indentation levels and no comment. - eq(gc(5), ([(2, 0, 'class C1():', 'class'), + eq(gc(5), ([(2, 0, 'class C1:', 'class'), (4, 4, ' def __init__(self, a, b):', 'def')], 0)) # Only one 'def' is returned, not both at the same indent level. - eq(gc(10), ([(2, 0, 'class C1():', 'class'), + eq(gc(10), ([(2, 0, 'class C1:', 'class'), (7, 4, ' def compare(self):', 'def'), (8, 8, ' if a > b:', 'if')], 0)) # With 'elif', also show the 'if' even though it's at the same level. - eq(gc(11), ([(2, 0, 'class C1():', 'class'), + eq(gc(11), ([(2, 0, 'class C1:', 'class'), (7, 4, ' def compare(self):', 'def'), (8, 8, ' if a > b:', 'if'), (10, 8, ' elif a < b:', 'elif')], 0)) # Set stop_line to not go back to first line in source code. # Return includes stop_line. - eq(gc(11, stopline=2), ([(2, 0, 'class C1():', 'class'), + eq(gc(11, stopline=2), ([(2, 0, 'class C1:', 'class'), (7, 4, ' def compare(self):', 'def'), (8, 8, ' if a > b:', 'if'), (10, 8, ' elif a < b:', 'elif')], 0)) @@ -240,37 +240,37 @@ def test_update_code_context(self): # Scroll down to line 2. cc.text.yview(2) cc.update_code_context() - eq(cc.info, [(0, -1, '', False), (2, 0, 'class C1():', 'class')]) + eq(cc.info, [(0, -1, '', False), (2, 0, 'class C1:', 'class')]) eq(cc.topvisible, 3) - eq(cc.context.get('1.0', 'end-1c'), 'class C1():') + eq(cc.context.get('1.0', 'end-1c'), 'class C1:') # Scroll down to line 3. Since it's a comment, nothing changes. cc.text.yview(3) cc.update_code_context() - eq(cc.info, [(0, -1, '', False), (2, 0, 'class C1():', 'class')]) + eq(cc.info, [(0, -1, '', False), (2, 0, 'class C1:', 'class')]) eq(cc.topvisible, 4) - eq(cc.context.get('1.0', 'end-1c'), 'class C1():') + eq(cc.context.get('1.0', 'end-1c'), 'class C1:') # Scroll down to line 4. cc.text.yview(4) cc.update_code_context() eq(cc.info, [(0, -1, '', False), - (2, 0, 'class C1():', 'class'), + (2, 0, 'class C1:', 'class'), (4, 4, ' def __init__(self, a, b):', 'def')]) eq(cc.topvisible, 5) - eq(cc.context.get('1.0', 'end-1c'), 'class C1():\n' + eq(cc.context.get('1.0', 'end-1c'), 'class C1:\n' ' def __init__(self, a, b):') # Scroll down to line 11. Last 'def' is removed. cc.text.yview(11) cc.update_code_context() eq(cc.info, [(0, -1, '', False), - (2, 0, 'class C1():', 'class'), + (2, 0, 'class C1:', 'class'), (7, 4, ' def compare(self):', 'def'), (8, 8, ' if a > b:', 'if'), (10, 8, ' elif a < b:', 'elif')]) eq(cc.topvisible, 12) - eq(cc.context.get('1.0', 'end-1c'), 'class C1():\n' + eq(cc.context.get('1.0', 'end-1c'), 'class C1:\n' ' def compare(self):\n' ' if a > b:\n' ' elif a < b:') @@ -279,12 +279,12 @@ def test_update_code_context(self): cc.update_code_context() cc.context_depth = 1 eq(cc.info, [(0, -1, '', False), - (2, 0, 'class C1():', 'class'), + (2, 0, 'class C1:', 'class'), (7, 4, ' def compare(self):', 'def'), (8, 8, ' if a > b:', 'if'), (10, 8, ' elif a < b:', 'elif')]) eq(cc.topvisible, 12) - eq(cc.context.get('1.0', 'end-1c'), 'class C1():\n' + eq(cc.context.get('1.0', 'end-1c'), 'class C1:\n' ' def compare(self):\n' ' if a > b:\n' ' elif a < b:') @@ -293,7 +293,7 @@ def test_update_code_context(self): cc.text.yview(5) cc.update_code_context() eq(cc.info, [(0, -1, '', False), - (2, 0, 'class C1():', 'class'), + (2, 0, 'class C1:', 'class'), (4, 4, ' def __init__(self, a, b):', 'def')]) eq(cc.topvisible, 6) # context_depth is 1. @@ -440,7 +440,7 @@ def test_get_line_info(self): # Line 1 is not a BLOCKOPENER. eq(gli(lines[0]), (codecontext.INFINITY, '', False)) # Line 2 is a BLOCKOPENER without an indent. - eq(gli(lines[1]), (0, 'class C1():', 'class')) + eq(gli(lines[1]), (0, 'class C1:', 'class')) # Line 3 is not a BLOCKOPENER and does not return the indent level. eq(gli(lines[2]), (codecontext.INFINITY, ' # Class comment.', False)) # Line 4 is a BLOCKOPENER and is indented. diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index 1fea6d41df811c..98ddc67afdcc08 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -423,7 +423,7 @@ def test_custom_name(self): def test_color(self): d = self.page d.on_new_color_set = Func() - # self.color is only set in get_color through ColorChooser. + # self.color is only set in get_color through colorchooser. d.color.set('green') self.assertEqual(d.on_new_color_set.called, 1) del d.on_new_color_set @@ -540,8 +540,8 @@ def test_set_theme_type(self): def test_get_color(self): eq = self.assertEqual d = self.page - orig_chooser = configdialog.tkColorChooser.askcolor - chooser = configdialog.tkColorChooser.askcolor = Func() + orig_chooser = configdialog.colorchooser.askcolor + chooser = configdialog.colorchooser.askcolor = Func() gntn = d.get_new_theme_name = Func() d.highlight_target.set('Editor Breakpoint') @@ -582,7 +582,7 @@ def test_get_color(self): eq(d.color.get(), '#de0000') del d.get_new_theme_name - configdialog.tkColorChooser.askcolor = orig_chooser + configdialog.colorchooser.askcolor = orig_chooser def test_on_new_color_set(self): d = self.page diff --git a/Lib/idlelib/idle_test/test_format.py b/Lib/idlelib/idle_test/test_format.py index a79bb515089e7b..e5e903688597aa 100644 --- a/Lib/idlelib/idle_test/test_format.py +++ b/Lib/idlelib/idle_test/test_format.py @@ -418,7 +418,7 @@ def tearDown(self): code_sample = """\ # WS line needed for test. -class C1(): +class C1: # Class comment. def __init__(self, a, b): self.a = a diff --git a/Lib/idlelib/idle_test/test_help_about.py b/Lib/idlelib/idle_test/test_help_about.py index 7c148d23a135b6..b915535acac0cc 100644 --- a/Lib/idlelib/idle_test/test_help_about.py +++ b/Lib/idlelib/idle_test/test_help_about.py @@ -134,7 +134,7 @@ def test_close(self): self.dialog.winfo_class() -class Dummy_about_dialog(): +class Dummy_about_dialog: # Dummy class for testing file display functions. idle_credits = About.show_idle_credits idle_readme = About.show_readme diff --git a/Lib/idlelib/idle_test/test_pyparse.py b/Lib/idlelib/idle_test/test_pyparse.py index f21baf7534420a..fb5726db1d821e 100644 --- a/Lib/idlelib/idle_test/test_pyparse.py +++ b/Lib/idlelib/idle_test/test_pyparse.py @@ -73,11 +73,12 @@ def char_in_string_false(index): return False # Split def across lines. setcode('"""This is a module docstring"""\n' - 'class C():\n' + 'class C:\n' ' def __init__(self, a,\n' ' b=True):\n' ' pass\n' ) + pos0, pos = 33, 42 # Start of 'class...', ' def' lines. # Passing no value or non-callable should fail (issue 32989). with self.assertRaises(TypeError): @@ -91,40 +92,41 @@ def char_in_string_false(index): return False # Make all text look like it's not in a string. This means that it # found a good start position. - eq(start(char_in_string_false), 44) + eq(start(char_in_string_false), pos) # If the beginning of the def line is not in a string, then it # returns that as the index. - eq(start(is_char_in_string=lambda index: index > 44), 44) + eq(start(is_char_in_string=lambda index: index > pos), pos) # If the beginning of the def line is in a string, then it # looks for a previous index. - eq(start(is_char_in_string=lambda index: index >= 44), 33) + eq(start(is_char_in_string=lambda index: index >= pos), pos0) # If everything before the 'def' is in a string, then returns None. # The non-continuation def line returns 44 (see below). - eq(start(is_char_in_string=lambda index: index < 44), None) + eq(start(is_char_in_string=lambda index: index < pos), None) # Code without extra line break in def line - mostly returns the same # values. setcode('"""This is a module docstring"""\n' - 'class C():\n' + 'class C:\n' ' def __init__(self, a, b=True):\n' ' pass\n' - ) - eq(start(char_in_string_false), 44) - eq(start(is_char_in_string=lambda index: index > 44), 44) - eq(start(is_char_in_string=lambda index: index >= 44), 33) + ) # Does not affect class, def positions. + eq(start(char_in_string_false), pos) + eq(start(is_char_in_string=lambda index: index > pos), pos) + eq(start(is_char_in_string=lambda index: index >= pos), pos0) # When the def line isn't split, this returns which doesn't match the # split line test. - eq(start(is_char_in_string=lambda index: index < 44), 44) + eq(start(is_char_in_string=lambda index: index < pos), pos) def test_set_lo(self): code = ( '"""This is a module docstring"""\n' - 'class C():\n' + 'class C:\n' ' def __init__(self, a,\n' ' b=True):\n' ' pass\n' ) + pos = 42 p = self.parser p.set_code(code) @@ -137,8 +139,8 @@ def test_set_lo(self): self.assertEqual(p.code, code) # An index that is preceded by a newline. - p.set_lo(44) - self.assertEqual(p.code, code[44:]) + p.set_lo(pos) + self.assertEqual(p.code, code[pos:]) def test_study1(self): eq = self.assertEqual diff --git a/Lib/idlelib/idle_test/test_replace.py b/Lib/idlelib/idle_test/test_replace.py index c3c5d2eeb94998..6c07389b29ad45 100644 --- a/Lib/idlelib/idle_test/test_replace.py +++ b/Lib/idlelib/idle_test/test_replace.py @@ -10,7 +10,7 @@ from idlelib.idle_test.mock_tk import Mbox import idlelib.searchengine as se -orig_mbox = se.tkMessageBox +orig_mbox = se.messagebox showerror = Mbox.showerror @@ -20,7 +20,7 @@ class ReplaceDialogTest(unittest.TestCase): def setUpClass(cls): cls.root = Tk() cls.root.withdraw() - se.tkMessageBox = Mbox + se.messagebox = Mbox cls.engine = se.SearchEngine(cls.root) cls.dialog = ReplaceDialog(cls.root, cls.engine) cls.dialog.bell = lambda: None @@ -32,7 +32,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - se.tkMessageBox = orig_mbox + se.messagebox = orig_mbox del cls.text, cls.dialog, cls.engine cls.root.destroy() del cls.root diff --git a/Lib/idlelib/idle_test/test_run.py b/Lib/idlelib/idle_test/test_run.py index 37c0d4525e56cd..a31671ee0485fa 100644 --- a/Lib/idlelib/idle_test/test_run.py +++ b/Lib/idlelib/idle_test/test_run.py @@ -1,16 +1,18 @@ "Test run, coverage 49%." from idlelib import run +import io +import sys +from test.support import captured_output, captured_stderr import unittest from unittest import mock +import idlelib from idlelib.idle_test.mock_idle import Func -from test.support import captured_output, captured_stderr -import io -import sys +idlelib.testing = True # Use {} for executing test user code. -class RunTest(unittest.TestCase): +class PrintExceptionTest(unittest.TestCase): def test_print_exception_unhashable(self): class UnhashableException(Exception): @@ -351,5 +353,38 @@ def test_fatal_error(self): self.assertIn('IndexError', msg) eq(func.called, 2) + +class ExecRuncodeTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.addClassCleanup(setattr,run,'print_exception',run.print_exception) + cls.prt = Func() # Need reference. + run.print_exception = cls.prt + mockrpc = mock.Mock() + mockrpc.console.getvar = Func(result=False) + cls.ex = run.Executive(mockrpc) + + @classmethod + def tearDownClass(cls): + assert sys.excepthook == sys.__excepthook__ + + def test_exceptions(self): + ex = self.ex + ex.runcode('1/0') + self.assertIs(ex.user_exc_info[0], ZeroDivisionError) + + self.addCleanup(setattr, sys, 'excepthook', sys.__excepthook__) + sys.excepthook = lambda t, e, tb: run.print_exception(t) + ex.runcode('1/0') + self.assertIs(self.prt.args[0], ZeroDivisionError) + + sys.excepthook = lambda: None + ex.runcode('1/0') + t, e, tb = ex.user_exc_info + self.assertIs(t, TypeError) + self.assertTrue(isinstance(e.__context__, ZeroDivisionError)) + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_searchengine.py b/Lib/idlelib/idle_test/test_searchengine.py index f8401ce9380f25..9d979839419586 100644 --- a/Lib/idlelib/idle_test/test_searchengine.py +++ b/Lib/idlelib/idle_test/test_searchengine.py @@ -4,7 +4,7 @@ import unittest # from test.support import requires from tkinter import BooleanVar, StringVar, TclError # ,Tk, Text -import tkinter.messagebox as tkMessageBox +from tkinter import messagebox from idlelib.idle_test.mock_tk import Var, Mbox from idlelib.idle_test.mock_tk import Text as mockText import re @@ -19,13 +19,13 @@ def setUpModule(): # Replace s-e module tkinter imports other than non-gui TclError. se.BooleanVar = Var se.StringVar = Var - se.tkMessageBox = Mbox + se.messagebox = Mbox def tearDownModule(): # Restore 'just in case', though other tests should also replace. se.BooleanVar = BooleanVar se.StringVar = StringVar - se.tkMessageBox = tkMessageBox + se.messagebox = messagebox class Mock: diff --git a/Lib/idlelib/idle_test/test_squeezer.py b/Lib/idlelib/idle_test/test_squeezer.py index e3912f4bbbec89..ee1bbd76b50562 100644 --- a/Lib/idlelib/idle_test/test_squeezer.py +++ b/Lib/idlelib/idle_test/test_squeezer.py @@ -396,7 +396,7 @@ def test_expand_dangerous_oupput(self): expandingbutton.base_text = expandingbutton.text # Patch the message box module to always return False. - with patch('idlelib.squeezer.tkMessageBox') as mock_msgbox: + with patch('idlelib.squeezer.messagebox') as mock_msgbox: mock_msgbox.askokcancel.return_value = False mock_msgbox.askyesno.return_value = False # Trigger the expand event. @@ -407,7 +407,7 @@ def test_expand_dangerous_oupput(self): self.assertEqual(expandingbutton.text.get('1.0', 'end-1c'), '') # Patch the message box module to always return True. - with patch('idlelib.squeezer.tkMessageBox') as mock_msgbox: + with patch('idlelib.squeezer.messagebox') as mock_msgbox: mock_msgbox.askokcancel.return_value = True mock_msgbox.askyesno.return_value = True # Trigger the expand event. diff --git a/Lib/idlelib/iomenu.py b/Lib/idlelib/iomenu.py index 8bb2fa6a6e7939..5ebf7089fb9abe 100644 --- a/Lib/idlelib/iomenu.py +++ b/Lib/idlelib/iomenu.py @@ -5,8 +5,8 @@ import tempfile import tokenize -import tkinter.filedialog as tkFileDialog -import tkinter.messagebox as tkMessageBox +from tkinter import filedialog +from tkinter import messagebox from tkinter.simpledialog import askstring import idlelib @@ -147,10 +147,10 @@ def loadfile(self, filename): eol_convention = f.newlines converted = True except OSError as err: - tkMessageBox.showerror("I/O Error", str(err), parent=self.text) + messagebox.showerror("I/O Error", str(err), parent=self.text) return False except UnicodeDecodeError: - tkMessageBox.showerror("Decoding Error", + messagebox.showerror("Decoding Error", "File %s\nFailed to Decode" % filename, parent=self.text) return False @@ -159,7 +159,7 @@ def loadfile(self, filename): # If the file does not contain line separators, it is None. # If the file contains mixed line separators, it is a tuple. if eol_convention is not None: - tkMessageBox.showwarning("Mixed Newlines", + messagebox.showwarning("Mixed Newlines", "Mixed newlines detected.\n" "The file will be changed on save.", parent=self.text) @@ -187,10 +187,10 @@ def maybesave(self): return "yes" message = "Do you want to save %s before closing?" % ( self.filename or "this untitled document") - confirm = tkMessageBox.askyesnocancel( + confirm = messagebox.askyesnocancel( title="Save On Close", message=message, - default=tkMessageBox.YES, + default=messagebox.YES, parent=self.text) if confirm: reply = "yes" @@ -249,7 +249,7 @@ def writefile(self, filename): os.fsync(f.fileno()) return True except OSError as msg: - tkMessageBox.showerror("I/O Error", str(msg), + messagebox.showerror("I/O Error", str(msg), parent=self.text) return False @@ -286,7 +286,7 @@ def encode(self, chars): failed = str(err) except UnicodeEncodeError: failed = "Invalid encoding '%s'" % enc - tkMessageBox.showerror( + messagebox.showerror( "I/O Error", "%s.\nSaving as UTF-8" % failed, parent=self.text) @@ -295,10 +295,10 @@ def encode(self, chars): return chars.encode('utf-8-sig') def print_window(self, event): - confirm = tkMessageBox.askokcancel( + confirm = messagebox.askokcancel( title="Print", message="Print to Default Printer", - default=tkMessageBox.OK, + default=messagebox.OK, parent=self.text) if not confirm: self.text.focus_set() @@ -336,10 +336,10 @@ def print_window(self, event): status + output if output: output = "Printing command: %s\n" % repr(command) + output - tkMessageBox.showerror("Print status", output, parent=self.text) + messagebox.showerror("Print status", output, parent=self.text) else: #no printing for this platform message = "Printing is not enabled for this platform: %s" % platform - tkMessageBox.showinfo("Print status", message, parent=self.text) + messagebox.showinfo("Print status", message, parent=self.text) if tempfilename: os.unlink(tempfilename) return "break" @@ -358,7 +358,7 @@ def print_window(self, event): def askopenfile(self): dir, base = self.defaultfilename("open") if not self.opendialog: - self.opendialog = tkFileDialog.Open(parent=self.text, + self.opendialog = filedialog.Open(parent=self.text, filetypes=self.filetypes) filename = self.opendialog.show(initialdir=dir, initialfile=base) return filename @@ -378,7 +378,7 @@ def defaultfilename(self, mode="open"): def asksavefile(self): dir, base = self.defaultfilename("save") if not self.savedialog: - self.savedialog = tkFileDialog.SaveAs( + self.savedialog = filedialog.SaveAs( parent=self.text, filetypes=self.filetypes, defaultextension=self.defaultextension) diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index abe8a85952bc9b..0407ca9cfd8bfd 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -21,13 +21,13 @@ except (ImportError, AttributeError, OSError): pass -import tkinter.messagebox as tkMessageBox +from tkinter import messagebox if TkVersion < 8.5: root = Tk() # otherwise create root in main root.withdraw() from idlelib.run import fix_scaling fix_scaling(root) - tkMessageBox.showerror("Idle Cannot Start", + messagebox.showerror("Idle Cannot Start", "Idle requires tcl/tk 8.5+, not %s." % TkVersion, parent=root) raise SystemExit(1) @@ -261,7 +261,7 @@ def store_file_breaks(self): except OSError as err: if not getattr(self.root, "breakpoint_error_displayed", False): self.root.breakpoint_error_displayed = True - tkMessageBox.showerror(title='IDLE Error', + messagebox.showerror(title='IDLE Error', message='Unable to update breakpoint list:\n%s' % str(err), parent=self.text) @@ -771,7 +771,7 @@ def runcode(self, code): exec(code, self.locals) except SystemExit: if not self.tkconsole.closing: - if tkMessageBox.askyesno( + if messagebox.askyesno( "Exit?", "Do you want to exit altogether?", default="yes", @@ -805,7 +805,7 @@ def write(self, s): return self.tkconsole.stderr.write(s) def display_port_binding_error(self): - tkMessageBox.showerror( + messagebox.showerror( "Port Binding Error", "IDLE can't bind to a TCP/IP port, which is necessary to " "communicate with its Python execution server. This might be " @@ -816,7 +816,7 @@ def display_port_binding_error(self): parent=self.tkconsole.text) def display_no_subprocess_error(self): - tkMessageBox.showerror( + messagebox.showerror( "Subprocess Connection Error", "IDLE's subprocess didn't make connection.\n" "See the 'Startup failure' section of the IDLE doc, online at\n" @@ -824,7 +824,7 @@ def display_no_subprocess_error(self): parent=self.tkconsole.text) def display_executing_dialog(self): - tkMessageBox.showerror( + messagebox.showerror( "Already executing", "The Python Shell window is already executing a command; " "please wait until it is finished.", @@ -945,7 +945,7 @@ def get_warning_stream(self): def toggle_debugger(self, event=None): if self.executing: - tkMessageBox.showerror("Don't debug now", + messagebox.showerror("Don't debug now", "You can only toggle the debugger when idle", parent=self.text) self.set_debugger_indicator() @@ -1003,7 +1003,7 @@ def endexecuting(self): def close(self): "Extend EditorWindow.close()" if self.executing: - response = tkMessageBox.askokcancel( + response = messagebox.askokcancel( "Kill?", "Your program is still running!\n Do you want to kill it?", default="ok", @@ -1254,7 +1254,7 @@ def open_stack_viewer(self, event=None): try: sys.last_traceback except: - tkMessageBox.showerror("No stack trace", + messagebox.showerror("No stack trace", "There is no stack trace yet.\n" "(sys.last_traceback is not defined)", parent=self.text) diff --git a/Lib/idlelib/rpc.py b/Lib/idlelib/rpc.py index aa8cbd36c47926..8efcf048fa3aa2 100644 --- a/Lib/idlelib/rpc.py +++ b/Lib/idlelib/rpc.py @@ -125,7 +125,7 @@ def handle_error(self, request, client_address): response_queue = queue.Queue(0) -class SocketIO(object): +class SocketIO: nextseq = 0 @@ -486,7 +486,7 @@ def EOFhook(self): #----------------- end class SocketIO -------------------- -class RemoteObject(object): +class RemoteObject: # Token mix-in class pass @@ -497,7 +497,7 @@ def remoteref(obj): return RemoteProxy(oid) -class RemoteProxy(object): +class RemoteProxy: def __init__(self, oid): self.oid = oid @@ -547,7 +547,7 @@ def get_remote_proxy(self, oid): return RPCProxy(self, oid) -class RPCProxy(object): +class RPCProxy: __methods = None __attributes = None @@ -596,7 +596,7 @@ def _getattributes(obj, attributes): attributes[name] = 1 -class MethodProxy(object): +class MethodProxy: def __init__(self, sockio, oid, name): self.sockio = sockio diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 1e84ecc6584ef1..07e9a2bf9ceeae 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -16,6 +16,7 @@ import threading import warnings +import idlelib # testing from idlelib import autocomplete # AutoComplete, fetch_encodings from idlelib import calltip # Calltip from idlelib import debugger_r # start_debugger @@ -538,18 +539,21 @@ def decode_interrupthook(self): thread.interrupt_main() -class Executive(object): +class Executive: def __init__(self, rpchandler): self.rpchandler = rpchandler - self.locals = __main__.__dict__ - self.calltip = calltip.Calltip() - self.autocomplete = autocomplete.AutoComplete() + if idlelib.testing is False: + self.locals = __main__.__dict__ + self.calltip = calltip.Calltip() + self.autocomplete = autocomplete.AutoComplete() + else: + self.locals = {} def runcode(self, code): global interruptable try: - self.usr_exc_info = None + self.user_exc_info = None interruptable = True try: exec(code, self.locals) @@ -562,10 +566,17 @@ def runcode(self, code): print('SystemExit: ' + str(ob), file=sys.stderr) # Return to the interactive prompt. except: - self.usr_exc_info = sys.exc_info() + self.user_exc_info = sys.exc_info() # For testing, hook, viewer. if quitting: exit() - print_exception() + if sys.excepthook is sys.__excepthook__: + print_exception() + else: + try: + sys.excepthook(*self.user_exc_info) + except: + self.user_exc_info = sys.exc_info() # For testing. + print_exception() jit = self.rpchandler.console.getvar("<>") if jit: self.rpchandler.interp.open_remote_stack_viewer() @@ -590,8 +601,8 @@ def get_the_completion_list(self, what, mode): return self.autocomplete.fetch_completions(what, mode) def stackviewer(self, flist_oid=None): - if self.usr_exc_info: - typ, val, tb = self.usr_exc_info + if self.user_exc_info: + typ, val, tb = self.user_exc_info else: return None flist = None diff --git a/Lib/idlelib/runscript.py b/Lib/idlelib/runscript.py index 028b0dbd21dfe6..55712e904603f8 100644 --- a/Lib/idlelib/runscript.py +++ b/Lib/idlelib/runscript.py @@ -14,7 +14,7 @@ import time import tokenize -import tkinter.messagebox as tkMessageBox +from tkinter import messagebox from idlelib.config import idleConf from idlelib import macosx @@ -195,15 +195,15 @@ def getfilename(self): def ask_save_dialog(self): msg = "Source Must Be Saved\n" + 5*' ' + "OK to Save?" - confirm = tkMessageBox.askokcancel(title="Save Before Run or Check", + confirm = messagebox.askokcancel(title="Save Before Run or Check", message=msg, - default=tkMessageBox.OK, + default=messagebox.OK, parent=self.editwin.text) return confirm def errorbox(self, title, message): # XXX This should really be a function of EditorWindow... - tkMessageBox.showerror(title, message, parent=self.editwin.text) + messagebox.showerror(title, message, parent=self.editwin.text) self.editwin.text.focus_set() self.perf = time.perf_counter() diff --git a/Lib/idlelib/searchengine.py b/Lib/idlelib/searchengine.py index a50038e282ba6c..eddef581ab40a7 100644 --- a/Lib/idlelib/searchengine.py +++ b/Lib/idlelib/searchengine.py @@ -2,7 +2,7 @@ import re from tkinter import StringVar, BooleanVar, TclError -import tkinter.messagebox as tkMessageBox +from tkinter import messagebox def get(root): '''Return the singleton SearchEngine instance for the process. @@ -96,7 +96,7 @@ def report_error(self, pat, msg, col=None): msg = msg + "\nPattern: " + str(pat) if col is not None: msg = msg + "\nOffset: " + str(col) - tkMessageBox.showerror("Regular expression error", + messagebox.showerror("Regular expression error", msg, master=self.root) def search_text(self, text, prog=None, ok=0): diff --git a/Lib/idlelib/squeezer.py b/Lib/idlelib/squeezer.py index be1538a25fdedf..3046d803b74a4e 100644 --- a/Lib/idlelib/squeezer.py +++ b/Lib/idlelib/squeezer.py @@ -17,7 +17,7 @@ import re import tkinter as tk -import tkinter.messagebox as tkMessageBox +from tkinter import messagebox from idlelib.config import idleConf from idlelib.textview import view_text @@ -147,7 +147,7 @@ def expand(self, event=None): if self.is_dangerous is None: self.set_is_dangerous() if self.is_dangerous: - confirm = tkMessageBox.askokcancel( + confirm = messagebox.askokcancel( title="Expand huge output?", message="\n\n".join([ "The squeezed output is very long: %d lines, %d chars.", @@ -155,7 +155,7 @@ def expand(self, event=None): "It is recommended to view or copy the output instead.", "Really expand?" ]) % (self.numoflines, len(self.s)), - default=tkMessageBox.CANCEL, + default=messagebox.CANCEL, parent=self.text) if not confirm: return "break" diff --git a/Lib/idlelib/tooltip.py b/Lib/idlelib/tooltip.py index 69658264dbd4a4..d714318dae8ef1 100644 --- a/Lib/idlelib/tooltip.py +++ b/Lib/idlelib/tooltip.py @@ -7,7 +7,7 @@ from tkinter import * -class TooltipBase(object): +class TooltipBase: """abstract base class for tooltips""" def __init__(self, anchor_widget): diff --git a/Lib/pdb.py b/Lib/pdb.py index d7d957159458be..7a5192cbadc3ad 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1686,8 +1686,9 @@ def main(): sys.argv[:] = args # Hide "pdb.py" and pdb options from argument list - # Replace pdb's dir with script's dir in front of module search path. if not run_as_module: + mainpyfile = os.path.realpath(mainpyfile) + # Replace pdb's dir with script's dir in front of module search path. sys.path[0] = os.path.dirname(mainpyfile) # Note on saving/restoring sys.argv: it's a good idea when sys.argv was diff --git a/Lib/profile.py b/Lib/profile.py index 5cb017ed830099..d8599fb4eebd66 100755 --- a/Lib/profile.py +++ b/Lib/profile.py @@ -595,7 +595,12 @@ def main(): '__package__': None, '__cached__': None, } - runctx(code, globs, None, options.outfile, options.sort) + try: + runctx(code, globs, None, options.outfile, options.sort) + except BrokenPipeError as exc: + # Prevent "Exception ignored" during interpreter shutdown. + sys.stdout = None + sys.exit(exc.errno) else: parser.print_usage() return parser diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 628f9fc7d1d1ef..282a9179983407 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -504,7 +504,7 @@ def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): not file.startswith(os.path.join(basedir, 'site-packages')))) and object.__name__ not in ('xml.etree', 'test.pydoc_mod')): if docloc.startswith(("http://", "https://")): - docloc = "%s/%s" % (docloc.rstrip("/"), object.__name__.lower()) + docloc = "{}/{}.html".format(docloc.rstrip("/"), object.__name__.lower()) else: docloc = os.path.join(docloc, object.__name__.lower() + ".html") else: diff --git a/Lib/random.py b/Lib/random.py index 4142e2e860c8c5..187b0a016947a8 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -78,6 +78,7 @@ "lognormvariate", "normalvariate", "paretovariate", + "randbytes", "randint", "random", "randrange", @@ -302,17 +303,15 @@ def randrange(self, start, stop=None, step=_ONE): try: istart = _index(start) except TypeError: - if int(start) == start: - istart = int(start) - _warn('Float arguments to randrange() have been deprecated\n' - 'since Python 3.10 and will be removed in a subsequent ' - 'version.', - DeprecationWarning, 2) - else: + istart = int(start) + if istart != start: _warn('randrange() will raise TypeError in the future', DeprecationWarning, 2) raise ValueError("non-integer arg 1 for randrange()") - + _warn('non-integer arguments to randrange() have been deprecated ' + 'since Python 3.10 and will be removed in a subsequent ' + 'version', + DeprecationWarning, 2) if stop is None: # We don't check for "step != 1" because it hasn't been # type checked and converted to an integer yet. @@ -326,31 +325,29 @@ def randrange(self, start, stop=None, step=_ONE): try: istop = _index(stop) except TypeError: - if int(stop) == stop: - istop = int(stop) - _warn('Float arguments to randrange() have been deprecated\n' - 'since Python 3.10 and will be removed in a subsequent ' - 'version.', - DeprecationWarning, 2) - else: + istop = int(stop) + if istop != stop: _warn('randrange() will raise TypeError in the future', DeprecationWarning, 2) raise ValueError("non-integer stop for randrange()") - + _warn('non-integer arguments to randrange() have been deprecated ' + 'since Python 3.10 and will be removed in a subsequent ' + 'version', + DeprecationWarning, 2) + width = istop - istart try: istep = _index(step) except TypeError: - if int(step) == step: - istep = int(step) - _warn('Float arguments to randrange() have been deprecated\n' - 'since Python 3.10 and will be removed in a subsequent ' - 'version.', - DeprecationWarning, 2) - else: + istep = int(step) + if istep != step: _warn('randrange() will raise TypeError in the future', DeprecationWarning, 2) raise ValueError("non-integer step for randrange()") - width = istop - istart + _warn('non-integer arguments to randrange() have been deprecated ' + 'since Python 3.10 and will be removed in a subsequent ' + 'version', + DeprecationWarning, 2) + # Fast path. if istep == 1: if width > 0: return istart + self._randbelow(width) @@ -478,7 +475,7 @@ def sample(self, population, k, *, counts=None): raise TypeError('Counts must be integers') if total <= 0: raise ValueError('Total of counts must be greater than zero') - selections = sample(range(total), k=k) + selections = self.sample(range(total), k=k) bisect = _bisect return [population[bisect(cum_counts, s)] for s in selections] randbelow = self._randbelow diff --git a/Lib/re.py b/Lib/re.py index bfb7b1ccd93466..a39ff047c26b22 100644 --- a/Lib/re.py +++ b/Lib/re.py @@ -142,7 +142,7 @@ __version__ = "2.2.1" -class RegexFlag(enum.IntFlag): +class RegexFlag(enum.IntFlag, boundary=enum.KEEP): ASCII = A = sre_compile.SRE_FLAG_ASCII # assume ascii "locale" IGNORECASE = I = sre_compile.SRE_FLAG_IGNORECASE # ignore case LOCALE = L = sre_compile.SRE_FLAG_LOCALE # assume current 8-bit locale @@ -155,26 +155,17 @@ class RegexFlag(enum.IntFlag): DEBUG = sre_compile.SRE_FLAG_DEBUG # dump pattern after compilation def __repr__(self): - if self._name_ is not None: - return f're.{self._name_}' - value = self._value_ - members = [] - negative = value < 0 - if negative: - value = ~value - for m in self.__class__: - if value & m._value_: - value &= ~m._value_ - members.append(f're.{m._name_}') - if value: - members.append(hex(value)) - res = '|'.join(members) - if negative: - if len(members) > 1: - res = f'~({res})' - else: - res = f'~{res}' + res = '' + if self._name_: + member_names = self._name_.split('|') + constant = None + if member_names[-1].startswith('0x'): + constant = member_names.pop() + res = 're.' + '|re.'.join(member_names) + if constant: + res += '|%s' % constant return res + __str__ = object.__str__ globals().update(RegexFlag.__members__) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index a4ebe4a0a1b5cb..1b18bfad553007 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -2,6 +2,8 @@ # these are all functions _testcapi exports whose name begins with 'test_'. from collections import OrderedDict +import importlib.machinery +import importlib.util import os import pickle import random @@ -13,8 +15,6 @@ import time import unittest import weakref -import importlib.machinery -import importlib.util from test import support from test.support import MISSING_C_DOCSTRINGS from test.support import import_helper @@ -35,6 +35,10 @@ Py_DEBUG = hasattr(sys, 'gettotalrefcount') +def decode_stderr(err): + return err.decode('utf-8', 'replace').replace('\r', '') + + def testfunction(self): """some doc""" return self @@ -207,23 +211,22 @@ def test_return_null_without_error(self): _testcapi.return_null_without_error() """) rc, out, err = assert_python_failure('-c', code) - self.assertRegex(err.replace(b'\r', b''), - br'Fatal Python error: _Py_CheckFunctionResult: ' - br'a function returned NULL ' - br'without setting an error\n' - br'Python runtime state: initialized\n' - br'SystemError: returned NULL ' - br'without setting an error\n' - br'\n' - br'Current thread.*:\n' - br' File .*", line 6 in ') + err = decode_stderr(err) + self.assertRegex(err, + r'Fatal Python error: _Py_CheckFunctionResult: ' + r'a function returned NULL without setting an exception\n' + r'Python runtime state: initialized\n' + r'SystemError: ' + r'returned NULL without setting an exception\n' + r'\n' + r'Current thread.*:\n' + r' File .*", line 6 in \n') else: with self.assertRaises(SystemError) as cm: _testcapi.return_null_without_error() self.assertRegex(str(cm.exception), 'return_null_without_error.* ' - 'returned NULL without setting an error') + 'returned NULL without setting an exception') def test_return_result_with_error(self): # Issue #23571: A function must not return a result with an error set @@ -236,28 +239,58 @@ def test_return_result_with_error(self): _testcapi.return_result_with_error() """) rc, out, err = assert_python_failure('-c', code) - self.assertRegex(err.replace(b'\r', b''), - br'Fatal Python error: _Py_CheckFunctionResult: ' - br'a function returned a result ' - br'with an error set\n' - br'Python runtime state: initialized\n' - br'ValueError\n' - br'\n' - br'The above exception was the direct cause ' - br'of the following exception:\n' - br'\n' - br'SystemError: ' - br'returned a result with an error set\n' - br'\n' - br'Current thread.*:\n' - br' File .*, line 6 in ') + err = decode_stderr(err) + self.assertRegex(err, + r'Fatal Python error: _Py_CheckFunctionResult: ' + r'a function returned a result with an exception set\n' + r'Python runtime state: initialized\n' + r'ValueError\n' + r'\n' + r'The above exception was the direct cause ' + r'of the following exception:\n' + r'\n' + r'SystemError: ' + r'returned a result with an exception set\n' + r'\n' + r'Current thread.*:\n' + r' File .*, line 6 in \n') else: with self.assertRaises(SystemError) as cm: _testcapi.return_result_with_error() self.assertRegex(str(cm.exception), 'return_result_with_error.* ' - 'returned a result with an error set') + 'returned a result with an exception set') + + def test_getitem_with_error(self): + # Test _Py_CheckSlotResult(). Raise an exception and then calls + # PyObject_GetItem(): check that the assertion catchs the bug. + # PyObject_GetItem() must not be called with an exception set. + code = textwrap.dedent(""" + import _testcapi + from test import support + + with support.SuppressCrashReport(): + _testcapi.getitem_with_error({1: 2}, 1) + """) + rc, out, err = assert_python_failure('-c', code) + err = decode_stderr(err) + if 'SystemError: ' not in err: + self.assertRegex(err, + r'Fatal Python error: _Py_CheckSlotResult: ' + r'Slot __getitem__ of type dict succeeded ' + r'with an exception set\n' + r'Python runtime state: initialized\n' + r'ValueError: bug\n' + r'\n' + r'Current thread .* \(most recent call first\):\n' + r' File .*, line 6 in \n' + r'\n' + r'Extension modules: _testcapi \(total: 1\)\n') + else: + # Python built with NDEBUG macro defined: + # test _Py_CheckFunctionResult() instead. + self.assertIn('returned a result with an exception set', err) def test_buildvalue_N(self): _testcapi.test_buildvalue_N() @@ -547,6 +580,45 @@ def test_pynumber_tobase(self): self.assertRaises(TypeError, pynumber_tobase, '123', 10) self.assertRaises(SystemError, pynumber_tobase, 123, 0) + def check_fatal_error(self, code, expected, not_expected=()): + with support.SuppressCrashReport(): + rc, out, err = assert_python_failure('-sSI', '-c', code) + + err = decode_stderr(err) + self.assertIn('Fatal Python error: test_fatal_error: MESSAGE\n', + err) + + match = re.search(r'^Extension modules:(.*) \(total: ([0-9]+)\)$', + err, re.MULTILINE) + if not match: + self.fail(f"Cannot find 'Extension modules:' in {err!r}") + modules = set(match.group(1).strip().split(', ')) + total = int(match.group(2)) + + for name in expected: + self.assertIn(name, modules) + for name in not_expected: + self.assertNotIn(name, modules) + self.assertEqual(len(modules), total) + + def test_fatal_error(self): + # By default, stdlib extension modules are ignored, + # but not test modules. + expected = ('_testcapi',) + not_expected = ('sys',) + code = 'import _testcapi, sys; _testcapi.fatal_error(b"MESSAGE")' + self.check_fatal_error(code, expected, not_expected) + + # Mark _testcapi as stdlib module, but not sys + expected = ('sys',) + not_expected = ('_testcapi',) + code = textwrap.dedent(''' + import _testcapi, sys + sys.stdlib_module_names = frozenset({"_testcapi"}) + _testcapi.fatal_error(b"MESSAGE") + ''') + self.check_fatal_error(code, expected) + class TestPendingCalls(unittest.TestCase): diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index 45d0a7de9d9253..1da6ca55c48f72 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -160,7 +160,6 @@ def test_incomplete(self): ai("","eval") ai("\n","eval") ai("(","eval") - ai("(\n\n\n","eval") ai("(9+","eval") ai("9+ \\","eval") ai("lambda z: \\","eval") diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index 4786d28f39a3d0..bfca0cd7fbe483 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -1,19 +1,68 @@ -import unittest -import unittest.mock -import test.support +import contextlib import os import os.path -import contextlib import sys +import tempfile +import test.support +import unittest +import unittest.mock import ensurepip import ensurepip._uninstall -class TestEnsurePipVersion(unittest.TestCase): +class TestPackages(unittest.TestCase): + def touch(self, directory, filename): + fullname = os.path.join(directory, filename) + open(fullname, "wb").close() + + def test_version(self): + # Test version() + with tempfile.TemporaryDirectory() as tmpdir: + self.touch(tmpdir, "pip-1.2.3b1-py2.py3-none-any.whl") + self.touch(tmpdir, "setuptools-49.1.3-py3-none-any.whl") + with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), + unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): + self.assertEqual(ensurepip.version(), '1.2.3b1') + + def test_get_packages_no_dir(self): + # Test _get_packages() without a wheel package directory + with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), + unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None)): + packages = ensurepip._get_packages() + + # when bundled wheel packages are used, we get _PIP_VERSION + self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version()) + + # use bundled wheel packages + self.assertIsNotNone(packages['pip'].wheel_name) + self.assertIsNotNone(packages['setuptools'].wheel_name) + + def test_get_packages_with_dir(self): + # Test _get_packages() with a wheel package directory + setuptools_filename = "setuptools-49.1.3-py3-none-any.whl" + pip_filename = "pip-20.2.2-py2.py3-none-any.whl" + + with tempfile.TemporaryDirectory() as tmpdir: + self.touch(tmpdir, setuptools_filename) + self.touch(tmpdir, pip_filename) + # not used, make sure that it's ignored + self.touch(tmpdir, "wheel-0.34.2-py2.py3-none-any.whl") + + with (unittest.mock.patch.object(ensurepip, '_PACKAGES', None), + unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', tmpdir)): + packages = ensurepip._get_packages() + + self.assertEqual(packages['setuptools'].version, '49.1.3') + self.assertEqual(packages['setuptools'].wheel_path, + os.path.join(tmpdir, setuptools_filename)) + self.assertEqual(packages['pip'].version, '20.2.2') + self.assertEqual(packages['pip'].wheel_path, + os.path.join(tmpdir, pip_filename)) + + # wheel package is ignored + self.assertEqual(sorted(packages), ['pip', 'setuptools']) - def test_returns_version(self): - self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version()) class EnsurepipMixin: @@ -27,6 +76,8 @@ def setUp(self): real_devnull = os.devnull os_patch = unittest.mock.patch("ensurepip.os") patched_os = os_patch.start() + # But expose os.listdir() used by _find_packages() + patched_os.listdir = os.listdir self.addCleanup(os_patch.stop) patched_os.devnull = real_devnull patched_os.path = os.path @@ -147,7 +198,7 @@ def test_pip_config_file_disabled(self): self.assertEqual(self.os_environ["PIP_CONFIG_FILE"], os.devnull) @contextlib.contextmanager -def fake_pip(version=ensurepip._PIP_VERSION): +def fake_pip(version=ensurepip.version()): if version is None: pip = None else: @@ -243,7 +294,7 @@ def test_pip_config_file_disabled(self): # Basic testing of the main functions and their argument parsing -EXPECTED_VERSION_OUTPUT = "pip " + ensurepip._PIP_VERSION +EXPECTED_VERSION_OUTPUT = "pip " + ensurepip.version() class TestBootstrappingMainFunction(EnsurepipMixin, unittest.TestCase): diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 3ea623e9c883c0..96de878faf72d4 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -1,11 +1,14 @@ import enum +import doctest import inspect +import os import pydoc import sys import unittest import threading from collections import OrderedDict from enum import Enum, IntEnum, StrEnum, EnumMeta, Flag, IntFlag, unique, auto +from enum import STRICT, CONFORM, EJECT, KEEP from io import StringIO from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL from test import support @@ -13,6 +16,14 @@ from test.support import threading_helper from datetime import timedelta +def load_tests(loader, tests, ignore): + tests.addTests(doctest.DocTestSuite(enum)) + if os.path.exists('../../Doc/library/enum.rst'): + tests.addTests(doctest.DocFileSuite( + '../../Doc/library/enum.rst', + optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE, + )) + return tests # for pickle tests try: @@ -2126,7 +2137,30 @@ class ThirdFailedStrEnum(StrEnum): one = '1' two = b'2', 'ascii', 9 - + def test_missing_value_error(self): + with self.assertRaisesRegex(TypeError, "_value_ not set in __new__"): + class Combined(str, Enum): + # + def __new__(cls, value, sequence): + enum = str.__new__(cls, value) + if '(' in value: + fis_name, segment = value.split('(', 1) + segment = segment.strip(' )') + else: + fis_name = value + segment = None + enum.fis_name = fis_name + enum.segment = segment + enum.sequence = sequence + return enum + # + def __repr__(self): + return "<%s.%s>" % (self.__class__.__name__, self._name_) + # + key_type = 'An$(1,2)', 0 + company_id = 'An$(3,2)', 1 + code = 'An$(5,1)', 2 + description = 'Bn$', 3 @unittest.skipUnless( sys.version_info[:2] == (3, 9), @@ -2264,9 +2298,12 @@ class Open(Flag): class Color(Flag): BLACK = 0 RED = 1 + ROJO = 1 GREEN = 2 BLUE = 4 PURPLE = RED|BLUE + WHITE = RED|GREEN|BLUE + BLANCO = RED|GREEN|BLUE def test_str(self): Perm = self.Perm @@ -2275,12 +2312,12 @@ def test_str(self): self.assertEqual(str(Perm.X), 'Perm.X') self.assertEqual(str(Perm.R | Perm.W), 'Perm.R|W') self.assertEqual(str(Perm.R | Perm.W | Perm.X), 'Perm.R|W|X') - self.assertEqual(str(Perm(0)), 'Perm.0') + self.assertEqual(str(Perm(0)), 'Perm(0)') self.assertEqual(str(~Perm.R), 'Perm.W|X') self.assertEqual(str(~Perm.W), 'Perm.R|X') self.assertEqual(str(~Perm.X), 'Perm.R|W') self.assertEqual(str(~(Perm.R | Perm.W)), 'Perm.X') - self.assertEqual(str(~(Perm.R | Perm.W | Perm.X)), 'Perm.0') + self.assertEqual(str(~(Perm.R | Perm.W | Perm.X)), 'Perm(0)') self.assertEqual(str(Perm(~0)), 'Perm.R|W|X') Open = self.Open @@ -2288,10 +2325,11 @@ def test_str(self): self.assertEqual(str(Open.WO), 'Open.WO') self.assertEqual(str(Open.AC), 'Open.AC') self.assertEqual(str(Open.RO | Open.CE), 'Open.CE') - self.assertEqual(str(Open.WO | Open.CE), 'Open.CE|WO') - self.assertEqual(str(~Open.RO), 'Open.CE|AC|RW|WO') - self.assertEqual(str(~Open.WO), 'Open.CE|RW') + self.assertEqual(str(Open.WO | Open.CE), 'Open.WO|CE') + self.assertEqual(str(~Open.RO), 'Open.WO|RW|CE') + self.assertEqual(str(~Open.WO), 'Open.RW|CE') self.assertEqual(str(~Open.AC), 'Open.CE') + self.assertEqual(str(~Open.CE), 'Open.AC') self.assertEqual(str(~(Open.RO | Open.CE)), 'Open.AC') self.assertEqual(str(~(Open.WO | Open.CE)), 'Open.RW') @@ -2302,12 +2340,12 @@ def test_repr(self): self.assertEqual(repr(Perm.X), '') self.assertEqual(repr(Perm.R | Perm.W), '') self.assertEqual(repr(Perm.R | Perm.W | Perm.X), '') - self.assertEqual(repr(Perm(0)), '') + self.assertEqual(repr(Perm(0)), '') self.assertEqual(repr(~Perm.R), '') self.assertEqual(repr(~Perm.W), '') self.assertEqual(repr(~Perm.X), '') self.assertEqual(repr(~(Perm.R | Perm.W)), '') - self.assertEqual(repr(~(Perm.R | Perm.W | Perm.X)), '') + self.assertEqual(repr(~(Perm.R | Perm.W | Perm.X)), '') self.assertEqual(repr(Perm(~0)), '') Open = self.Open @@ -2315,10 +2353,11 @@ def test_repr(self): self.assertEqual(repr(Open.WO), '') self.assertEqual(repr(Open.AC), '') self.assertEqual(repr(Open.RO | Open.CE), '') - self.assertEqual(repr(Open.WO | Open.CE), '') - self.assertEqual(repr(~Open.RO), '') - self.assertEqual(repr(~Open.WO), '') + self.assertEqual(repr(Open.WO | Open.CE), '') + self.assertEqual(repr(~Open.RO), '') + self.assertEqual(repr(~Open.WO), '') self.assertEqual(repr(~Open.AC), '') + self.assertEqual(repr(~Open.CE), '') self.assertEqual(repr(~(Open.RO | Open.CE)), '') self.assertEqual(repr(~(Open.WO | Open.CE)), '') @@ -2394,6 +2433,46 @@ def test_bool(self): for f in Open: self.assertEqual(bool(f.value), bool(f)) + def test_boundary(self): + self.assertIs(enum.Flag._boundary_, STRICT) + class Iron(Flag, boundary=STRICT): + ONE = 1 + TWO = 2 + EIGHT = 8 + self.assertIs(Iron._boundary_, STRICT) + # + class Water(Flag, boundary=CONFORM): + ONE = 1 + TWO = 2 + EIGHT = 8 + self.assertIs(Water._boundary_, CONFORM) + # + class Space(Flag, boundary=EJECT): + ONE = 1 + TWO = 2 + EIGHT = 8 + self.assertIs(Space._boundary_, EJECT) + # + class Bizarre(Flag, boundary=KEEP): + b = 3 + c = 4 + d = 6 + # + self.assertRaisesRegex(ValueError, 'invalid value: 7', Iron, 7) + self.assertIs(Water(7), Water.ONE|Water.TWO) + self.assertIs(Water(~9), Water.TWO) + self.assertEqual(Space(7), 7) + self.assertTrue(type(Space(7)) is int) + self.assertEqual(list(Bizarre), [Bizarre.c]) + self.assertIs(Bizarre(3), Bizarre.b) + self.assertIs(Bizarre(6), Bizarre.d) + + def test_iter(self): + Color = self.Color + Open = self.Open + self.assertEqual(list(Color), [Color.RED, Color.GREEN, Color.BLUE]) + self.assertEqual(list(Open), [Open.WO, Open.RW, Open.CE]) + def test_programatic_function_string(self): Perm = Flag('Perm', 'R W X') lst = list(Perm) @@ -2511,9 +2590,45 @@ def test_member_contains(self): def test_member_iter(self): Color = self.Color - self.assertEqual(list(Color.PURPLE), [Color.BLUE, Color.RED]) + self.assertEqual(list(Color.BLACK), []) + self.assertEqual(list(Color.PURPLE), [Color.RED, Color.BLUE]) self.assertEqual(list(Color.BLUE), [Color.BLUE]) self.assertEqual(list(Color.GREEN), [Color.GREEN]) + self.assertEqual(list(Color.WHITE), [Color.RED, Color.GREEN, Color.BLUE]) + self.assertEqual(list(Color.WHITE), [Color.RED, Color.GREEN, Color.BLUE]) + + def test_member_length(self): + self.assertEqual(self.Color.__len__(self.Color.BLACK), 0) + self.assertEqual(self.Color.__len__(self.Color.GREEN), 1) + self.assertEqual(self.Color.__len__(self.Color.PURPLE), 2) + self.assertEqual(self.Color.__len__(self.Color.BLANCO), 3) + + def test_number_reset_and_order_cleanup(self): + class Confused(Flag): + _order_ = 'ONE TWO FOUR DOS EIGHT SIXTEEN' + ONE = auto() + TWO = auto() + FOUR = auto() + DOS = 2 + EIGHT = auto() + SIXTEEN = auto() + self.assertEqual( + list(Confused), + [Confused.ONE, Confused.TWO, Confused.FOUR, Confused.EIGHT, Confused.SIXTEEN]) + self.assertIs(Confused.TWO, Confused.DOS) + self.assertEqual(Confused.DOS._value_, 2) + self.assertEqual(Confused.EIGHT._value_, 8) + self.assertEqual(Confused.SIXTEEN._value_, 16) + + def test_aliases(self): + Color = self.Color + self.assertEqual(Color(1).name, 'RED') + self.assertEqual(Color['ROJO'].name, 'RED') + self.assertEqual(Color(7).name, 'WHITE') + self.assertEqual(Color['BLANCO'].name, 'WHITE') + self.assertIs(Color.BLANCO, Color.WHITE) + Open = self.Open + self.assertIs(Open['AC'], Open.AC) def test_auto_number(self): class Color(Flag): @@ -2532,20 +2647,6 @@ class Color(Flag): red = 'not an int' blue = auto() - def test_cascading_failure(self): - class Bizarre(Flag): - c = 3 - d = 4 - f = 6 - # Bizarre.c | Bizarre.d - name = "TestFlag.test_cascading_failure..Bizarre" - self.assertRaisesRegex(ValueError, "5 is not a valid " + name, Bizarre, 5) - self.assertRaisesRegex(ValueError, "5 is not a valid " + name, Bizarre, 5) - self.assertRaisesRegex(ValueError, "2 is not a valid " + name, Bizarre, 2) - self.assertRaisesRegex(ValueError, "2 is not a valid " + name, Bizarre, 2) - self.assertRaisesRegex(ValueError, "1 is not a valid " + name, Bizarre, 1) - self.assertRaisesRegex(ValueError, "1 is not a valid " + name, Bizarre, 1) - def test_duplicate_auto(self): class Dupes(Enum): first = primero = auto() @@ -2554,11 +2655,11 @@ class Dupes(Enum): self.assertEqual([Dupes.first, Dupes.second, Dupes.third], list(Dupes)) def test_bizarre(self): - class Bizarre(Flag): - b = 3 - c = 4 - d = 6 - self.assertEqual(repr(Bizarre(7)), '') + with self.assertRaisesRegex(TypeError, "invalid Flag 'Bizarre' -- missing values: 1, 2"): + class Bizarre(Flag): + b = 3 + c = 4 + d = 6 def test_multiple_mixin(self): class AllMixin: @@ -2682,9 +2783,9 @@ class TestIntFlag(unittest.TestCase): """Tests of the IntFlags.""" class Perm(IntFlag): - X = 1 << 0 - W = 1 << 1 R = 1 << 2 + W = 1 << 1 + X = 1 << 0 class Open(IntFlag): RO = 0 @@ -2696,9 +2797,17 @@ class Open(IntFlag): class Color(IntFlag): BLACK = 0 RED = 1 + ROJO = 1 GREEN = 2 BLUE = 4 PURPLE = RED|BLUE + WHITE = RED|GREEN|BLUE + BLANCO = RED|GREEN|BLUE + + class Skip(IntFlag): + FIRST = 1 + SECOND = 2 + EIGHTH = 8 def test_type(self): Perm = self.Perm @@ -2723,31 +2832,35 @@ def test_str(self): self.assertEqual(str(Perm.X), 'Perm.X') self.assertEqual(str(Perm.R | Perm.W), 'Perm.R|W') self.assertEqual(str(Perm.R | Perm.W | Perm.X), 'Perm.R|W|X') - self.assertEqual(str(Perm.R | 8), 'Perm.8|R') - self.assertEqual(str(Perm(0)), 'Perm.0') - self.assertEqual(str(Perm(8)), 'Perm.8') + self.assertEqual(str(Perm.R | 8), '12') + self.assertEqual(str(Perm(0)), 'Perm(0)') + self.assertEqual(str(Perm(8)), '8') self.assertEqual(str(~Perm.R), 'Perm.W|X') self.assertEqual(str(~Perm.W), 'Perm.R|X') self.assertEqual(str(~Perm.X), 'Perm.R|W') self.assertEqual(str(~(Perm.R | Perm.W)), 'Perm.X') - self.assertEqual(str(~(Perm.R | Perm.W | Perm.X)), 'Perm.-8') - self.assertEqual(str(~(Perm.R | 8)), 'Perm.W|X') + self.assertEqual(str(~(Perm.R | Perm.W | Perm.X)), 'Perm(0)') + self.assertEqual(str(~(Perm.R | 8)), '-13') self.assertEqual(str(Perm(~0)), 'Perm.R|W|X') - self.assertEqual(str(Perm(~8)), 'Perm.R|W|X') + self.assertEqual(str(Perm(~8)), '-9') Open = self.Open self.assertEqual(str(Open.RO), 'Open.RO') self.assertEqual(str(Open.WO), 'Open.WO') self.assertEqual(str(Open.AC), 'Open.AC') self.assertEqual(str(Open.RO | Open.CE), 'Open.CE') - self.assertEqual(str(Open.WO | Open.CE), 'Open.CE|WO') - self.assertEqual(str(Open(4)), 'Open.4') - self.assertEqual(str(~Open.RO), 'Open.CE|AC|RW|WO') - self.assertEqual(str(~Open.WO), 'Open.CE|RW') + self.assertEqual(str(Open.WO | Open.CE), 'Open.WO|CE') + self.assertEqual(str(Open(4)), '4') + self.assertEqual(str(~Open.RO), 'Open.WO|RW|CE') + self.assertEqual(str(~Open.WO), 'Open.RW|CE') self.assertEqual(str(~Open.AC), 'Open.CE') - self.assertEqual(str(~(Open.RO | Open.CE)), 'Open.AC|RW|WO') + self.assertEqual(str(~Open.CE), 'Open.AC') + self.assertEqual(str(~(Open.RO | Open.CE)), 'Open.AC') self.assertEqual(str(~(Open.WO | Open.CE)), 'Open.RW') - self.assertEqual(str(Open(~4)), 'Open.CE|AC|RW|WO') + self.assertEqual(str(Open(~4)), '-5') + + Skip = self.Skip + self.assertEqual(str(Skip(~4)), 'Skip.FIRST|SECOND|EIGHTH') def test_repr(self): Perm = self.Perm @@ -2756,31 +2869,34 @@ def test_repr(self): self.assertEqual(repr(Perm.X), '') self.assertEqual(repr(Perm.R | Perm.W), '') self.assertEqual(repr(Perm.R | Perm.W | Perm.X), '') - self.assertEqual(repr(Perm.R | 8), '') - self.assertEqual(repr(Perm(0)), '') - self.assertEqual(repr(Perm(8)), '') - self.assertEqual(repr(~Perm.R), '') - self.assertEqual(repr(~Perm.W), '') - self.assertEqual(repr(~Perm.X), '') - self.assertEqual(repr(~(Perm.R | Perm.W)), '') - self.assertEqual(repr(~(Perm.R | Perm.W | Perm.X)), '') - self.assertEqual(repr(~(Perm.R | 8)), '') - self.assertEqual(repr(Perm(~0)), '') - self.assertEqual(repr(Perm(~8)), '') + self.assertEqual(repr(Perm.R | 8), '12') + self.assertEqual(repr(Perm(0)), '') + self.assertEqual(repr(Perm(8)), '8') + self.assertEqual(repr(~Perm.R), '') + self.assertEqual(repr(~Perm.W), '') + self.assertEqual(repr(~Perm.X), '') + self.assertEqual(repr(~(Perm.R | Perm.W)), '') + self.assertEqual(repr(~(Perm.R | Perm.W | Perm.X)), '') + self.assertEqual(repr(~(Perm.R | 8)), '-13') + self.assertEqual(repr(Perm(~0)), '') + self.assertEqual(repr(Perm(~8)), '-9') Open = self.Open self.assertEqual(repr(Open.RO), '') self.assertEqual(repr(Open.WO), '') self.assertEqual(repr(Open.AC), '') self.assertEqual(repr(Open.RO | Open.CE), '') - self.assertEqual(repr(Open.WO | Open.CE), '') - self.assertEqual(repr(Open(4)), '') - self.assertEqual(repr(~Open.RO), '') - self.assertEqual(repr(~Open.WO), '') - self.assertEqual(repr(~Open.AC), '') - self.assertEqual(repr(~(Open.RO | Open.CE)), '') - self.assertEqual(repr(~(Open.WO | Open.CE)), '') - self.assertEqual(repr(Open(~4)), '') + self.assertEqual(repr(Open.WO | Open.CE), '') + self.assertEqual(repr(Open(4)), '4') + self.assertEqual(repr(~Open.RO), '') + self.assertEqual(repr(~Open.WO), '') + self.assertEqual(repr(~Open.AC), '') + self.assertEqual(repr(~(Open.RO | Open.CE)), '') + self.assertEqual(repr(~(Open.WO | Open.CE)), '') + self.assertEqual(repr(Open(~4)), '-5') + + Skip = self.Skip + self.assertEqual(repr(Skip(~4)), '') def test_format(self): Perm = self.Perm @@ -2863,8 +2979,7 @@ def test_invert(self): RWX = Perm.R | Perm.W | Perm.X values = list(Perm) + [RW, RX, WX, RWX, Perm(0)] for i in values: - self.assertEqual(~i, ~i.value) - self.assertEqual((~i).value, ~i.value) + self.assertEqual(~i, (~i).value) self.assertIs(type(~i), Perm) self.assertEqual(~~i, i) for i in Perm: @@ -2873,6 +2988,46 @@ def test_invert(self): self.assertIs(Open.WO & ~Open.WO, Open.RO) self.assertIs((Open.WO|Open.CE) & ~Open.WO, Open.CE) + def test_boundary(self): + self.assertIs(enum.IntFlag._boundary_, EJECT) + class Iron(IntFlag, boundary=STRICT): + ONE = 1 + TWO = 2 + EIGHT = 8 + self.assertIs(Iron._boundary_, STRICT) + # + class Water(IntFlag, boundary=CONFORM): + ONE = 1 + TWO = 2 + EIGHT = 8 + self.assertIs(Water._boundary_, CONFORM) + # + class Space(IntFlag, boundary=EJECT): + ONE = 1 + TWO = 2 + EIGHT = 8 + self.assertIs(Space._boundary_, EJECT) + # + class Bizarre(IntFlag, boundary=KEEP): + b = 3 + c = 4 + d = 6 + # + self.assertRaisesRegex(ValueError, 'invalid value: 5', Iron, 5) + self.assertIs(Water(7), Water.ONE|Water.TWO) + self.assertIs(Water(~9), Water.TWO) + self.assertEqual(Space(7), 7) + self.assertTrue(type(Space(7)) is int) + self.assertEqual(list(Bizarre), [Bizarre.c]) + self.assertIs(Bizarre(3), Bizarre.b) + self.assertIs(Bizarre(6), Bizarre.d) + + def test_iter(self): + Color = self.Color + Open = self.Open + self.assertEqual(list(Color), [Color.RED, Color.GREEN, Color.BLUE]) + self.assertEqual(list(Open), [Open.WO, Open.RW, Open.CE]) + def test_programatic_function_string(self): Perm = IntFlag('Perm', 'R W X') lst = list(Perm) @@ -3014,9 +3169,27 @@ def test_member_contains(self): def test_member_iter(self): Color = self.Color - self.assertEqual(list(Color.PURPLE), [Color.BLUE, Color.RED]) + self.assertEqual(list(Color.BLACK), []) + self.assertEqual(list(Color.PURPLE), [Color.RED, Color.BLUE]) self.assertEqual(list(Color.BLUE), [Color.BLUE]) self.assertEqual(list(Color.GREEN), [Color.GREEN]) + self.assertEqual(list(Color.WHITE), [Color.RED, Color.GREEN, Color.BLUE]) + + def test_member_length(self): + self.assertEqual(self.Color.__len__(self.Color.BLACK), 0) + self.assertEqual(self.Color.__len__(self.Color.GREEN), 1) + self.assertEqual(self.Color.__len__(self.Color.PURPLE), 2) + self.assertEqual(self.Color.__len__(self.Color.BLANCO), 3) + + def test_aliases(self): + Color = self.Color + self.assertEqual(Color(1).name, 'RED') + self.assertEqual(Color['ROJO'].name, 'RED') + self.assertEqual(Color(7).name, 'WHITE') + self.assertEqual(Color['BLANCO'].name, 'WHITE') + self.assertIs(Color.BLANCO, Color.WHITE) + Open = self.Open + self.assertIs(Open['AC'], Open.AC) def test_bool(self): Perm = self.Perm @@ -3026,6 +3199,13 @@ def test_bool(self): for f in Open: self.assertEqual(bool(f.value), bool(f)) + def test_bizarre(self): + with self.assertRaisesRegex(TypeError, "invalid Flag 'Bizarre' -- missing values: 1, 2"): + class Bizarre(IntFlag): + b = 3 + c = 4 + d = 6 + def test_multiple_mixin(self): class AllMixin: @classproperty @@ -3176,7 +3356,7 @@ class Sillier(IntEnum): Help on class Color in module %s: class Color(enum.Enum) - | Color(value, names=None, *, module=None, qualname=None, type=None, start=1) + | Color(value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) |\x20\x20 | An enumeration. |\x20\x20 @@ -3328,7 +3508,7 @@ def test_inspect_classify_class_attrs(self): class MiscTestCase(unittest.TestCase): def test__all__(self): - support.check__all__(self, enum) + support.check__all__(self, enum, not_exported={'bin'}) # These are unordered here on purpose to ensure that declaration order diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index 2cf263d27463c4..b370e27161cee6 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -7,23 +7,25 @@ import unittest class EOFTestCase(unittest.TestCase): - def test_EOFC(self): - expect = "EOL while scanning string literal (, line 1)" - try: - eval("""'this is a test\ - """) - except SyntaxError as msg: - self.assertEqual(str(msg), expect) - else: - raise support.TestFailed + def test_EOF_single_quote(self): + expect = "unterminated string literal (detected at line 1) (, line 1)" + for quote in ("'", "\""): + try: + eval(f"""{quote}this is a test\ + """) + except SyntaxError as msg: + self.assertEqual(str(msg), expect) + self.assertEqual(msg.offset, 1) + else: + raise support.TestFailed def test_EOFS(self): - expect = ("EOF while scanning triple-quoted string literal " - "(, line 1)") + expect = ("unterminated triple-quoted string literal (detected at line 1) (, line 1)") try: eval("""'''this is a test""") except SyntaxError as msg: self.assertEqual(str(msg), expect) + self.assertEqual(msg.offset, 1) else: raise support.TestFailed diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 864422390ad302..21878c39f4fec9 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -206,9 +206,12 @@ def testSyntaxErrorOffset(self): check(b'# -*- coding: cp1251 -*-\nPython = "\xcf\xb3\xf2\xee\xed" +', 2, 19, encoding='cp1251') check(b'Python = "\xcf\xb3\xf2\xee\xed" +', 1, 18) - check('x = "a', 1, 7) + check('x = "a', 1, 5) check('lambda x: x = 2', 1, 1) check('f{a + b + c}', 1, 2) + check('[file for str(file) in []\n])', 1, 11) + check('[\nfile\nfor str(file)\nin\n[]\n]', 3, 5) + check('[file for\n str(file) in []]', 2, 2) # Errors thrown by compile.c check('class foo:return 1', 1, 11) @@ -235,7 +238,7 @@ def bar(): def baz(): '''quux''' - """, 9, 20) + """, 9, 24) check("pass\npass\npass\n(1+)\npass\npass\npass", 4, 4) check("(1+)", 1, 4) diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 80c1f7d90adf0f..648624482e5553 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -2,6 +2,7 @@ import datetime import faulthandler import os +import re import signal import subprocess import sys @@ -227,25 +228,23 @@ def test_sigill(self): 5, 'Illegal instruction') + def check_fatal_error_func(self, release_gil): + # Test that Py_FatalError() dumps a traceback + with support.SuppressCrashReport(): + self.check_fatal_error(f""" + import _testcapi + _testcapi.fatal_error(b'xyz', {release_gil}) + """, + 2, + 'xyz', + func='test_fatal_error', + py_fatal_error=True) + def test_fatal_error(self): - self.check_fatal_error(""" - import faulthandler - faulthandler._fatal_error(b'xyz') - """, - 2, - 'xyz', - func='faulthandler_fatal_error_py', - py_fatal_error=True) + self.check_fatal_error_func(False) def test_fatal_error_without_gil(self): - self.check_fatal_error(""" - import faulthandler - faulthandler._fatal_error(b'xyz', True) - """, - 2, - 'xyz', - func='faulthandler_fatal_error_py', - py_fatal_error=True) + self.check_fatal_error_func(True) @unittest.skipIf(sys.platform.startswith('openbsd'), "Issue #12868: sigaltstack() doesn't work on " @@ -331,6 +330,26 @@ def test_disable(self): "%r is present in %r" % (not_expected, stderr)) self.assertNotEqual(exitcode, 0) + @skip_segfault_on_android + def test_dump_ext_modules(self): + code = """ + import faulthandler + import sys + # Don't filter stdlib module names + sys.stdlib_module_names = frozenset() + faulthandler.enable() + faulthandler._sigsegv() + """ + stderr, exitcode = self.get_output(code) + stderr = '\n'.join(stderr) + match = re.search(r'^Extension modules:(.*) \(total: [0-9]+\)$', + stderr, re.MULTILINE) + if not match: + self.fail(f"Cannot find 'Extension modules:' in {stderr!r}") + modules = set(match.group(1).strip().split(', ')) + for name in ('sys', 'faulthandler'): + self.assertIn(name, modules) + def test_is_enabled(self): orig_stderr = sys.stderr try: diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 2345832abce624..7ca1512ebbf1bf 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -661,7 +661,7 @@ def test_parens_in_expressions(self): ["f'{3)+(4}'", ]) - self.assertAllRaise(SyntaxError, 'EOL while scanning string literal', + self.assertAllRaise(SyntaxError, 'unterminated string literal', ["f'{\n}'", ]) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 2f6716dfc9a130..0be869ef69b7ce 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -260,7 +260,7 @@ def test_eof_error(self): for s in samples: with self.assertRaises(SyntaxError) as cm: compile(s, "", "exec") - self.assertIn("unexpected EOF", str(cm.exception)) + self.assertIn("was never closed", str(cm.exception)) var_annot_global: int # a global annotated is necessary for test_var_annot diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 08d7ab8a30ba7e..96fddc7d057663 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3881,6 +3881,33 @@ def test_set_inheritable_cloexec(self): self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, 0) + @unittest.skipUnless(hasattr(os, 'O_PATH'), "need os.O_PATH") + def test_get_set_inheritable_o_path(self): + fd = os.open(__file__, os.O_PATH) + self.addCleanup(os.close, fd) + self.assertEqual(os.get_inheritable(fd), False) + + os.set_inheritable(fd, True) + self.assertEqual(os.get_inheritable(fd), True) + + os.set_inheritable(fd, False) + self.assertEqual(os.get_inheritable(fd), False) + + def test_get_set_inheritable_badf(self): + fd = os_helper.make_bad_fd() + + with self.assertRaises(OSError) as ctx: + os.get_inheritable(fd) + self.assertEqual(ctx.exception.errno, errno.EBADF) + + with self.assertRaises(OSError) as ctx: + os.set_inheritable(fd, True) + self.assertEqual(ctx.exception.errno, errno.EBADF) + + with self.assertRaises(OSError) as ctx: + os.set_inheritable(fd, False) + self.assertEqual(ctx.exception.errno, errno.EBADF) + def test_open(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 4bb574fc5b7bff..51cd378648378e 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1649,10 +1649,10 @@ def test_errors_in_command(self): self.assertEqual(stdout.splitlines()[1:], [ '-> pass', - '(Pdb) *** SyntaxError: unexpected EOF while parsing', + '(Pdb) *** SyntaxError: \'(\' was never closed', '(Pdb) ENTERING RECURSIVE DEBUGGER', - '*** SyntaxError: unexpected EOF while parsing', + '*** SyntaxError: \'(\' was never closed', 'LEAVING RECURSIVE DEBUGGER', '(Pdb) ENTERING RECURSIVE DEBUGGER', @@ -1662,6 +1662,71 @@ def test_errors_in_command(self): '(Pdb) ', ]) + + def test_issue42384(self): + '''When running `python foo.py` sys.path[0] is an absolute path. `python -m pdb foo.py` should behave the same''' + script = textwrap.dedent(""" + import sys + print('sys.path[0] is', sys.path[0]) + """) + commands = 'c\nq' + + with os_helper.temp_cwd() as cwd: + expected = f'(Pdb) sys.path[0] is {os.path.realpath(cwd)}' + + stdout, stderr = self.run_pdb_script(script, commands) + + self.assertEqual(stdout.split('\n')[2].rstrip('\r'), expected) + + @os_helper.skip_unless_symlink + def test_issue42384_symlink(self): + '''When running `python foo.py` sys.path[0] resolves symlinks. `python -m pdb foo.py` should behave the same''' + script = textwrap.dedent(""" + import sys + print('sys.path[0] is', sys.path[0]) + """) + commands = 'c\nq' + + with os_helper.temp_cwd() as cwd: + cwd = os.path.realpath(cwd) + dir_one = os.path.join(cwd, 'dir_one') + dir_two = os.path.join(cwd, 'dir_two') + expected = f'(Pdb) sys.path[0] is {dir_one}' + + os.mkdir(dir_one) + with open(os.path.join(dir_one, 'foo.py'), 'w') as f: + f.write(script) + os.mkdir(dir_two) + os.symlink(os.path.join(dir_one, 'foo.py'), os.path.join(dir_two, 'foo.py')) + + stdout, stderr = self._run_pdb([os.path.join('dir_two', 'foo.py')], commands) + + self.assertEqual(stdout.split('\n')[2].rstrip('\r'), expected) + + def test_issue42383(self): + with os_helper.temp_cwd() as cwd: + with open('foo.py', 'w') as f: + s = textwrap.dedent(""" + print('The correct file was executed') + + import os + os.chdir("subdir") + """) + f.write(s) + + subdir = os.path.join(cwd, 'subdir') + os.mkdir(subdir) + os.mkdir(os.path.join(subdir, 'subdir')) + wrong_file = os.path.join(subdir, 'foo.py') + + with open(wrong_file, 'w') as f: + f.write('print("The wrong file was executed")') + + stdout, stderr = self._run_pdb(['foo.py'], 'c\nc\nq') + expected = '(Pdb) The correct file was executed' + self.assertEqual(stdout.split('\n')[6].rstrip('\r'), expected) + + def load_tests(*args): from test import test_pdb suites = [ diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index 190d8d787a2cc9..7585c42bf01338 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -91,7 +91,6 @@ def _set_term_winsz(fd, winsz): # Marginal testing of pty suite. Cannot do extensive 'do or fail' testing # because pty code is not too portable. -# XXX(nnorwitz): these tests leak fds when there is an error. class PtyTest(unittest.TestCase): def setUp(self): old_alarm = signal.signal(signal.SIGALRM, self.handle_sig) @@ -176,6 +175,12 @@ def test_openpty(self): # " An optional feature could not be imported " ... ? raise unittest.SkipTest("Pseudo-terminals (seemingly) not functional.") + # closing master_fd can raise a SIGHUP if the process is + # the session leader: we installed a SIGHUP signal handler + # to ignore this signal. + self.addCleanup(os.close, master_fd) + self.addCleanup(os.close, slave_fd) + self.assertTrue(os.isatty(slave_fd), "slave_fd is not a tty") if mode: @@ -218,15 +223,10 @@ def test_openpty(self): s2 = _readline(master_fd) self.assertEqual(b'For my pet fish, Eric.\n', normalize_output(s2)) - os.close(slave_fd) - # closing master_fd can raise a SIGHUP if the process is - # the session leader: we installed a SIGHUP signal handler - # to ignore this signal. - os.close(master_fd) - def test_fork(self): debug("calling pty.fork()") pid, master_fd = pty.fork() + self.addCleanup(os.close, master_fd) if pid == pty.CHILD: # stdout should be connected to a tty. if not os.isatty(1): @@ -305,13 +305,14 @@ def test_fork(self): ##else: ## raise TestFailed("Read from master_fd did not raise exception") - os.close(master_fd) - def test_master_read(self): + # XXX(nnorwitz): this test leaks fds when there is an error. debug("Calling pty.openpty()") master_fd, slave_fd = pty.openpty() debug(f"Got master_fd '{master_fd}', slave_fd '{slave_fd}'") + self.addCleanup(os.close, master_fd) + debug("Closing slave_fd") os.close(slave_fd) @@ -321,7 +322,6 @@ def test_master_read(self): except OSError: # Linux data = b"" - os.close(master_fd) self.assertEqual(data, b"") class SmallPtyTests(unittest.TestCase): diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 41a26e376d3a58..66908868a6e9ef 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -223,33 +223,6 @@ def test_sample_with_counts(self): with self.assertRaises(ValueError): sample(['red', 'green', 'blue'], counts=[1, 2, 3, 4], k=2) # too many counts - def test_sample_counts_equivalence(self): - # Test the documented strong equivalence to a sample with repeated elements. - # We run this test on random.Random() which makes deterministic selections - # for a given seed value. - sample = random.sample - seed = random.seed - - colors = ['red', 'green', 'blue', 'orange', 'black', 'amber'] - counts = [500, 200, 20, 10, 5, 1 ] - k = 700 - seed(8675309) - s1 = sample(colors, counts=counts, k=k) - seed(8675309) - expanded = [color for (color, count) in zip(colors, counts) for i in range(count)] - self.assertEqual(len(expanded), sum(counts)) - s2 = sample(expanded, k=k) - self.assertEqual(s1, s2) - - pop = 'abcdefghi' - counts = [10, 9, 8, 7, 6, 5, 4, 3, 2] - seed(8675309) - s1 = ''.join(sample(pop, counts=counts, k=30)) - expanded = ''.join([letter for (letter, count) in zip(pop, counts) for i in range(count)]) - seed(8675309) - s2 = ''.join(sample(expanded, k=30)) - self.assertEqual(s1, s2) - def test_choices(self): choices = self.gen.choices data = ['red', 'green', 'blue', 'yellow'] @@ -536,11 +509,24 @@ def test_randrange_errors(self): raises(-721) raises(0, 100, -12) # Non-integer start/stop - raises(3.14159) - raises(0, 2.71828) + self.assertWarns(DeprecationWarning, raises, 3.14159) + self.assertWarns(DeprecationWarning, self.gen.randrange, 3.0) + self.assertWarns(DeprecationWarning, self.gen.randrange, Fraction(3, 1)) + self.assertWarns(DeprecationWarning, raises, '3') + self.assertWarns(DeprecationWarning, raises, 0, 2.71828) + self.assertWarns(DeprecationWarning, self.gen.randrange, 0, 2.0) + self.assertWarns(DeprecationWarning, self.gen.randrange, 0, Fraction(2, 1)) + self.assertWarns(DeprecationWarning, raises, 0, '2') # Zero and non-integer step raises(0, 42, 0) - raises(0, 42, 3.14159) + self.assertWarns(DeprecationWarning, raises, 0, 42, 0.0) + self.assertWarns(DeprecationWarning, raises, 0, 0, 0.0) + self.assertWarns(DeprecationWarning, raises, 0, 42, 3.14159) + self.assertWarns(DeprecationWarning, self.gen.randrange, 0, 42, 3.0) + self.assertWarns(DeprecationWarning, self.gen.randrange, 0, 42, Fraction(3, 1)) + self.assertWarns(DeprecationWarning, raises, 0, 42, '3') + self.assertWarns(DeprecationWarning, self.gen.randrange, 0, 42, 1.0) + self.assertWarns(DeprecationWarning, raises, 0, 0, 1.0) def test_randrange_argument_handling(self): randrange = self.gen.randrange @@ -957,6 +943,33 @@ def test_randbytes_getrandbits(self): self.assertEqual(self.gen.randbytes(n), gen2.getrandbits(n * 8).to_bytes(n, 'little')) + def test_sample_counts_equivalence(self): + # Test the documented strong equivalence to a sample with repeated elements. + # We run this test on random.Random() which makes deterministic selections + # for a given seed value. + sample = self.gen.sample + seed = self.gen.seed + + colors = ['red', 'green', 'blue', 'orange', 'black', 'amber'] + counts = [500, 200, 20, 10, 5, 1 ] + k = 700 + seed(8675309) + s1 = sample(colors, counts=counts, k=k) + seed(8675309) + expanded = [color for (color, count) in zip(colors, counts) for i in range(count)] + self.assertEqual(len(expanded), sum(counts)) + s2 = sample(expanded, k=k) + self.assertEqual(s1, s2) + + pop = 'abcdefghi' + counts = [10, 9, 8, 7, 6, 5, 4, 3, 2] + seed(8675309) + s1 = ''.join(sample(pop, counts=counts, k=30)) + expanded = ''.join([letter for (letter, count) in zip(pop, counts) for i in range(count)]) + seed(8675309) + s2 = ''.join(sample(expanded, k=30)) + self.assertEqual(s1, s2) + def gamma(z, sqrt2pi=(2.0*pi)**0.5): # Reflection to right half of complex plane diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index c1d02cfaf0dcb6..bd689582523c32 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2176,11 +2176,13 @@ def test_flags_repr(self): "re.IGNORECASE|re.DOTALL|re.VERBOSE") self.assertEqual(repr(re.I|re.S|re.X|(1<<20)), "re.IGNORECASE|re.DOTALL|re.VERBOSE|0x100000") - self.assertEqual(repr(~re.I), "~re.IGNORECASE") + self.assertEqual( + repr(~re.I), + "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.DOTALL|re.VERBOSE|re.TEMPLATE|re.DEBUG") self.assertEqual(repr(~(re.I|re.S|re.X)), - "~(re.IGNORECASE|re.DOTALL|re.VERBOSE)") + "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.TEMPLATE|re.DEBUG") self.assertEqual(repr(~(re.I|re.S|re.X|(1<<20))), - "~(re.IGNORECASE|re.DOTALL|re.VERBOSE|0x100000)") + "re.ASCII|re.LOCALE|re.UNICODE|re.MULTILINE|re.TEMPLATE|re.DEBUG|0xffe00") class ImplementationTest(unittest.TestCase): diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index d8255607dcfd5c..c8d191df4cc495 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -987,6 +987,14 @@ def test_invalid_line_continuation_left_recursive(self): self._check_error("A.\u03bc\\\n", "unexpected EOF while parsing") + def test_error_parenthesis(self): + for paren in "([{": + self._check_error(paren + "1 + 2", f"\\{paren}' was never closed") + + for paren in ")]}": + self._check_error(paren + "1 + 2", f"unmatched '\\{paren}'") + + def test_main(): support.run_unittest(SyntaxTestCase) from test import test_syntax diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 3af5b117affde6..c4e053594800b2 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -986,6 +986,11 @@ def test_orig_argv(self): self.assertEqual(proc.stdout.rstrip().splitlines(), expected, proc) + def test_module_names(self): + self.assertIsInstance(sys.stdlib_module_names, frozenset) + for name in sys.stdlib_module_names: + self.assertIsInstance(name, str) + @test.support.cpython_only class UnraisableHookTest(unittest.TestCase): diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 42989862502ac0..49e89289956862 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -956,6 +956,24 @@ def func(): (2, 'line'), (2, 'return')]) + def test_implicit_return_in_class(self): + + def func(): + class A: + if 3 < 9: + a = 1 + else: + a = 2 + + self.run_and_compare(func, + [(0, 'call'), + (1, 'line'), + (1, 'call'), + (1, 'line'), + (2, 'line'), + (3, 'line'), + (3, 'return'), + (1, 'return')]) class SkipLineEventsTraceTestCase(TraceTestCase): """Repeat the trace tests, but with per-line events skipped""" diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 07555a0411a082..33bdda026662cc 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1173,6 +1173,46 @@ def f(): self.assertIn( "RecursionError: maximum recursion depth exceeded", res[-1]) + def test_compact_with_cause(self): + try: + try: + 1/0 + finally: + cause = Exception("cause") + raise Exception("uh oh") from cause + except Exception: + exc_info = sys.exc_info() + exc = traceback.TracebackException(*exc_info, compact=True) + expected_stack = traceback.StackSummary.extract( + traceback.walk_tb(exc_info[2])) + exc_cause = traceback.TracebackException(Exception, cause, None) + self.assertEqual(exc_cause, exc.__cause__) + self.assertEqual(None, exc.__context__) + self.assertEqual(True, exc.__suppress_context__) + self.assertEqual(expected_stack, exc.stack) + self.assertEqual(exc_info[0], exc.exc_type) + self.assertEqual(str(exc_info[1]), str(exc)) + + def test_compact_no_cause(self): + try: + try: + 1/0 + finally: + exc_info_context = sys.exc_info() + exc_context = traceback.TracebackException(*exc_info_context) + raise Exception("uh oh") + except Exception: + exc_info = sys.exc_info() + exc = traceback.TracebackException(*exc_info, compact=True) + expected_stack = traceback.StackSummary.extract( + traceback.walk_tb(exc_info[2])) + self.assertEqual(None, exc.__cause__) + self.assertEqual(exc_context, exc.__context__) + self.assertEqual(False, exc.__suppress_context__) + self.assertEqual(expected_stack, exc.stack) + self.assertEqual(exc_info[0], exc.exc_type) + self.assertEqual(str(exc_info[1]), str(exc)) + def test_no_refs_to_exception_and_traceback_objects(self): try: 1/0 diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 4f5636e1426f4d..df8f2c92b38f88 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1098,6 +1098,12 @@ def __repr__(self): self.assertEqual('{0:^8s}'.format('result'), ' result ') self.assertEqual('{0:^9s}'.format('result'), ' result ') self.assertEqual('{0:^10s}'.format('result'), ' result ') + self.assertEqual('{0:8s}'.format('result'), 'result ') + self.assertEqual('{0:0s}'.format('result'), 'result') + self.assertEqual('{0:08s}'.format('result'), 'result00') + self.assertEqual('{0:<08s}'.format('result'), 'result00') + self.assertEqual('{0:>08s}'.format('result'), '00result') + self.assertEqual('{0:^08s}'.format('result'), '0result0') self.assertEqual('{0:10000}'.format('a'), 'a' + ' ' * 9999) self.assertEqual('{0:10000}'.format(''), ' ' * 10000) self.assertEqual('{0:10000000}'.format(''), ' ' * 10000000) diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 57ac3fb5cf6ef4..01dce7eff25c54 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -1185,8 +1185,7 @@ def winfo_reqwidth(self): self.tk.call('winfo', 'reqwidth', self._w)) def winfo_rgb(self, color): - """Return tuple of decimal values for red, green, blue for - COLOR in this widget.""" + """Return a tuple of integer RGB values in range(65536) for color in this widget.""" return self._getints( self.tk.call('winfo', 'rgb', self._w, color)) diff --git a/Lib/tkinter/colorchooser.py b/Lib/tkinter/colorchooser.py index 3cfc06f6f1fae6..e2fb69dba92763 100644 --- a/Lib/tkinter/colorchooser.py +++ b/Lib/tkinter/colorchooser.py @@ -8,57 +8,69 @@ # fixed initialcolor handling in August 1998 # -# -# options (all have default values): -# -# - initialcolor: color to mark as selected when dialog is displayed -# (given as an RGB triplet or a Tk color string) -# -# - parent: which window to place the dialog on top of -# -# - title: dialog title -# from tkinter.commondialog import Dialog __all__ = ["Chooser", "askcolor"] -# -# color chooser class - class Chooser(Dialog): - "Ask for a color" + """Create a dialog for the tk_chooseColor command. + + Args: + master: The master widget for this dialog. If not provided, + defaults to options['parent'] (if defined). + options: Dictionary of options for the tk_chooseColor call. + initialcolor: Specifies the selected color when the + dialog is first displayed. This can be a tk color + string or a 3-tuple of ints in the range (0, 255) + for an RGB triplet. + parent: The parent window of the color dialog. The + color dialog is displayed on top of this. + title: A string for the title of the dialog box. + """ command = "tk_chooseColor" def _fixoptions(self): + """Ensure initialcolor is a tk color string. + + Convert initialcolor from a RGB triplet to a color string. + """ try: - # make sure initialcolor is a tk color string color = self.options["initialcolor"] if isinstance(color, tuple): - # assume an RGB triplet + # Assume an RGB triplet. self.options["initialcolor"] = "#%02x%02x%02x" % color except KeyError: pass def _fixresult(self, widget, result): - # result can be somethings: an empty tuple, an empty string or - # a Tcl_Obj, so this somewhat weird check handles that + """Adjust result returned from call to tk_chooseColor. + + Return both an RGB tuple of ints in the range (0, 255) and the + tk color string in the form #rrggbb. + """ + # Result can be many things: an empty tuple, an empty string, or + # a _tkinter.Tcl_Obj, so this somewhat weird check handles that. if not result or not str(result): - return None, None # canceled + return None, None # canceled - # to simplify application code, the color chooser returns - # an RGB tuple together with the Tk color string + # To simplify application code, the color chooser returns + # an RGB tuple together with the Tk color string. r, g, b = widget.winfo_rgb(result) - return (r/256, g/256, b/256), str(result) + return (r//256, g//256, b//256), str(result) # # convenience stuff -def askcolor(color = None, **options): - "Ask for a color" +def askcolor(color=None, **options): + """Display dialog window for selection of a color. + + Convenience wrapper for the Chooser class. Displays the color + chooser dialog with color as the initial value. + """ if color: options = options.copy() diff --git a/Lib/tkinter/test/test_tkinter/test_colorchooser.py b/Lib/tkinter/test/test_tkinter/test_colorchooser.py index 600c8cde67e762..41da86c2adef49 100644 --- a/Lib/tkinter/test/test_tkinter/test_colorchooser.py +++ b/Lib/tkinter/test/test_tkinter/test_colorchooser.py @@ -1,13 +1,44 @@ import unittest import tkinter from test.support import requires, run_unittest, swap_attr -from tkinter.test.support import AbstractDefaultRootTest -from tkinter.commondialog import Dialog +from tkinter.test.support import AbstractDefaultRootTest, AbstractTkTest +from tkinter import colorchooser from tkinter.colorchooser import askcolor +from tkinter.commondialog import Dialog requires('gui') +class ChooserTest(AbstractTkTest, unittest.TestCase): + + @classmethod + def setUpClass(cls): + AbstractTkTest.setUpClass.__func__(cls) + cls.cc = colorchooser.Chooser(initialcolor='dark blue slate') + + def test_fixoptions(self): + cc = self.cc + cc._fixoptions() + self.assertEqual(cc.options['initialcolor'], 'dark blue slate') + + cc.options['initialcolor'] = '#D2D269691E1E' + cc._fixoptions() + self.assertEqual(cc.options['initialcolor'], '#D2D269691E1E') + + cc.options['initialcolor'] = (210, 105, 30) + cc._fixoptions() + self.assertEqual(cc.options['initialcolor'], '#d2691e') + + def test_fixresult(self): + cc = self.cc + self.assertEqual(cc._fixresult(self.root, ()), (None, None)) + self.assertEqual(cc._fixresult(self.root, ''), (None, None)) + self.assertEqual(cc._fixresult(self.root, 'chocolate'), + ((210, 105, 30), 'chocolate')) + self.assertEqual(cc._fixresult(self.root, '#4a3c8c'), + ((74, 60, 140), '#4a3c8c')) + + class DefaultRootTest(AbstractDefaultRootTest, unittest.TestCase): def test_askcolor(self): @@ -33,7 +64,7 @@ def test_callback(dialog, master): self.assertRaises(RuntimeError, askcolor) -tests_gui = (DefaultRootTest,) +tests_gui = (ChooserTest, DefaultRootTest,) if __name__ == "__main__": run_unittest(*tests_gui) diff --git a/Lib/tkinter/test/test_tkinter/test_misc.py b/Lib/tkinter/test/test_tkinter/test_misc.py index 585d81ddf9f2d6..f6e5b4db1ae1fb 100644 --- a/Lib/tkinter/test/test_tkinter/test_misc.py +++ b/Lib/tkinter/test/test_tkinter/test_misc.py @@ -192,6 +192,26 @@ def test_clipboard_astral(self): with self.assertRaises(tkinter.TclError): root.clipboard_get() + def test_winfo_rgb(self): + root = self.root + rgb = root.winfo_rgb + + # Color name. + self.assertEqual(rgb('red'), (65535, 0, 0)) + self.assertEqual(rgb('dark slate blue'), (18504, 15677, 35723)) + # #RGB - extends each 4-bit hex value to be 16-bit. + self.assertEqual(rgb('#F0F'), (0xFFFF, 0x0000, 0xFFFF)) + # #RRGGBB - extends each 8-bit hex value to be 16-bit. + self.assertEqual(rgb('#4a3c8c'), (0x4a4a, 0x3c3c, 0x8c8c)) + # #RRRRGGGGBBBB + self.assertEqual(rgb('#dede14143939'), (0xdede, 0x1414, 0x3939)) + # Invalid string. + with self.assertRaises(tkinter.TclError): + rgb('#123456789a') + # RGB triplet is invalid input. + with self.assertRaises(tkinter.TclError): + rgb((111, 78, 55)) + def test_event_repr_defaults(self): e = tkinter.Event() e.serial = 12345 diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 1aee21b5e18fa7..42c1f10373de9b 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -27,6 +27,7 @@ from builtins import open as _builtin_open from codecs import lookup, BOM_UTF8 import collections +import functools from io import TextIOWrapper import itertools as _itertools import re @@ -95,6 +96,7 @@ def _all_string_prefixes(): result.add(''.join(u)) return result +@functools.lru_cache def _compile(expr): return re.compile(expr, re.UNICODE) diff --git a/Lib/traceback.py b/Lib/traceback.py index aef37c9a7af684..090465a3584b77 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -110,8 +110,8 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ value, tb = _parse_value_tb(exc, value, tb) if file is None: file = sys.stderr - for line in TracebackException( - type(value), value, tb, limit=limit).format(chain=chain): + te = TracebackException(type(value), value, tb, limit=limit, compact=True) + for line in te.format(chain=chain): print(line, file=file, end="") @@ -126,8 +126,8 @@ def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ printed as does print_exception(). """ value, tb = _parse_value_tb(exc, value, tb) - return list(TracebackException( - type(value), value, tb, limit=limit).format(chain=chain)) + te = TracebackException(type(value), value, tb, limit=limit, compact=True) + return list(te.format(chain=chain)) def format_exception_only(exc, /, value=_sentinel): @@ -146,8 +146,8 @@ def format_exception_only(exc, /, value=_sentinel): """ if value is _sentinel: value = exc - return list(TracebackException( - type(value), value, None).format_exception_only()) + te = TracebackException(type(value), value, None, compact=True) + return list(te.format_exception_only()) # -- not official API but folk probably use these two functions. @@ -476,7 +476,8 @@ class TracebackException: """ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, - lookup_lines=True, capture_locals=False, _seen=None): + lookup_lines=True, capture_locals=False, compact=False, + _seen=None): # NB: we need to accept exc_traceback, exc_value, exc_traceback to # permit backwards compat with the existing API, otherwise we # need stub thunk objects just to glue it together. @@ -485,6 +486,7 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, if _seen is None: _seen = set() _seen.add(id(exc_value)) + # TODO: locals. self.stack = StackSummary.extract( walk_tb(exc_traceback), limit=limit, lookup_lines=lookup_lines, @@ -504,7 +506,7 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, if lookup_lines: self._load_lines() self.__suppress_context__ = \ - exc_value.__suppress_context__ if exc_value else False + exc_value.__suppress_context__ if exc_value is not None else False # Convert __cause__ and __context__ to `TracebackExceptions`s, use a # queue to avoid recursion (only the top-level call gets _seen == None) @@ -524,8 +526,13 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, _seen=_seen) else: cause = None + + if compact: + need_context = cause is None and not e.__suppress_context__ + else: + need_context = True if (e and e.__context__ is not None - and id(e.__context__) not in _seen): + and need_context and id(e.__context__) not in _seen): context = TracebackException( type(e.__context__), e.__context__, diff --git a/Lib/types.py b/Lib/types.py index 532f4806fc0226..c509b242d5d8f3 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -155,7 +155,12 @@ class DynamicClassAttribute: class's __getattr__ method; this is done by raising AttributeError. This allows one to have properties active on an instance, and have virtual - attributes on the class with the same name (see Enum for an example). + attributes on the class with the same name. (Enum used this between Python + versions 3.4 - 3.9 .) + + Subclass from this to use a different method of accessing virtual atributes + and still be treated properly by the inspect module. (Enum uses this since + Python 3.10 .) """ def __init__(self, fget=None, fset=None, fdel=None, doc=None): diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 111317b329a852..ce7468e31481f0 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -183,7 +183,8 @@ def _exc_info_to_string(self, err, test): else: length = None tb_e = traceback.TracebackException( - exctype, value, tb, limit=length, capture_locals=self.tb_locals) + exctype, value, tb, + limit=length, capture_locals=self.tb_locals, compact=True) msgLines = list(tb_e.format()) if self.buffer: diff --git a/Makefile.pre.in b/Makefile.pre.in index 7f7f7596419416..0b22bdd5591b9d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -146,6 +146,8 @@ CONFINCLUDEDIR= $(exec_prefix)/include PLATLIBDIR= @PLATLIBDIR@ SCRIPTDIR= $(prefix)/$(PLATLIBDIR) ABIFLAGS= @ABIFLAGS@ +# Variable used by ensurepip +WHEEL_PKG_DIR= @WHEEL_PKG_DIR@ # Detailed destination directories BINLIBDEST= @BINLIBDEST@ @@ -252,7 +254,7 @@ PYTHON= python$(EXE) BUILDPYTHON= python$(BUILDEXE) PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@ -UPDATE_FILE=@PYTHON_FOR_REGEN@ $(srcdir)/Tools/scripts/update_file.py +UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ _PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@ BUILD_GNU_TYPE= @build@ @@ -755,8 +757,10 @@ regen-limited-abi: all # Regenerate all generated files regen-all: regen-opcode regen-opcode-targets regen-typeslots \ - regen-token regen-ast regen-importlib clinic \ + regen-token regen-ast regen-keyword regen-importlib clinic \ regen-pegen-metaparser regen-pegen + @echo + @echo "Note: make regen-stdlib-module-names and autoconf should be run manually" ############################################################################ # Special rules for object files @@ -896,6 +900,15 @@ regen-keyword: $(srcdir)/Lib/keyword.py.new $(UPDATE_FILE) $(srcdir)/Lib/keyword.py $(srcdir)/Lib/keyword.py.new +.PHONY: regen-stdlib-module-names +regen-stdlib-module-names: build_all + # Regenerate Python/stdlib_module_names.h + # using Tools/scripts/generate_stdlib_module_names.py + $(RUNSHARED) ./$(BUILDPYTHON) \ + $(srcdir)/Tools/scripts/generate_stdlib_module_names.py \ + > $(srcdir)/Python/stdlib_module_names.h.new + $(UPDATE_FILE) $(srcdir)/Python/stdlib_module_names.h $(srcdir)/Python/stdlib_module_names.h.new + Python/compile.o Python/symtable.o Python/ast_unparse.o Python/ast.o Python/future.o: $(srcdir)/Include/Python-ast.h Python/getplatform.o: $(srcdir)/Python/getplatform.c @@ -1145,7 +1158,9 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_ucnhash.h \ $(srcdir)/Include/internal/pycore_unionobject.h \ $(srcdir)/Include/internal/pycore_warnings.h \ - $(DTRACE_HEADERS) + $(DTRACE_HEADERS) \ + \ + $(srcdir)/Python/stdlib_module_names.h $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) diff --git a/Misc/ACKS b/Misc/ACKS index 136266965a869b..29ef9864f98271 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -141,6 +141,7 @@ Stefan Behnel Reimer Behrends Ben Bell Thomas Bellman +John Belmonte Alexander “Саша” Belopolsky Eli Bendersky Nikhil Benesch diff --git a/Misc/NEWS.d/3.10.0a4.rst b/Misc/NEWS.d/3.10.0a4.rst index 882e03d82169a9..57da9254587b44 100644 --- a/Misc/NEWS.d/3.10.0a4.rst +++ b/Misc/NEWS.d/3.10.0a4.rst @@ -605,12 +605,12 @@ deprecated in Python 3.7. Patch by Erlend E. Aasland .. nonce: Cfl1eR .. section: Library -Harmonized random.randrange() argument handling to match range(). +Harmonized :func:`random.randrange` argument handling to match :func:`range`. -* The integer test and conversion in randrange() now uses - operator.index(). -* Non-integer arguments to randrange() are deprecated. -* The *ValueError* is deprecated in favor of a *TypeError*. +* The integer test and conversion in ``randrange()`` now uses + :func:`operator.index`. +* Non-integer arguments to ``randrange()`` are deprecated. +* The ``ValueError`` is deprecated in favor of a ``TypeError``. * It now runs a little faster than before. (Contributed by Raymond Hettinger and Serhiy Storchaka.) diff --git a/Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst b/Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst new file mode 100644 index 00000000000000..6aab7a6e51d071 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst @@ -0,0 +1,9 @@ +Add ``--with-wheel-pkg-dir=PATH`` option to the ``./configure`` script. If +specified, the :mod:`ensurepip` module looks for ``setuptools`` and ``pip`` +wheel packages in this directory: if both are present, these wheel packages are +used instead of ensurepip bundled wheel packages. + +Some Linux distribution packaging policies recommend against bundling +dependencies. For example, Fedora installs wheel packages in the +``/usr/share/python-wheels/`` directory and don't install the +``ensurepip._bundled`` package. diff --git a/Misc/NEWS.d/next/Build/2021-01-11-23-26-00.bpo-31904.ty8f3h.rst b/Misc/NEWS.d/next/Build/2021-01-11-23-26-00.bpo-31904.ty8f3h.rst new file mode 100644 index 00000000000000..bc02d0a04f5283 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-01-11-23-26-00.bpo-31904.ty8f3h.rst @@ -0,0 +1 @@ +Add library search path by wr-cc in add_cross_compiling_paths() for VxWorks. diff --git a/Misc/NEWS.d/next/Build/2021-01-18-20-52-06.bpo-36143.kgnIYo.rst b/Misc/NEWS.d/next/Build/2021-01-18-20-52-06.bpo-36143.kgnIYo.rst new file mode 100644 index 00000000000000..5ac3269d955408 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-01-18-20-52-06.bpo-36143.kgnIYo.rst @@ -0,0 +1 @@ +``make regen-all`` now also runs ``regen-keyword``. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Build/2021-01-26-14-48-40.bpo-43031.44nK9U.rst b/Misc/NEWS.d/next/Build/2021-01-26-14-48-40.bpo-43031.44nK9U.rst new file mode 100644 index 00000000000000..6e8377fb306127 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-01-26-14-48-40.bpo-43031.44nK9U.rst @@ -0,0 +1,2 @@ +Pass ``--timeout=$(TESTTIMEOUT)`` option to the default profile task +``./python -m test --pgo`` command. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-12-20-23-59-23.bpo-27772.idHEcj.rst b/Misc/NEWS.d/next/Core and Builtins/2018-12-20-23-59-23.bpo-27772.idHEcj.rst new file mode 100644 index 00000000000000..7345152fee3568 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-12-20-23-59-23.bpo-27772.idHEcj.rst @@ -0,0 +1,2 @@ +In string formatting, preceding the *width* field by ``'0'`` no longer +affects the default alignment for strings. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-01-06-17-06-37.bpo-42827.jtRR0D.rst b/Misc/NEWS.d/next/Core and Builtins/2021-01-06-17-06-37.bpo-42827.jtRR0D.rst new file mode 100644 index 00000000000000..8e40ab6a65341a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-01-06-17-06-37.bpo-42827.jtRR0D.rst @@ -0,0 +1,2 @@ +Fix a crash when working out the error line of a :exc:`SyntaxError` in some +multi-line expressions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-01-14-23-15-34.bpo-42864.QgOAQ1.rst b/Misc/NEWS.d/next/Core and Builtins/2021-01-14-23-15-34.bpo-42864.QgOAQ1.rst new file mode 100644 index 00000000000000..127a29f518d798 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-01-14-23-15-34.bpo-42864.QgOAQ1.rst @@ -0,0 +1,2 @@ +Improve error messages in the parser when parentheses are not closed. Patch +by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-01-20-22-31-01.bpo-40176.anjyWw.rst b/Misc/NEWS.d/next/Core and Builtins/2021-01-20-22-31-01.bpo-40176.anjyWw.rst new file mode 100644 index 00000000000000..df7de3bdf37bc2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-01-20-22-31-01.bpo-40176.anjyWw.rst @@ -0,0 +1,2 @@ +Syntax errors for unterminated string literals now point to the start +of the string instead of reporting EOF/EOL. diff --git a/Misc/NEWS.d/next/Documentation/2021-01-20-23-03-49.bpo-40304.-LK7Ps.rst b/Misc/NEWS.d/next/Documentation/2021-01-20-23-03-49.bpo-40304.-LK7Ps.rst new file mode 100644 index 00000000000000..3f2f14c2d7b893 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2021-01-20-23-03-49.bpo-40304.-LK7Ps.rst @@ -0,0 +1,2 @@ +Fix doc for type(name, bases, dict). Patch by Boris Verkhovskiy and +Éric Araujo. diff --git a/Misc/NEWS.d/next/IDLE/2021-01-26-18-12-17.bpo-43008.mbQUc7.rst b/Misc/NEWS.d/next/IDLE/2021-01-26-18-12-17.bpo-43008.mbQUc7.rst new file mode 100644 index 00000000000000..3e0b80a909d728 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2021-01-26-18-12-17.bpo-43008.mbQUc7.rst @@ -0,0 +1 @@ +Make IDLE invoke :func:`sys.excepthook` in normal, 2-process mode. diff --git a/Misc/NEWS.d/next/Library/2018-04-23-13-44-10.bpo-33289.anBnUr.rst b/Misc/NEWS.d/next/Library/2018-04-23-13-44-10.bpo-33289.anBnUr.rst new file mode 100644 index 00000000000000..52d9ac9dd902cd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-04-23-13-44-10.bpo-33289.anBnUr.rst @@ -0,0 +1,2 @@ +Correct call to :mod:`tkinter.colorchooser` to return RGB triplet of ints +instead of floats. Patch by Cheryl Sabella. diff --git a/Misc/NEWS.d/next/Library/2020-01-13-23-37-58.bpo-39273.m5hzxV.rst b/Misc/NEWS.d/next/Library/2020-01-13-23-37-58.bpo-39273.m5hzxV.rst new file mode 100644 index 00000000000000..c942da07da3779 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-01-13-23-37-58.bpo-39273.m5hzxV.rst @@ -0,0 +1,2 @@ +The ``BUTTON5_*`` constants are now exposed in the :mod:`curses` module if +available. diff --git a/Misc/NEWS.d/next/Library/2020-10-11-13-48-03.bpo-42005.Jq6Az-.rst b/Misc/NEWS.d/next/Library/2020-10-11-13-48-03.bpo-42005.Jq6Az-.rst new file mode 100644 index 00000000000000..be4ed7f55ffded --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-10-11-13-48-03.bpo-42005.Jq6Az-.rst @@ -0,0 +1,2 @@ +Fix CLI of :mod:`cProfile` and :mod:`profile` to catch +:exc:`BrokenPipeError`. diff --git a/Misc/NEWS.d/next/Library/2020-11-17-14-30-12.bpo-42383.ubl0Y_.rst b/Misc/NEWS.d/next/Library/2020-11-17-14-30-12.bpo-42383.ubl0Y_.rst new file mode 100644 index 00000000000000..ccf2106f28a93d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-11-17-14-30-12.bpo-42383.ubl0Y_.rst @@ -0,0 +1,2 @@ +Fix pdb: previously pdb would fail to restart the debugging target if it was +specified using a relative path and the current directory changed. diff --git a/Misc/NEWS.d/next/Library/2020-11-17-14-32-39.bpo-42384.1ZnQSn.rst b/Misc/NEWS.d/next/Library/2020-11-17-14-32-39.bpo-42384.1ZnQSn.rst new file mode 100644 index 00000000000000..ae990162fbab75 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-11-17-14-32-39.bpo-42384.1ZnQSn.rst @@ -0,0 +1 @@ +Make pdb populate sys.path[0] exactly the same as regular python execution. diff --git a/Misc/NEWS.d/next/Library/2021-01-08-15-49-20.bpo-42780.rtqi6B.rst b/Misc/NEWS.d/next/Library/2021-01-08-15-49-20.bpo-42780.rtqi6B.rst new file mode 100644 index 00000000000000..a491690507129e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-01-08-15-49-20.bpo-42780.rtqi6B.rst @@ -0,0 +1 @@ +Fix os.set_inheritable() for O_PATH file descriptors on Linux. diff --git a/Misc/NEWS.d/next/Library/2021-01-13-12-15-13.bpo-42923.zBiNls.rst b/Misc/NEWS.d/next/Library/2021-01-13-12-15-13.bpo-42923.zBiNls.rst new file mode 100644 index 00000000000000..bb566f982b5ce1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-01-13-12-15-13.bpo-42923.zBiNls.rst @@ -0,0 +1,2 @@ +The :c:func:`Py_FatalError` function and the :mod:`faulthandler` module now +dump the list of extension modules on a fatal error. diff --git a/Misc/NEWS.d/next/Library/2021-01-13-12-55-41.bpo-42877.Fi1zEG.rst b/Misc/NEWS.d/next/Library/2021-01-13-12-55-41.bpo-42877.Fi1zEG.rst new file mode 100644 index 00000000000000..49bb74bc536653 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-01-13-12-55-41.bpo-42877.Fi1zEG.rst @@ -0,0 +1,4 @@ +Added the ``compact`` parameter to the constructor of +:class:`traceback.TracebackException` to reduce time and memory +for use cases that only need to call :func:`TracebackException.format` +and :func:`TracebackException.format_exception_only`. diff --git a/Misc/NEWS.d/next/Library/2021-01-14-15-07-16.bpo-38250.1fvhOk.rst b/Misc/NEWS.d/next/Library/2021-01-14-15-07-16.bpo-38250.1fvhOk.rst new file mode 100644 index 00000000000000..e5a72468370fba --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-01-14-15-07-16.bpo-38250.1fvhOk.rst @@ -0,0 +1,5 @@ +[Enum] Flags consisting of a single bit are now considered canonical, and +will be the only flags returned from listing and iterating over a Flag class +or a Flag member. Multi-bit flags are considered aliases; they will be +returned from lookups and operations that result in their value. +Iteration for both Flag and Flag members is in definition order. diff --git a/Misc/NEWS.d/next/Library/2021-01-15-00-23-50.bpo-42931.QD6U2B.rst b/Misc/NEWS.d/next/Library/2021-01-15-00-23-50.bpo-42931.QD6U2B.rst new file mode 100644 index 00000000000000..01f8094944f70c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-01-15-00-23-50.bpo-42931.QD6U2B.rst @@ -0,0 +1 @@ +Add :func:`randbytes` to ``random.__all__``. diff --git a/Misc/NEWS.d/next/Library/2021-01-15-11-48-00.bpo-42934.ILKoOI.rst b/Misc/NEWS.d/next/Library/2021-01-15-11-48-00.bpo-42934.ILKoOI.rst new file mode 100644 index 00000000000000..92f2402d2324a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-01-15-11-48-00.bpo-42934.ILKoOI.rst @@ -0,0 +1,3 @@ +Use :class:`~traceback.TracebackException`'s new ``compact`` param in +:class:`~unittest.TestResult` to reduce time and memory consumed by +traceback formatting. diff --git a/Misc/NEWS.d/next/Library/2021-01-18-10-41-44.bpo-42944.RrONvy.rst b/Misc/NEWS.d/next/Library/2021-01-18-10-41-44.bpo-42944.RrONvy.rst new file mode 100644 index 00000000000000..b78d10aa255454 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-01-18-10-41-44.bpo-42944.RrONvy.rst @@ -0,0 +1 @@ +Fix ``random.Random.sample`` when ``counts`` argument is not ``None``. diff --git a/Misc/NEWS.d/next/Library/2021-01-18-11-59-46.bpo-42955.CSWLC9.rst b/Misc/NEWS.d/next/Library/2021-01-18-11-59-46.bpo-42955.CSWLC9.rst new file mode 100644 index 00000000000000..373b829b0fb76d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-01-18-11-59-46.bpo-42955.CSWLC9.rst @@ -0,0 +1,2 @@ +Add :data:`sys.stdlib_module_names`, containing the list of the standard library +module names. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2021-01-20-12-10-47.bpo-42323.PONB8e.rst b/Misc/NEWS.d/next/Library/2021-01-20-12-10-47.bpo-42323.PONB8e.rst new file mode 100644 index 00000000000000..b2f7becee9d23c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-01-20-12-10-47.bpo-42323.PONB8e.rst @@ -0,0 +1 @@ +Fix :func:`math.nextafter` for NaN on AIX. diff --git a/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst b/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst new file mode 100644 index 00000000000000..02898e4a3a42e9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst @@ -0,0 +1 @@ +Improve performance of :mod:`tokenize` by 20-30%. Patch by Anthony Sottile. diff --git a/Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst b/Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst new file mode 100644 index 00000000000000..7df65a156feabd --- /dev/null +++ b/Misc/NEWS.d/next/Security/2021-01-18-09-27-31.bpo-42938.4Zn4Mp.rst @@ -0,0 +1,2 @@ +Avoid static buffers when computing the repr of :class:`ctypes.c_double` and +:class:`ctypes.c_longdouble` values. diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 40a05a44edd4c1..56ccc2f1e0b5da 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -487,58 +487,47 @@ is_literal_char(unsigned char c) static PyObject * PyCArg_repr(PyCArgObject *self) { - char buffer[256]; switch(self->tag) { case 'b': case 'B': - sprintf(buffer, "", + return PyUnicode_FromFormat("", self->tag, self->value.b); - break; case 'h': case 'H': - sprintf(buffer, "", + return PyUnicode_FromFormat("", self->tag, self->value.h); - break; case 'i': case 'I': - sprintf(buffer, "", + return PyUnicode_FromFormat("", self->tag, self->value.i); - break; case 'l': case 'L': - sprintf(buffer, "", + return PyUnicode_FromFormat("", self->tag, self->value.l); - break; case 'q': case 'Q': - sprintf(buffer, -#ifdef MS_WIN32 - "", -#else - "", -#endif + return PyUnicode_FromFormat("", self->tag, self->value.q); - break; case 'd': - sprintf(buffer, "", - self->tag, self->value.d); - break; - case 'f': - sprintf(buffer, "", - self->tag, self->value.f); - break; - + case 'f': { + PyObject *f = PyFloat_FromDouble((self->tag == 'f') ? self->value.f : self->value.d); + if (f == NULL) { + return NULL; + } + PyObject *result = PyUnicode_FromFormat("", self->tag, f); + Py_DECREF(f); + return result; + } case 'c': if (is_literal_char((unsigned char)self->value.c)) { - sprintf(buffer, "", + return PyUnicode_FromFormat("", self->tag, self->value.c); } else { - sprintf(buffer, "", + return PyUnicode_FromFormat("", self->tag, (unsigned char)self->value.c); } - break; /* Hm, are these 'z' and 'Z' codes useful at all? Shouldn't they be replaced by the functionality of c_string @@ -547,22 +536,20 @@ PyCArg_repr(PyCArgObject *self) case 'z': case 'Z': case 'P': - sprintf(buffer, "", + return PyUnicode_FromFormat("", self->tag, self->value.p); break; default: if (is_literal_char((unsigned char)self->tag)) { - sprintf(buffer, "", + return PyUnicode_FromFormat("", (unsigned char)self->tag, (void *)self); } else { - sprintf(buffer, "", + return PyUnicode_FromFormat("", (unsigned char)self->tag, (void *)self); } - break; } - return PyUnicode_FromString(buffer); } static PyMemberDef PyCArgType_members[] = { diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 7175c722965794..4fcedc5fc4820f 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -4725,22 +4725,23 @@ static struct PyModuleDef _cursesmodule = { NULL }; +static void +curses_destructor(PyObject *op) +{ + void *ptr = PyCapsule_GetPointer(op, PyCurses_CAPSULE_NAME); + Py_DECREF(*(void **)ptr); + PyMem_Free(ptr); +} + PyMODINIT_FUNC PyInit__curses(void) { PyObject *m, *d, *v, *c_api_object; - static void *PyCurses_API[PyCurses_API_pointers]; /* Initialize object type */ if (PyType_Ready(&PyCursesWindow_Type) < 0) return NULL; - /* Initialize the C API pointer array */ - PyCurses_API[0] = (void *)&PyCursesWindow_Type; - PyCurses_API[1] = (void *)func_PyCursesSetupTermCalled; - PyCurses_API[2] = (void *)func_PyCursesInitialised; - PyCurses_API[3] = (void *)func_PyCursesInitialisedColor; - /* Create the module and add the functions */ m = PyModule_Create(&_cursesmodule); if (m == NULL) @@ -4752,9 +4753,29 @@ PyInit__curses(void) return NULL; ModDict = d; /* For PyCurses_InitScr to use later */ + void **PyCurses_API = PyMem_Calloc(PyCurses_API_pointers, sizeof(void *)); + if (PyCurses_API == NULL) { + PyErr_NoMemory(); + return NULL; + } + /* Initialize the C API pointer array */ + PyCurses_API[0] = (void *)Py_NewRef(&PyCursesWindow_Type); + PyCurses_API[1] = (void *)func_PyCursesSetupTermCalled; + PyCurses_API[2] = (void *)func_PyCursesInitialised; + PyCurses_API[3] = (void *)func_PyCursesInitialisedColor; + /* Add a capsule for the C API */ - c_api_object = PyCapsule_New(PyCurses_API, PyCurses_CAPSULE_NAME, NULL); - PyDict_SetItemString(d, "_C_API", c_api_object); + c_api_object = PyCapsule_New(PyCurses_API, PyCurses_CAPSULE_NAME, + curses_destructor); + if (c_api_object == NULL) { + Py_DECREF(PyCurses_API[0]); + PyMem_Free(PyCurses_API); + return NULL; + } + if (PyDict_SetItemString(d, "_C_API", c_api_object) < 0) { + Py_DECREF(c_api_object); + return NULL; + } Py_DECREF(c_api_object); /* For exception curses.error */ @@ -4870,6 +4891,14 @@ PyInit__curses(void) SetDictInt("BUTTON4_DOUBLE_CLICKED", BUTTON4_DOUBLE_CLICKED); SetDictInt("BUTTON4_TRIPLE_CLICKED", BUTTON4_TRIPLE_CLICKED); +#if NCURSES_MOUSE_VERSION > 1 + SetDictInt("BUTTON5_PRESSED", BUTTON5_PRESSED); + SetDictInt("BUTTON5_RELEASED", BUTTON5_RELEASED); + SetDictInt("BUTTON5_CLICKED", BUTTON5_CLICKED); + SetDictInt("BUTTON5_DOUBLE_CLICKED", BUTTON5_DOUBLE_CLICKED); + SetDictInt("BUTTON5_TRIPLE_CLICKED", BUTTON5_TRIPLE_CLICKED); +#endif + SetDictInt("BUTTON_SHIFT", BUTTON_SHIFT); SetDictInt("BUTTON_CTRL", BUTTON_CTRL); SetDictInt("BUTTON_ALT", BUTTON_ALT); diff --git a/Modules/_sqlite/clinic/cursor.c.h b/Modules/_sqlite/clinic/cursor.c.h index d5cf0a5eaadae1..7a79d74818a2e2 100644 --- a/Modules/_sqlite/clinic/cursor.c.h +++ b/Modules/_sqlite/clinic/cursor.c.h @@ -138,10 +138,13 @@ pysqlite_cursor_fetchone(pysqlite_Cursor *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(pysqlite_cursor_fetchmany__doc__, -"fetchmany($self, /, size=cursor.arraysize)\n" +"fetchmany($self, /, size=1)\n" "--\n" "\n" -"Fetches several rows from the resultset."); +"Fetches several rows from the resultset.\n" +"\n" +" size\n" +" The default value is set by the Cursor.arraysize attribute."); #define PYSQLITE_CURSOR_FETCHMANY_METHODDEF \ {"fetchmany", (PyCFunction)(void(*)(void))pysqlite_cursor_fetchmany, METH_FASTCALL|METH_KEYWORDS, pysqlite_cursor_fetchmany__doc__}, @@ -256,4 +259,4 @@ pysqlite_cursor_close(pysqlite_Cursor *self, PyObject *Py_UNUSED(ignored)) { return pysqlite_cursor_close_impl(self); } -/*[clinic end generated code: output=11db0de4fb1951a9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6a2d4d49784aa686 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 296d569148f8ae..0852aa940264a2 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -822,14 +822,15 @@ pysqlite_cursor_fetchone_impl(pysqlite_Cursor *self) /*[clinic input] _sqlite3.Cursor.fetchmany as pysqlite_cursor_fetchmany - size as maxrows: int(c_default='self->arraysize') = cursor.arraysize + size as maxrows: int(c_default='self->arraysize') = 1 + The default value is set by the Cursor.arraysize attribute. Fetches several rows from the resultset. [clinic start generated code]*/ static PyObject * pysqlite_cursor_fetchmany_impl(pysqlite_Cursor *self, int maxrows) -/*[clinic end generated code: output=a8ef31fea64d0906 input=d80ff999a7701ffb]*/ +/*[clinic end generated code: output=a8ef31fea64d0906 input=c26e6ca3f34debd0]*/ { PyObject* row; PyObject* list; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 4f97927fa23229..ed59c32e9c5e7f 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4736,6 +4736,18 @@ return_result_with_error(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject* +getitem_with_error(PyObject *self, PyObject *args) +{ + PyObject *map, *key; + if (!PyArg_ParseTuple(args, "OO", &map, &key)) { + return NULL; + } + + PyErr_SetString(PyExc_ValueError, "bug"); + return PyObject_GetItem(map, key); +} + static PyObject * test_pytime_fromseconds(PyObject *self, PyObject *args) { @@ -5665,6 +5677,27 @@ test_refcount(PyObject *self, PyObject *Py_UNUSED(ignored)) } +static PyObject * +test_fatal_error(PyObject *self, PyObject *args) +{ + char *message; + int release_gil = 0; + if (!PyArg_ParseTuple(args, "y|i:fatal_error", &message, &release_gil)) + return NULL; + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + Py_FatalError(message); + Py_END_ALLOW_THREADS + } + else { + Py_FatalError(message); + } + // Py_FatalError() does not return, but exits the process. + Py_RETURN_NONE; +} + + + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, {"raise_memoryerror", raise_memoryerror, METH_NOARGS}, @@ -5880,10 +5913,9 @@ static PyMethodDef TestMethods[] = { pymarshal_read_last_object_from_file, METH_VARARGS}, {"pymarshal_read_object_from_file", pymarshal_read_object_from_file, METH_VARARGS}, - {"return_null_without_error", - return_null_without_error, METH_NOARGS}, - {"return_result_with_error", - return_result_with_error, METH_NOARGS}, + {"return_null_without_error", return_null_without_error, METH_NOARGS}, + {"return_result_with_error", return_result_with_error, METH_NOARGS}, + {"getitem_with_error", getitem_with_error, METH_VARARGS}, {"PyTime_FromSeconds", test_pytime_fromseconds, METH_VARARGS}, {"PyTime_FromSecondsObject", test_pytime_fromsecondsobject, METH_VARARGS}, {"PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, @@ -5938,6 +5970,8 @@ static PyMethodDef TestMethods[] = { {"without_gc", without_gc, METH_O}, {"test_set_type_size", test_set_type_size, METH_NOARGS}, {"test_refcount", test_refcount, METH_NOARGS}, + {"fatal_error", test_fatal_error, METH_VARARGS, + PyDoc_STR("fatal_error(message, release_gil=False): call Py_FatalError(message)")}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index bb32b1496683e4..d0c462fb86ab50 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -904,7 +904,13 @@ load_data(PyZoneInfo_ZoneInfo *self, PyObject *file_obj) // Load the transition indices and list self->trans_list_utc = PyMem_Malloc(self->num_transitions * sizeof(int64_t)); + if (self->trans_list_utc == NULL) { + goto error; + } trans_idx = PyMem_Malloc(self->num_transitions * sizeof(Py_ssize_t)); + if (trans_idx == NULL) { + goto error; + } for (size_t i = 0; i < self->num_transitions; ++i) { PyObject *num = PyTuple_GetItem(trans_utc, i); @@ -986,6 +992,9 @@ load_data(PyZoneInfo_ZoneInfo *self, PyObject *file_obj) // Build _ttinfo objects from utcoff, dstoff and abbr self->_ttinfos = PyMem_Malloc(self->num_ttinfos * sizeof(_ttinfo)); + if (self->_ttinfos == NULL) { + goto error; + } for (size_t i = 0; i < self->num_ttinfos; ++i) { PyObject *tzname = PyTuple_GetItem(abbr, i); if (tzname == NULL) { @@ -1001,6 +1010,9 @@ load_data(PyZoneInfo_ZoneInfo *self, PyObject *file_obj) // Build our mapping from transition to the ttinfo that applies self->trans_ttinfos = PyMem_Calloc(self->num_transitions, sizeof(_ttinfo *)); + if (self->trans_ttinfos == NULL) { + goto error; + } for (size_t i = 0; i < self->num_transitions; ++i) { size_t ttinfo_idx = trans_idx[i]; assert(ttinfo_idx < self->num_ttinfos); @@ -2513,7 +2525,11 @@ zoneinfo_init_subclass(PyTypeObject *cls, PyObject *args, PyObject **kwargs) return NULL; } - PyObject_SetAttrString((PyObject *)cls, "_weak_cache", weak_cache); + if (PyObject_SetAttrString((PyObject *)cls, "_weak_cache", + weak_cache) < 0) { + Py_DECREF(weak_cache); + return NULL; + } Py_DECREF(weak_cache); Py_RETURN_NONE; } @@ -2613,6 +2629,9 @@ static int zoneinfomodule_exec(PyObject *m) { PyDateTime_IMPORT; + if (PyDateTimeAPI == NULL) { + goto error; + } PyZoneInfo_ZoneInfoType.tp_base = PyDateTimeAPI->TZInfoType; if (PyType_Ready(&PyZoneInfo_ZoneInfoType) < 0) { goto error; diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 67fe1ca9ffffd2..da8b7741345de0 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1,6 +1,7 @@ #include "Python.h" -#include "pycore_initconfig.h" -#include "pycore_traceback.h" +#include "pycore_initconfig.h" // _PyStatus_ERR +#include "pycore_pyerrors.h" // _Py_DumpExtensionModules +#include "pycore_traceback.h" // _Py_DumpTracebackThreads #include #include #include @@ -349,6 +350,8 @@ faulthandler_fatal_error(int signum) faulthandler_dump_traceback(fd, fatal_error.all_threads, fatal_error.interp); + _Py_DumpExtensionModules(fd, fatal_error.interp); + errno = save_errno; #ifdef MS_WINDOWS if (signum == SIGSEGV) { @@ -1123,25 +1126,6 @@ faulthandler_sigabrt(PyObject *self, PyObject *args) Py_RETURN_NONE; } -static PyObject * -faulthandler_fatal_error_py(PyObject *self, PyObject *args) -{ - char *message; - int release_gil = 0; - if (!PyArg_ParseTuple(args, "y|i:fatal_error", &message, &release_gil)) - return NULL; - faulthandler_suppress_crash_report(); - if (release_gil) { - Py_BEGIN_ALLOW_THREADS - Py_FatalError(message); - Py_END_ALLOW_THREADS - } - else { - Py_FatalError(message); - } - Py_RETURN_NONE; -} - #if defined(FAULTHANDLER_USE_ALT_STACK) #define FAULTHANDLER_STACK_OVERFLOW @@ -1278,8 +1262,6 @@ static PyMethodDef module_methods[] = { PyDoc_STR("_sigabrt(): raise a SIGABRT signal")}, {"_sigfpe", (PyCFunction)faulthandler_sigfpe, METH_NOARGS, PyDoc_STR("_sigfpe(): raise a SIGFPE signal")}, - {"_fatal_error", faulthandler_fatal_error_py, METH_VARARGS, - PyDoc_STR("_fatal_error(message): call Py_FatalError(message)")}, #ifdef FAULTHANDLER_STACK_OVERFLOW {"_stack_overflow", faulthandler_stack_overflow, METH_NOARGS, PyDoc_STR("_stack_overflow(): recursive call to raise a stack overflow")}, diff --git a/Modules/main.c b/Modules/main.c index ccf096352e928b..2684d230672b94 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -87,7 +87,7 @@ static inline int config_run_code(const PyConfig *config) } -/* Return non-zero is stdin is a TTY or if -i command line option is used */ +/* Return non-zero if stdin is a TTY or if -i command line option is used */ static int stdin_is_interactive(const PyConfig *config) { diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 86b64fb4226907..8133d6b3aaefb8 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -3473,6 +3473,12 @@ math_nextafter_impl(PyObject *module, double x, double y) Bug fixed in bos.adt.libm 7.2.2.0 by APAR IV95512. */ return PyFloat_FromDouble(y); } + if (Py_IS_NAN(x)) { + return x; + } + if (Py_IS_NAN(y)) { + return y; + } #endif return PyFloat_FromDouble(nextafter(x, y)); } diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 4b8c46c7797665..aebae7da576561 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -1308,10 +1308,31 @@ capi_getcode(const char* name, int namelen, Py_UCS4* code, } -static const _PyUnicode_Name_CAPI unicodedata_capi = +static void +unicodedata_destroy_capi(PyObject *capsule) { - .getname = capi_getucname, - .getcode = capi_getcode, + void *capi = PyCapsule_GetPointer(capsule, PyUnicodeData_CAPSULE_NAME); + PyMem_Free(capi); +} + +static PyObject * +unicodedata_create_capi(void) +{ + _PyUnicode_Name_CAPI *capi = PyMem_Malloc(sizeof(_PyUnicode_Name_CAPI)); + if (capi == NULL) { + PyErr_NoMemory(); + return NULL; + } + capi->getname = capi_getucname; + capi->getcode = capi_getcode; + + PyObject *capsule = PyCapsule_New(capi, + PyUnicodeData_CAPSULE_NAME, + unicodedata_destroy_capi); + if (capsule == NULL) { + PyMem_Free(capi); + } + return capsule; }; @@ -1477,13 +1498,13 @@ unicodedata_exec(PyObject *module) } /* Export C API */ - v = PyCapsule_New((void *)&unicodedata_capi, PyUnicodeData_CAPSULE_NAME, - NULL); - if (v == NULL) { + PyObject *capsule = unicodedata_create_capi(); + if (capsule == NULL) { return -1; } - if (PyModule_AddObject(module, "_ucnhash_CAPI", v) < 0) { - Py_DECREF(v); + int rc = PyModule_AddObjectRef(module, "_ucnhash_CAPI", capsule); + Py_DECREF(capsule); + if (rc < 0) { return -1; } return 0; diff --git a/Objects/abstract.c b/Objects/abstract.c index 44ed5b3932bf21..cc452eaaf3daaf 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1,11 +1,12 @@ /* Abstract Object Interface (many thanks to Jim Fulton) */ #include "Python.h" -#include "pycore_unionobject.h" // _Py_UnionType && _Py_Union() #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _Py_EnterRecursiveCall() +#include "pycore_object.h" // _Py_CheckSlotResult() #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_unionobject.h" // _Py_UnionType && _Py_Union() #include #include // offsetof() #include "longintrepr.h" @@ -61,7 +62,7 @@ PyObject_Size(PyObject *o) m = Py_TYPE(o)->tp_as_sequence; if (m && m->sq_length) { Py_ssize_t len = m->sq_length(o); - assert(len >= 0 || PyErr_Occurred()); + assert(_Py_CheckSlotResult(o, "__len__", len >= 0)); return len; } @@ -160,7 +161,7 @@ PyObject_GetItem(PyObject *o, PyObject *key) m = Py_TYPE(o)->tp_as_mapping; if (m && m->mp_subscript) { PyObject *item = m->mp_subscript(o, key); - assert((item != NULL) ^ (PyErr_Occurred() != NULL)); + assert(_Py_CheckSlotResult(o, "__getitem__", item != NULL)); return item; } @@ -1564,7 +1565,7 @@ PySequence_Size(PyObject *s) m = Py_TYPE(s)->tp_as_sequence; if (m && m->sq_length) { Py_ssize_t len = m->sq_length(s); - assert(len >= 0 || PyErr_Occurred()); + assert(_Py_CheckSlotResult(s, "__len__", len >= 0)); return len; } @@ -1708,8 +1709,8 @@ PySequence_GetItem(PyObject *s, Py_ssize_t i) if (i < 0) { if (m->sq_length) { Py_ssize_t l = (*m->sq_length)(s); + assert(_Py_CheckSlotResult(s, "__len__", l >= 0)); if (l < 0) { - assert(PyErr_Occurred()); return NULL; } i += l; @@ -1762,8 +1763,8 @@ PySequence_SetItem(PyObject *s, Py_ssize_t i, PyObject *o) if (i < 0) { if (m->sq_length) { Py_ssize_t l = (*m->sq_length)(s); + assert(_Py_CheckSlotResult(s, "__len__", l >= 0)); if (l < 0) { - assert(PyErr_Occurred()); return -1; } i += l; @@ -1795,8 +1796,8 @@ PySequence_DelItem(PyObject *s, Py_ssize_t i) if (i < 0) { if (m->sq_length) { Py_ssize_t l = (*m->sq_length)(s); + assert(_Py_CheckSlotResult(s, "__len__", l >= 0)); if (l < 0) { - assert(PyErr_Occurred()); return -1; } i += l; @@ -2145,7 +2146,7 @@ PyMapping_Size(PyObject *o) m = Py_TYPE(o)->tp_as_mapping; if (m && m->mp_length) { Py_ssize_t len = m->mp_length(o); - assert(len >= 0 || PyErr_Occurred()); + assert(_Py_CheckSlotResult(o, "__len__", len >= 0)); return len; } diff --git a/Objects/call.c b/Objects/call.c index 35b06a9b9c4d73..1fb85efab6169f 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -39,16 +39,16 @@ _Py_CheckFunctionResult(PyThreadState *tstate, PyObject *callable, if (!_PyErr_Occurred(tstate)) { if (callable) _PyErr_Format(tstate, PyExc_SystemError, - "%R returned NULL without setting an error", + "%R returned NULL without setting an exception", callable); else _PyErr_Format(tstate, PyExc_SystemError, - "%s returned NULL without setting an error", + "%s returned NULL without setting an exception", where); #ifdef Py_DEBUG /* Ensure that the bug is caught in debug mode. Py_FatalError() logs the SystemError exception raised above. */ - Py_FatalError("a function returned NULL without setting an error"); + Py_FatalError("a function returned NULL without setting an exception"); #endif return NULL; } @@ -60,17 +60,17 @@ _Py_CheckFunctionResult(PyThreadState *tstate, PyObject *callable, if (callable) { _PyErr_FormatFromCauseTstate( tstate, PyExc_SystemError, - "%R returned a result with an error set", callable); + "%R returned a result with an exception set", callable); } else { _PyErr_FormatFromCauseTstate( tstate, PyExc_SystemError, - "%s returned a result with an error set", where); + "%s returned a result with an exception set", where); } #ifdef Py_DEBUG /* Ensure that the bug is caught in debug mode. Py_FatalError() logs the SystemError exception raised above. */ - Py_FatalError("a function returned a result with an error set"); + Py_FatalError("a function returned a result with an exception set"); #endif return NULL; } @@ -79,6 +79,30 @@ _Py_CheckFunctionResult(PyThreadState *tstate, PyObject *callable, } +int +_Py_CheckSlotResult(PyObject *obj, const char *slot_name, int success) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (!success) { + if (!_PyErr_Occurred(tstate)) { + _Py_FatalErrorFormat(__func__, + "Slot %s of type %s failed " + "without setting an exception", + slot_name, Py_TYPE(obj)->tp_name); + } + } + else { + if (_PyErr_Occurred(tstate)) { + _Py_FatalErrorFormat(__func__, + "Slot %s of type %s succeeded " + "with an exception set", + slot_name, Py_TYPE(obj)->tp_name); + } + } + return 1; +} + + /* --- Core PyObject call functions ------------------------------- */ /* Call a callable Python object without any arguments */ diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 6590387dac531b..e57ea86e7694ce 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -35,6 +35,19 @@ PyTypeObject PyModuleDef_Type = { }; +int +_PyModule_IsExtension(PyObject *obj) +{ + if (!PyModule_Check(obj)) { + return 0; + } + PyModuleObject *module = (PyModuleObject*)obj; + + struct PyModuleDef *def = module->md_def; + return (def != NULL && def->m_methods != NULL); +} + + PyObject* PyModuleDef_Init(struct PyModuleDef* def) { diff --git a/PC/python3dll.c b/PC/python3dll.c index 683bba3a9364d1..542853abc894d5 100644 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -459,7 +459,6 @@ EXPORT_FUNC(PyOS_CheckStack) EXPORT_FUNC(PyOS_double_to_string) EXPORT_FUNC(PyOS_FSPath) EXPORT_FUNC(PyOS_getsig) -EXPORT_FUNC(PyOS_InitInterrupts) EXPORT_FUNC(PyOS_InterruptOccurred) EXPORT_FUNC(PyOS_mystricmp) EXPORT_FUNC(PyOS_mystrnicmp) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index fd27dea9daec71..f172f2a5786c63 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -291,6 +291,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 75a653dcbdab29..3bafdb8d297117 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -360,6 +360,9 @@ Python + + Python + Python diff --git a/Parser/pegen.c b/Parser/pegen.c index 188fd282b76043..0e7f86bc99e451 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -265,6 +265,16 @@ raise_decode_error(Parser *p) return -1; } +static inline void +raise_unclosed_parentheses_error(Parser *p) { + int error_lineno = p->tok->parenlinenostack[p->tok->level-1]; + int error_col = p->tok->parencolstack[p->tok->level-1]; + RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, + error_lineno, error_col, + "'%c' was never closed", + p->tok->parenstack[p->tok->level-1]); +} + static void raise_tokenizer_init_error(PyObject *filename) { @@ -317,14 +327,12 @@ tokenizer_error(Parser *p) case E_TOKEN: msg = "invalid token"; break; - case E_EOFS: - RAISE_SYNTAX_ERROR("EOF while scanning triple-quoted string literal"); - return -1; - case E_EOLS: - RAISE_SYNTAX_ERROR("EOL while scanning string literal"); - return -1; case E_EOF: - RAISE_SYNTAX_ERROR("unexpected EOF while parsing"); + if (p->tok->level) { + raise_unclosed_parentheses_error(p); + } else { + RAISE_SYNTAX_ERROR("unexpected EOF while parsing"); + } return -1; case E_DEDENT: RAISE_INDENTATION_ERROR("unindent does not match any outer indentation level"); @@ -380,6 +388,27 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...) return NULL; } +static PyObject * +get_error_line(Parser *p, Py_ssize_t lineno) +{ + /* If p->tok->fp == NULL, then we're parsing from a string, which means that + the whole source is stored in p->tok->str. If not, then we're parsing + from the REPL, so the source lines of the current (multi-line) statement + are stored in p->tok->stdin_content */ + assert(p->tok->fp == NULL || p->tok->fp == stdin); + + char *cur_line = p->tok->fp == NULL ? p->tok->str : p->tok->stdin_content; + for (int i = 0; i < lineno - 1; i++) { + cur_line = strchr(cur_line, '\n') + 1; + } + + char *next_newline; + if ((next_newline = strchr(cur_line, '\n')) == NULL) { // This is the last line + next_newline = cur_line + strlen(cur_line); + } + return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace"); +} + void * _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, Py_ssize_t lineno, Py_ssize_t col_offset, @@ -416,8 +445,22 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, } if (!error_line) { - Py_ssize_t size = p->tok->inp - p->tok->buf; - error_line = PyUnicode_DecodeUTF8(p->tok->buf, size, "replace"); + /* PyErr_ProgramTextObject was not called or returned NULL. If it was not called, + then we need to find the error line from some other source, because + p->start_rule != Py_file_input. If it returned NULL, then it either unexpectedly + failed or we're parsing from a string or the REPL. There's a third edge case where + we're actually parsing from a file, which has an E_EOF SyntaxError and in that case + `PyErr_ProgramTextObject` fails because lineno points to last_file_line + 1, which + does not physically exist */ + assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF); + + if (p->tok->lineno == lineno) { + Py_ssize_t size = p->tok->inp - p->tok->buf; + error_line = PyUnicode_DecodeUTF8(p->tok->buf, size, "replace"); + } + else { + error_line = get_error_line(p, lineno); + } if (!error_line) { goto error; } @@ -1116,6 +1159,45 @@ reset_parser_state(Parser *p) p->call_invalid_rules = 1; } +static int +_PyPegen_check_tokenizer_errors(Parser *p) { + // Tokenize the whole input to see if there are any tokenization + // errors such as mistmatching parentheses. These will get priority + // over generic syntax errors only if the line number of the error is + // before the one that we had for the generic error. + + // We don't want to tokenize to the end for interactive input + if (p->tok->prompt != NULL) { + return 0; + } + + Token *current_token = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1]; + Py_ssize_t current_err_line = current_token->lineno; + + for (;;) { + const char *start; + const char *end; + switch (PyTokenizer_Get(p->tok, &start, &end)) { + case ERRORTOKEN: + if (p->tok->level != 0) { + int error_lineno = p->tok->parenlinenostack[p->tok->level-1]; + if (current_err_line > error_lineno) { + raise_unclosed_parentheses_error(p); + return -1; + } + } + break; + case ENDMARKER: + break; + default: + continue; + } + break; + } + + return 0; +} + void * _PyPegen_run_parser(Parser *p) { @@ -1129,8 +1211,12 @@ _PyPegen_run_parser(Parser *p) if (p->fill == 0) { RAISE_SYNTAX_ERROR("error at start before reading any input"); } - else if (p->tok->done == E_EOF) { - RAISE_SYNTAX_ERROR("unexpected EOF while parsing"); + else if (p->tok->done == E_EOF) { + if (p->tok->level) { + raise_unclosed_parentheses_error(p); + } else { + RAISE_SYNTAX_ERROR("unexpected EOF while parsing"); + } } else { if (p->tokens[p->fill-1]->type == INDENT) { @@ -1141,6 +1227,9 @@ _PyPegen_run_parser(Parser *p) } else { RAISE_SYNTAX_ERROR("invalid syntax"); + // _PyPegen_check_tokenizer_errors will override the existing + // generic SyntaxError we just raised if errors are found. + _PyPegen_check_tokenizer_errors(p); } } return NULL; diff --git a/Parser/string_parser.c b/Parser/string_parser.c index a41f41ce2784d1..0f3665c3453e2e 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -250,7 +250,7 @@ _PyPegen_parsestr(Parser *p, int *bytesmode, int *rawmode, PyObject **result, if (Py_CHARMASK(*ch) >= 0x80) { RAISE_SYNTAX_ERROR( "bytes can only contain ASCII " - "literal characters."); + "literal characters"); return -1; } } diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 96539bd556529a..d9334aaf148ba2 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -64,7 +64,6 @@ tok_new(void) tok->tabsize = TABSIZE; tok->indent = 0; tok->indstack[0] = 0; - tok->atbol = 1; tok->pendin = 0; tok->prompt = tok->nextprompt = NULL; @@ -81,6 +80,7 @@ tok_new(void) tok->decoding_readline = NULL; tok->decoding_buffer = NULL; tok->type_comments = 0; + tok->stdin_content = NULL; tok->async_hacks = 0; tok->async_def = 0; @@ -816,6 +816,8 @@ PyTokenizer_Free(struct tok_state *tok) PyMem_Free(tok->buf); if (tok->input) PyMem_Free(tok->input); + if (tok->stdin_content) + PyMem_Free(tok->stdin_content); PyMem_Free(tok); } @@ -856,6 +858,24 @@ tok_nextc(struct tok_state *tok) if (translated == NULL) return EOF; newtok = translated; + if (tok->stdin_content == NULL) { + tok->stdin_content = PyMem_Malloc(strlen(translated) + 1); + if (tok->stdin_content == NULL) { + tok->done = E_NOMEM; + return EOF; + } + sprintf(tok->stdin_content, "%s", translated); + } + else { + char *new_str = PyMem_Malloc(strlen(tok->stdin_content) + strlen(translated) + 1); + if (new_str == NULL) { + tok->done = E_NOMEM; + return EOF; + } + sprintf(new_str, "%s%s", tok->stdin_content, translated); + PyMem_Free(tok->stdin_content); + tok->stdin_content = new_str; + } } if (tok->encoding && newtok && *newtok) { /* Recode to UTF-8 */ @@ -1375,6 +1395,9 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) /* Check for EOF and errors now */ if (c == EOF) { + if (tok->level) { + return ERRORTOKEN; + } return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN; } @@ -1716,20 +1739,26 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) /* Get rest of string */ while (end_quote_size != quote_size) { c = tok_nextc(tok); - if (c == EOF) { + if (c == EOF || (quote_size == 1 && c == '\n')) { + // shift the tok_state's location into + // the start of string, and report the error + // from the initial quote character + tok->cur = (char *)tok->start; + tok->cur++; + tok->line_start = tok->multi_line_start; + int start = tok->lineno; + tok->lineno = tok->first_lineno; + if (quote_size == 3) { - tok->done = E_EOFS; + return syntaxerror(tok, + "unterminated triple-quoted string literal" + " (detected at line %d)", start); } else { - tok->done = E_EOLS; + return syntaxerror(tok, + "unterminated string literal (detected at" + " line %d)", start); } - tok->cur = tok->inp; - return ERRORTOKEN; - } - if (quote_size == 1 && c == '\n') { - tok->done = E_EOLS; - tok->cur = tok->inp; - return ERRORTOKEN; } if (c == quote) { end_quote_size += 1; @@ -1797,6 +1826,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) } tok->parenstack[tok->level] = c; tok->parenlinenostack[tok->level] = tok->lineno; + tok->parencolstack[tok->level] = (int)(tok->start - tok->line_start); tok->level++; break; case ')': diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 5660ea38e9443d..56074b61ae100e 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -37,6 +37,7 @@ struct tok_state { int atbol; /* Nonzero if at begin of new line */ int pendin; /* Pending indents (if > 0) or dedents (if < 0) */ const char *prompt, *nextprompt; /* For interactive prompting */ + char *stdin_content; int lineno; /* Current line number */ int first_lineno; /* First line of a single line or multi line string expression (cf. issue 16806) */ @@ -44,6 +45,7 @@ struct tok_state { /* Used to allow free continuations inside them */ char parenstack[MAXLEVEL]; int parenlinenostack[MAXLEVEL]; + int parencolstack[MAXLEVEL]; PyObject *filename; /* Stuff for checking on different tab sizes */ int altindstack[MAXINDENT]; /* Stack of alternate indents */ diff --git a/Python/compile.c b/Python/compile.c index d373d8a4061a3c..223c63637ff411 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -898,8 +898,6 @@ compiler_next_instr(basicblock *b) * 1 -- when jump * -1 -- maximal */ -/* XXX Make the stack effect of WITH_CLEANUP_START and - WITH_CLEANUP_FINISH deterministic. */ static int stack_effect(int opcode, int oparg, int jump) { @@ -2371,6 +2369,8 @@ compiler_class(struct compiler *c, stmt_ty s) compiler_exit_scope(c); return 0; } + /* The following code is artificial */ + c->u->u_lineno = -1; /* Return __classcell__ if it is referenced, otherwise return None */ if (c->u->u_ste->ste_needs_class_closure) { /* Store __classcell__ into class namespace & return it */ diff --git a/Python/fileutils.c b/Python/fileutils.c index 8dc90fbe2b2e71..f2b4681ea849c5 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1234,6 +1234,13 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) return 0; } +#ifdef __linux__ + if (errno == EBADF) { + // On Linux, ioctl(FIOCLEX) will fail with EBADF for O_PATH file descriptors + // Fall through to the fcntl() path + } + else +#endif if (errno != ENOTTY && errno != EACCES) { if (raise) PyErr_SetFromErrno(PyExc_OSError); diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c index ed95f267d476c7..5ccf9d303e34ec 100644 --- a/Python/formatter_unicode.c +++ b/Python/formatter_unicode.c @@ -219,7 +219,7 @@ parse_internal_render_format_spec(PyObject *format_spec, /* The special case for 0-padding (backwards compat) */ if (!fill_char_specified && end-pos >= 1 && READ_spec(pos) == '0') { format->fill_char = '0'; - if (!align_specified) { + if (!align_specified && default_align == '>') { format->align = '='; } ++pos; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 9828dffad5c630..bf5dcdd107e20f 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -38,6 +38,9 @@ #endif +#define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) + + _Py_IDENTIFIER(flush); _Py_IDENTIFIER(name); _Py_IDENTIFIER(stdin); @@ -2348,8 +2351,7 @@ static void _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, PyThreadState *tstate) { - fputc('\n', stderr); - fflush(stderr); + PUTS(fd, "\n"); /* display the current Python stack */ _Py_DumpTracebackThreads(fd, interp, tstate); @@ -2451,30 +2453,31 @@ fatal_output_debug(const char *msg) static void -fatal_error_dump_runtime(FILE *stream, _PyRuntimeState *runtime) +fatal_error_dump_runtime(int fd, _PyRuntimeState *runtime) { - fprintf(stream, "Python runtime state: "); + PUTS(fd, "Python runtime state: "); PyThreadState *finalizing = _PyRuntimeState_GetFinalizing(runtime); if (finalizing) { - fprintf(stream, "finalizing (tstate=%p)", finalizing); + PUTS(fd, "finalizing (tstate=0x"); + _Py_DumpHexadecimal(fd, (uintptr_t)finalizing, sizeof(finalizing) * 2); + PUTS(fd, ")"); } else if (runtime->initialized) { - fprintf(stream, "initialized"); + PUTS(fd, "initialized"); } else if (runtime->core_initialized) { - fprintf(stream, "core initialized"); + PUTS(fd, "core initialized"); } else if (runtime->preinitialized) { - fprintf(stream, "preinitialized"); + PUTS(fd, "preinitialized"); } else if (runtime->preinitializing) { - fprintf(stream, "preinitializing"); + PUTS(fd, "preinitializing"); } else { - fprintf(stream, "unknown"); + PUTS(fd, "unknown"); } - fprintf(stream, "\n"); - fflush(stream); + PUTS(fd, "\n"); } @@ -2493,11 +2496,102 @@ fatal_error_exit(int status) } +// Dump the list of extension modules of sys.modules, excluding stdlib modules +// (sys.stdlib_module_names), into fd file descriptor. +// +// This function is called by a signal handler in faulthandler: avoid memory +// allocations and keep the implementation simple. For example, the list is not +// sorted on purpose. +void +_Py_DumpExtensionModules(int fd, PyInterpreterState *interp) +{ + if (interp == NULL) { + return; + } + PyObject *modules = interp->modules; + if (modules == NULL || !PyDict_Check(modules)) { + return; + } + + Py_ssize_t pos; + PyObject *key, *value; + + // Avoid PyDict_GetItemString() which calls PyUnicode_FromString(), + // memory cannot be allocated on the heap in a signal handler. + // Iterate on the dict instead. + PyObject *stdlib_module_names = NULL; + pos = 0; + while (PyDict_Next(interp->sysdict, &pos, &key, &value)) { + if (PyUnicode_Check(key) + && PyUnicode_CompareWithASCIIString(key, "stdlib_module_names") == 0) { + stdlib_module_names = value; + break; + } + } + // If we failed to get sys.stdlib_module_names or it's not a frozenset, + // don't exclude stdlib modules. + if (stdlib_module_names != NULL && !PyFrozenSet_Check(stdlib_module_names)) { + stdlib_module_names = NULL; + } + + // List extensions + int header = 1; + Py_ssize_t count = 0; + pos = 0; + while (PyDict_Next(modules, &pos, &key, &value)) { + if (!PyUnicode_Check(key)) { + continue; + } + if (!_PyModule_IsExtension(value)) { + continue; + } + // Use the module name from the sys.modules key, + // don't attempt to get the module object name. + if (stdlib_module_names != NULL) { + int is_stdlib_ext = 0; + + Py_ssize_t i = 0; + PyObject *item; + Py_hash_t hash; + while (_PySet_NextEntry(stdlib_module_names, &i, &item, &hash)) { + if (PyUnicode_Check(item) + && PyUnicode_Compare(key, item) == 0) + { + is_stdlib_ext = 1; + break; + } + } + if (is_stdlib_ext) { + // Ignore stdlib extension + continue; + } + } + + if (header) { + PUTS(fd, "\nExtension modules: "); + header = 0; + } + else { + PUTS(fd, ", "); + } + + _Py_DumpASCII(fd, key); + count++; + } + + if (count) { + PUTS(fd, " (total: "); + _Py_DumpDecimal(fd, count); + PUTS(fd, ")"); + PUTS(fd, "\n"); + } +} + + static void _Py_NO_RETURN -fatal_error(FILE *stream, int header, const char *prefix, const char *msg, +fatal_error(int fd, int header, const char *prefix, const char *msg, int status) { - const int fd = fileno(stream); static int reentrant = 0; if (reentrant) { @@ -2508,29 +2602,22 @@ fatal_error(FILE *stream, int header, const char *prefix, const char *msg, reentrant = 1; if (header) { - fprintf(stream, "Fatal Python error: "); + PUTS(fd, "Fatal Python error: "); if (prefix) { - fputs(prefix, stream); - fputs(": ", stream); + PUTS(fd, prefix); + PUTS(fd, ": "); } if (msg) { - fputs(msg, stream); + PUTS(fd, msg); } else { - fprintf(stream, ""); + PUTS(fd, ""); } - fputs("\n", stream); - fflush(stream); + PUTS(fd, "\n"); } _PyRuntimeState *runtime = &_PyRuntime; - fatal_error_dump_runtime(stream, runtime); - - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); - PyInterpreterState *interp = NULL; - if (tstate != NULL) { - interp = tstate->interp; - } + fatal_error_dump_runtime(fd, runtime); /* Check if the current thread has a Python thread state and holds the GIL. @@ -2540,8 +2627,17 @@ fatal_error(FILE *stream, int header, const char *prefix, const char *msg, tss_tstate != tstate if the current Python thread does not hold the GIL. */ + PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyInterpreterState *interp = NULL; PyThreadState *tss_tstate = PyGILState_GetThisThreadState(); + if (tstate != NULL) { + interp = tstate->interp; + } + else if (tss_tstate != NULL) { + interp = tss_tstate->interp; + } int has_tstate_and_gil = (tss_tstate != NULL && tss_tstate == tstate); + if (has_tstate_and_gil) { /* If an exception is set, print the exception with its traceback */ if (!_Py_FatalError_PrintExc(tss_tstate)) { @@ -2553,6 +2649,8 @@ fatal_error(FILE *stream, int header, const char *prefix, const char *msg, _Py_FatalError_DumpTracebacks(fd, interp, tss_tstate); } + _Py_DumpExtensionModules(fd, interp); + /* The main purpose of faulthandler is to display the traceback. This function already did its best to display a traceback. Disable faulthandler to prevent writing a second traceback @@ -2578,14 +2676,14 @@ fatal_error(FILE *stream, int header, const char *prefix, const char *msg, void _Py_NO_RETURN Py_FatalError(const char *msg) { - fatal_error(stderr, 1, NULL, msg, -1); + fatal_error(fileno(stderr), 1, NULL, msg, -1); } void _Py_NO_RETURN _Py_FatalErrorFunc(const char *func, const char *msg) { - fatal_error(stderr, 1, func, msg, -1); + fatal_error(fileno(stderr), 1, func, msg, -1); } @@ -2600,12 +2698,12 @@ _Py_FatalErrorFormat(const char *func, const char *format, ...) reentrant = 1; FILE *stream = stderr; - fprintf(stream, "Fatal Python error: "); + const int fd = fileno(stream); + PUTS(fd, "Fatal Python error: "); if (func) { - fputs(func, stream); - fputs(": ", stream); + PUTS(fd, func); + PUTS(fd, ": "); } - fflush(stream); va_list vargs; #ifdef HAVE_STDARG_PROTOTYPES @@ -2619,7 +2717,7 @@ _Py_FatalErrorFormat(const char *func, const char *format, ...) fputs("\n", stream); fflush(stream); - fatal_error(stream, 0, NULL, NULL, -1); + fatal_error(fd, 0, NULL, NULL, -1); } @@ -2630,7 +2728,7 @@ Py_ExitStatusException(PyStatus status) exit(status.exitcode); } else if (_PyStatus_IS_ERROR(status)) { - fatal_error(stderr, 1, status.func, status.err_msg, 1); + fatal_error(fileno(stderr), 1, status.func, status.err_msg, 1); } else { Py_FatalError("Py_ExitStatusException() must not be called on success"); diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h new file mode 100644 index 00000000000000..01aa6753e42015 --- /dev/null +++ b/Python/stdlib_module_names.h @@ -0,0 +1,305 @@ +// Auto-generated by Tools/scripts/generate_stdlib_module_names.py. +// List used to create sys.stdlib_module_names. + +static const char* _Py_stdlib_module_names[] = { +"__future__", +"_abc", +"_aix_support", +"_ast", +"_asyncio", +"_bisect", +"_blake2", +"_bootsubprocess", +"_bz2", +"_codecs", +"_codecs_cn", +"_codecs_hk", +"_codecs_iso2022", +"_codecs_jp", +"_codecs_kr", +"_codecs_tw", +"_collections", +"_collections_abc", +"_compat_pickle", +"_compression", +"_contextvars", +"_crypt", +"_csv", +"_ctypes", +"_curses", +"_curses_panel", +"_datetime", +"_dbm", +"_decimal", +"_elementtree", +"_functools", +"_gdbm", +"_hashlib", +"_heapq", +"_imp", +"_io", +"_json", +"_locale", +"_lsprof", +"_lzma", +"_markupbase", +"_md5", +"_msi", +"_multibytecodec", +"_multiprocessing", +"_opcode", +"_operator", +"_osx_support", +"_pickle", +"_posixshmem", +"_posixsubprocess", +"_py_abc", +"_pydecimal", +"_pyio", +"_queue", +"_random", +"_sha1", +"_sha256", +"_sha3", +"_sha512", +"_signal", +"_sitebuiltins", +"_socket", +"_sqlite3", +"_sre", +"_ssl", +"_stat", +"_statistics", +"_string", +"_strptime", +"_struct", +"_symtable", +"_thread", +"_threading_local", +"_tkinter", +"_tracemalloc", +"_uuid", +"_warnings", +"_weakref", +"_weakrefset", +"_winapi", +"_xxsubinterpreters", +"_zoneinfo", +"abc", +"aifc", +"antigravity", +"argparse", +"array", +"ast", +"asynchat", +"asyncio", +"asyncore", +"atexit", +"audioop", +"base64", +"bdb", +"binascii", +"binhex", +"bisect", +"builtins", +"bz2", +"cProfile", +"calendar", +"cgi", +"cgitb", +"chunk", +"cmath", +"cmd", +"code", +"codecs", +"codeop", +"collections", +"colorsys", +"compileall", +"concurrent", +"configparser", +"contextlib", +"contextvars", +"copy", +"copyreg", +"crypt", +"csv", +"ctypes", +"curses", +"dataclasses", +"datetime", +"dbm", +"decimal", +"difflib", +"dis", +"distutils", +"doctest", +"email", +"encodings", +"ensurepip", +"enum", +"errno", +"faulthandler", +"fcntl", +"filecmp", +"fileinput", +"fnmatch", +"fractions", +"ftplib", +"functools", +"gc", +"genericpath", +"getopt", +"getpass", +"gettext", +"glob", +"graphlib", +"grp", +"gzip", +"hashlib", +"heapq", +"hmac", +"html", +"http", +"idlelib", +"imaplib", +"imghdr", +"imp", +"importlib", +"inspect", +"io", +"ipaddress", +"itertools", +"json", +"keyword", +"lib2to3", +"linecache", +"locale", +"logging", +"lzma", +"mailbox", +"mailcap", +"marshal", +"math", +"mimetypes", +"mmap", +"modulefinder", +"msilib", +"msvcrt", +"multiprocessing", +"netrc", +"nis", +"nntplib", +"nt", +"ntpath", +"nturl2path", +"numbers", +"opcode", +"operator", +"optparse", +"os", +"ossaudiodev", +"pathlib", +"pdb", +"pickle", +"pickletools", +"pipes", +"pkgutil", +"platform", +"plistlib", +"poplib", +"posix", +"posixpath", +"pprint", +"profile", +"pstats", +"pty", +"pwd", +"py_compile", +"pyclbr", +"pydoc", +"pydoc_data", +"pyexpat", +"queue", +"quopri", +"random", +"re", +"readline", +"reprlib", +"resource", +"rlcompleter", +"runpy", +"sched", +"secrets", +"select", +"selectors", +"shelve", +"shlex", +"shutil", +"signal", +"site", +"smtpd", +"smtplib", +"sndhdr", +"socket", +"socketserver", +"spwd", +"sqlite3", +"sre_compile", +"sre_constants", +"sre_parse", +"ssl", +"stat", +"statistics", +"string", +"stringprep", +"struct", +"subprocess", +"sunau", +"symtable", +"sys", +"sysconfig", +"syslog", +"tabnanny", +"tarfile", +"telnetlib", +"tempfile", +"termios", +"textwrap", +"this", +"threading", +"time", +"timeit", +"tkinter", +"token", +"tokenize", +"trace", +"traceback", +"tracemalloc", +"tty", +"turtle", +"turtledemo", +"types", +"typing", +"unicodedata", +"unittest", +"urllib", +"uu", +"uuid", +"venv", +"warnings", +"wave", +"weakref", +"webbrowser", +"winreg", +"winsound", +"wsgiref", +"xdrlib", +"xml", +"xmlrpc", +"zipapp", +"zipfile", +"zipimport", +"zlib", +"zoneinfo", +}; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 720532eade29b3..b9349effe3c87b 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -29,6 +29,7 @@ Data members: #include "frameobject.h" // PyFrame_GetBack() #include "pydtrace.h" #include "osdefs.h" // DELIM +#include "stdlib_module_names.h" // _Py_stdlib_module_names #include #ifdef MS_WINDOWS @@ -2020,33 +2021,63 @@ static PyMethodDef sys_methods[] = { {NULL, NULL} /* sentinel */ }; + static PyObject * list_builtin_module_names(void) { PyObject *list = PyList_New(0); - int i; - if (list == NULL) + if (list == NULL) { return NULL; - for (i = 0; PyImport_Inittab[i].name != NULL; i++) { - PyObject *name = PyUnicode_FromString( - PyImport_Inittab[i].name); - if (name == NULL) - break; - PyList_Append(list, name); + } + for (Py_ssize_t i = 0; PyImport_Inittab[i].name != NULL; i++) { + PyObject *name = PyUnicode_FromString(PyImport_Inittab[i].name); + if (name == NULL) { + goto error; + } + if (PyList_Append(list, name) < 0) { + Py_DECREF(name); + goto error; + } Py_DECREF(name); } if (PyList_Sort(list) != 0) { - Py_DECREF(list); - list = NULL; + goto error; + } + PyObject *tuple = PyList_AsTuple(list); + Py_DECREF(list); + return tuple; + +error: + Py_DECREF(list); + return NULL; +} + + +static PyObject * +list_stdlib_module_names(void) +{ + Py_ssize_t len = Py_ARRAY_LENGTH(_Py_stdlib_module_names); + PyObject *names = PyTuple_New(len); + if (names == NULL) { + return NULL; } - if (list) { - PyObject *v = PyList_AsTuple(list); - Py_DECREF(list); - list = v; + + for (Py_ssize_t i = 0; i < len; i++) { + PyObject *name = PyUnicode_FromString(_Py_stdlib_module_names[i]); + if (name == NULL) { + Py_DECREF(names); + return NULL; + } + PyTuple_SET_ITEM(names, i, name); } - return list; + + PyObject *set = PyObject_CallFunction((PyObject *)&PyFrozenSet_Type, + "(O)", names); + Py_DECREF(names); + return set; } + /* Pre-initialization support for sys.warnoptions and sys._xoptions * * Modern internal code paths: @@ -2753,6 +2784,7 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) SET_SYS("hash_info", get_hash_info(tstate)); SET_SYS("maxunicode", PyLong_FromLong(0x10FFFF)); SET_SYS("builtin_module_names", list_builtin_module_names()); + SET_SYS("stdlib_module_names", list_stdlib_module_names()); #if PY_BIG_ENDIAN SET_SYS_FROM_STRING("byteorder", "big"); #else diff --git a/Python/traceback.c b/Python/traceback.c index b82cfd3665ce1d..9363d4eb1b5b74 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -628,12 +628,12 @@ PyTraceBack_Print(PyObject *v, PyObject *f) This function is signal safe. */ void -_Py_DumpDecimal(int fd, unsigned long value) +_Py_DumpDecimal(int fd, size_t value) { /* maximum number of characters required for output of %lld or %p. We need at most ceil(log10(256)*SIZEOF_LONG_LONG) digits, plus 1 for the null byte. 53/22 is an upper bound for log10(256). */ - char buffer[1 + (sizeof(unsigned long)*53-1) / 22 + 1]; + char buffer[1 + (sizeof(size_t)*53-1) / 22 + 1]; char *ptr, *end; end = &buffer[Py_ARRAY_LENGTH(buffer) - 1]; @@ -649,15 +649,12 @@ _Py_DumpDecimal(int fd, unsigned long value) _Py_write_noraise(fd, ptr, end - ptr); } -/* Format an integer in range [0; 0xffffffff] to hexadecimal of 'width' digits, - and write it into the file fd. - - This function is signal safe. */ - +/* Format an integer as hexadecimal with width digits into fd file descriptor. + The function is signal safe. */ void -_Py_DumpHexadecimal(int fd, unsigned long value, Py_ssize_t width) +_Py_DumpHexadecimal(int fd, uintptr_t value, Py_ssize_t width) { - char buffer[sizeof(unsigned long) * 2 + 1], *ptr, *end; + char buffer[sizeof(uintptr_t) * 2 + 1], *ptr, *end; const Py_ssize_t size = Py_ARRAY_LENGTH(buffer) - 1; if (width > size) @@ -770,7 +767,7 @@ dump_frame(int fd, PyFrameObject *frame) int lineno = PyCode_Addr2Line(code, frame->f_lasti); PUTS(fd, ", line "); if (lineno >= 0) { - _Py_DumpDecimal(fd, (unsigned long)lineno); + _Py_DumpDecimal(fd, (size_t)lineno); } else { PUTS(fd, "???"); diff --git a/Tools/scripts/generate_stdlib_module_names.py b/Tools/scripts/generate_stdlib_module_names.py new file mode 100644 index 00000000000000..379b262e822db7 --- /dev/null +++ b/Tools/scripts/generate_stdlib_module_names.py @@ -0,0 +1,139 @@ +# This script lists the names of standard library modules +# to update Python/stdlib_mod_names.h +import os.path +import re +import subprocess +import sys +import sysconfig + + +SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) +STDLIB_PATH = os.path.join(SRC_DIR, 'Lib') +MODULES_SETUP = os.path.join(SRC_DIR, 'Modules', 'Setup') +SETUP_PY = os.path.join(SRC_DIR, 'setup.py') + +IGNORE = { + '__init__', + '__pycache__', + 'site-packages', + + # test modules + '__phello__.foo', + '_ctypes_test', + '_testbuffer', + '_testcapi', + '_testconsole', + '_testimportmultiple', + '_testinternalcapi', + '_testmultiphase', + '_xxtestfuzz', + 'distutils.tests', + 'idlelib.idle_test', + 'lib2to3.tests', + 'test', + 'xxlimited', + 'xxlimited_35', + 'xxsubtype', +} + +# Windows extension modules +WINDOWS_MODULES = ( + '_msi', + '_testconsole', + '_winapi', + 'msvcrt', + 'nt', + 'winreg', + 'winsound' +) + + +# Pure Python modules (Lib/*.py) +def list_python_modules(names): + for filename in os.listdir(STDLIB_PATH): + if not filename.endswith(".py"): + continue + name = filename.removesuffix(".py") + names.add(name) + + +# Packages in Lib/ +def list_packages(names): + for name in os.listdir(STDLIB_PATH): + if name in IGNORE: + continue + package_path = os.path.join(STDLIB_PATH, name) + if not os.path.isdir(package_path): + continue + if any(package_file.endswith(".py") + for package_file in os.listdir(package_path)): + names.add(name) + + +# Extension modules built by setup.py +def list_setup_extensions(names): + cmd = [sys.executable, SETUP_PY, "-q", "build", "--list-module-names"] + output = subprocess.check_output(cmd) + output = output.decode("utf8") + extensions = output.splitlines() + names |= set(extensions) + + +# Built-in and extension modules built by Modules/Setup +def list_modules_setup_extensions(names): + assign_var = re.compile("^[A-Z]+=") + + with open(MODULES_SETUP, encoding="utf-8") as modules_fp: + for line in modules_fp: + # Strip comment + line = line.partition("#")[0] + line = line.rstrip() + if not line: + continue + if assign_var.match(line): + # Ignore "VAR=VALUE" + continue + if line in ("*disabled*", "*shared*"): + continue + parts = line.split() + if len(parts) < 2: + continue + # "errno errnomodule.c" => write "errno" + name = parts[0] + names.add(name) + + +def list_modules(): + names = set(sys.builtin_module_names) | set(WINDOWS_MODULES) + list_modules_setup_extensions(names) + list_setup_extensions(names) + list_packages(names) + list_python_modules(names) + names -= set(IGNORE) + return names + + +def write_modules(fp, names): + print("// Auto-generated by Tools/scripts/generate_stdlib_module_names.py.", + file=fp) + print("// List used to create sys.stdlib_module_names.", file=fp) + print(file=fp) + print("static const char* _Py_stdlib_module_names[] = {", file=fp) + for name in sorted(names): + print(f'"{name}",', file=fp) + print("};", file=fp) + + +def main(): + if not sysconfig.is_python_build(): + print(f"ERROR: {sys.executable} is not a Python build", + file=sys.stderr) + sys.exit(1) + + fp = sys.stdout + names = list_modules() + write_modules(fp, names) + + +if __name__ == "__main__": + main() diff --git a/configure b/configure index 1d81c00c6359c6..39fb15f5c7959d 100755 --- a/configure +++ b/configure @@ -630,6 +630,7 @@ OPENSSL_INCLUDES ENSUREPIP SRCDIRS THREADHEADERS +WHEEL_PKG_DIR LIBPL PY_ENABLE_SHARED PLATLIBDIR @@ -847,6 +848,7 @@ with_libm with_libc enable_big_digits with_platlibdir +with_wheel_pkg_dir with_computed_gotos with_ensurepip with_openssl @@ -1576,6 +1578,9 @@ Optional Packages: system-dependent) --with-platlibdir=DIRNAME Python library directory name (default is "lib") + --with-wheel-pkg-dir=PATH + Directory of wheel packages used by ensurepip + (default: none) --with-computed-gotos enable computed gotos in evaluation loop (enabled by default on supported compilers) --with-ensurepip[=install|upgrade|no] @@ -6527,7 +6532,7 @@ fi $as_echo_n "checking PROFILE_TASK... " >&6; } if test -z "$PROFILE_TASK" then - PROFILE_TASK='-m test --pgo' + PROFILE_TASK='-m test --pgo --timeout=$(TESTTIMEOUT)' fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PROFILE_TASK" >&5 $as_echo "$PROFILE_TASK" >&6; } @@ -15493,6 +15498,29 @@ else fi +# Check for --with-wheel-pkg-dir=PATH + +WHEEL_PKG_DIR="" +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-wheel-pkg-dir" >&5 +$as_echo_n "checking for --with-wheel-pkg-dir... " >&6; } + +# Check whether --with-wheel-pkg-dir was given. +if test "${with_wheel_pkg_dir+set}" = set; then : + withval=$with_wheel_pkg_dir; +if test -n "$withval"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + WHEEL_PKG_DIR="$withval" +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + # Check whether right shifting a negative integer extends the sign bit # or fills with zeros (like the Cray J90, according to Tim Peters). { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether right shift extends the sign bit" >&5 diff --git a/configure.ac b/configure.ac index 08c462ac9f629a..1f5a008388a1e3 100644 --- a/configure.ac +++ b/configure.ac @@ -1325,7 +1325,7 @@ AC_ARG_VAR(PROFILE_TASK, Python args for PGO generation task) AC_MSG_CHECKING(PROFILE_TASK) if test -z "$PROFILE_TASK" then - PROFILE_TASK='-m test --pgo' + PROFILE_TASK='-m test --pgo --timeout=$(TESTTIMEOUT)' fi AC_MSG_RESULT($PROFILE_TASK) @@ -4838,6 +4838,22 @@ else fi AC_SUBST(LIBPL) +# Check for --with-wheel-pkg-dir=PATH +AC_SUBST(WHEEL_PKG_DIR) +WHEEL_PKG_DIR="" +AC_MSG_CHECKING(for --with-wheel-pkg-dir) +AC_ARG_WITH(wheel-pkg-dir, + AS_HELP_STRING([--with-wheel-pkg-dir=PATH], + [Directory of wheel packages used by ensurepip (default: none)]), +[ +if test -n "$withval"; then + AC_MSG_RESULT(yes) + WHEEL_PKG_DIR="$withval" +else + AC_MSG_RESULT(no) +fi], +[AC_MSG_RESULT(no)]) + # Check whether right shifting a negative integer extends the sign bit # or fills with zeros (like the Cray J90, according to Tim Peters). AC_MSG_CHECKING(whether right shift extends the sign bit) diff --git a/setup.py b/setup.py index ddc0bd067d4e4f..8445546c0116ff 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,9 @@ # This global variable is used to hold the list of modules to be disabled. DISABLED_MODULE_LIST = [] +# --list-module-names option used by Tools/scripts/generate_module_names.py +LIST_MODULE_NAMES = False + def get_platform(): # Cross compiling @@ -447,12 +450,20 @@ def build_extensions(self): # Detect which modules should be compiled self.detect_modules() - self.remove_disabled() + if not LIST_MODULE_NAMES: + self.remove_disabled() self.update_sources_depends() mods_built, mods_disabled = self.remove_configured_extensions() self.set_compiler_executables() + if LIST_MODULE_NAMES: + for ext in self.extensions: + print(ext.name) + for name in self.missing: + print(name) + return + build_ext.build_extensions(self) if SUBPROCESS_BOOTSTRAP: @@ -671,6 +682,51 @@ def add_multiarch_paths(self): finally: os.unlink(tmpfile) + def add_wrcc_search_dirs(self): + # add library search path by wr-cc, the compiler wrapper + + def convert_mixed_path(path): + # convert path like C:\folder1\folder2/folder3/folder4 + # to msys style /c/folder1/folder2/folder3/folder4 + drive = path[0].lower() + left = path[2:].replace("\\", "/") + return "/" + drive + left + + def add_search_path(line): + # On Windows building machine, VxWorks does + # cross builds under msys2 environment. + pathsep = (";" if sys.platform == "msys" else ":") + for d in line.strip().split("=")[1].split(pathsep): + d = d.strip() + if sys.platform == "msys": + # On Windows building machine, compiler + # returns mixed style path like: + # C:\folder1\folder2/folder3/folder4 + d = convert_mixed_path(d) + d = os.path.normpath(d) + add_dir_to_list(self.compiler.library_dirs, d) + + cc = sysconfig.get_config_var('CC') + tmpfile = os.path.join(self.build_temp, 'wrccpaths') + os.makedirs(self.build_temp, exist_ok=True) + try: + ret = run_command('%s --print-search-dirs >%s' % (cc, tmpfile)) + if ret: + return + with open(tmpfile) as fp: + # Parse paths in libraries line. The line is like: + # On Linux, "libraries: = path1:path2:path3" + # On Windows, "libraries: = path1;path2;path3" + for line in fp: + if not line.startswith("libraries"): + continue + add_search_path(line) + finally: + try: + os.unlink(tmpfile) + except OSError: + pass + def add_cross_compiling_paths(self): cc = sysconfig.get_config_var('CC') tmpfile = os.path.join(self.build_temp, 'ccpaths') @@ -704,6 +760,9 @@ def add_cross_compiling_paths(self): finally: os.unlink(tmpfile) + if VXWORKS: + self.add_wrcc_search_dirs() + def add_ldflags_cppflags(self): # Add paths specified in the environment variables LDFLAGS and # CPPFLAGS for header and library files. @@ -1118,6 +1177,7 @@ def detect_crypt(self): # bpo-31904: crypt() function is not provided by VxWorks. # DES_crypt() OpenSSL provides is too weak to implement # the encryption. + self.missing.append('_crypt') return if self.compiler.find_library_file(self.lib_dirs, 'crypt'): @@ -1125,8 +1185,7 @@ def detect_crypt(self): else: libs = [] - self.add(Extension('_crypt', ['_cryptmodule.c'], - libraries=libs)) + self.add(Extension('_crypt', ['_cryptmodule.c'], libraries=libs)) def detect_socket(self): # socket(2) @@ -1735,27 +1794,29 @@ def detect_multiprocessing(self): if MS_WINDOWS: multiprocessing_srcs = ['_multiprocessing/multiprocessing.c', '_multiprocessing/semaphore.c'] - else: multiprocessing_srcs = ['_multiprocessing/multiprocessing.c'] if (sysconfig.get_config_var('HAVE_SEM_OPEN') and not sysconfig.get_config_var('POSIX_SEMAPHORES_NOT_ENABLED')): multiprocessing_srcs.append('_multiprocessing/semaphore.c') - if (sysconfig.get_config_var('HAVE_SHM_OPEN') and - sysconfig.get_config_var('HAVE_SHM_UNLINK')): - posixshmem_srcs = ['_multiprocessing/posixshmem.c'] - libs = [] - if sysconfig.get_config_var('SHM_NEEDS_LIBRT'): - # need to link with librt to get shm_open() - libs.append('rt') - self.add(Extension('_posixshmem', posixshmem_srcs, - define_macros={}, - libraries=libs, - include_dirs=["Modules/_multiprocessing"])) - self.add(Extension('_multiprocessing', multiprocessing_srcs, include_dirs=["Modules/_multiprocessing"])) + if (not MS_WINDOWS and + sysconfig.get_config_var('HAVE_SHM_OPEN') and + sysconfig.get_config_var('HAVE_SHM_UNLINK')): + posixshmem_srcs = ['_multiprocessing/posixshmem.c'] + libs = [] + if sysconfig.get_config_var('SHM_NEEDS_LIBRT'): + # need to link with librt to get shm_open() + libs.append('rt') + self.add(Extension('_posixshmem', posixshmem_srcs, + define_macros={}, + libraries=libs, + include_dirs=["Modules/_multiprocessing"])) + else: + self.missing.append('_posixshmem') + def detect_uuid(self): # Build the _uuid module if possible uuid_incs = find_file("uuid.h", self.inc_dirs, ["/usr/include/uuid"]) @@ -2549,6 +2610,12 @@ def copy_scripts(self): def main(): + global LIST_MODULE_NAMES + + if "--list-module-names" in sys.argv: + LIST_MODULE_NAMES = True + sys.argv.remove("--list-module-names") + set_compiler_flags('CFLAGS', 'PY_CFLAGS_NODIST') set_compiler_flags('LDFLAGS', 'PY_LDFLAGS_NODIST')