diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index df0f107a541614..f0a2dd5694c078 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,6 +36,7 @@ jobs: timeout-minutes: 10 outputs: run_tests: ${{ steps.check.outputs.run_tests }} + run_hypothesis: ${{ steps.check.outputs.run_hypothesis }} steps: - uses: actions/checkout@v3 - name: Check for source changes @@ -61,6 +62,17 @@ jobs: git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true fi + # Check if we should run hypothesis tests + GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} + echo $GIT_BRANCH + if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then + echo "Branch too old for hypothesis tests" + echo "run_hypothesis=false" >> $GITHUB_OUTPUT + else + echo "Run hypothesis tests" + echo "run_hypothesis=true" >> $GITHUB_OUTPUT + fi + check_generated_files: name: 'Check if generated files are up to date' runs-on: ubuntu-latest @@ -291,6 +303,90 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + test_hypothesis: + name: "Hypothesis Tests on Ubuntu" + runs-on: ubuntu-20.04 + timeout-minutes: 60 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' + env: + OPENSSL_VER: 1.1.1t + PYTHONSTRICTEXTENSIONBUILD: 1 + steps: + - uses: actions/checkout@v3 + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v3 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV + echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + - name: Create directories for read-only out-of-tree builds + run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + - name: Bind mount sources read-only + run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + - name: Configure CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR + - name: Build CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make -j4 + - name: Display build info + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make pythoninfo + - name: Remount sources writable for tests + # some tests write to srcdir, lack of pyc files slows down testing + run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + - name: "Create hypothesis venv" + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + VENV_LOC=$(realpath -m .)/hypovenv + VENV_PYTHON=$VENV_LOC/bin/python + echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV + echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV + ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis + - name: "Run tests" + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + # Most of the excluded tests are slow test suites with no property tests + # + # (GH-104097) test_sysconfig is skipped because it has tests that are + # failing when executed from inside a virtual environment. + ${{ env.VENV_PYTHON }} -m test \ + -W \ + -x test_asyncio \ + -x test_multiprocessing_fork \ + -x test_multiprocessing_forkserver \ + -x test_multiprocessing_spawn \ + -x test_concurrent_futures \ + -x test_socket \ + -x test_subprocess \ + -x test_signal \ + -x test_sysconfig + build_asan: name: 'Address sanitizer' diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 6ebc2d9b0a71a9..4620b4617e3450 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -775,6 +775,9 @@ All Argument Clinic converters accept the following arguments: because :pep:`8` mandates that the Python library may not use annotations. + ``unused`` + Wrap the argument with :c:macro:`Py_UNUSED` in the impl function signature. + In addition, some converters accept additional arguments. Here is a list of these arguments, along with their meanings: diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 43a3286ba832cf..158f4851634652 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -14,7 +14,10 @@ .. testsetup:: * - from collections.abc import * + import warnings + # Ignore warning when ByteString is imported + with warnings.catch_warnings(action='ignore', category=DeprecationWarning): + from collections.abc import * import itertools __name__ = '' diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 74bffef5562ae1..ef52370bff8058 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -602,9 +602,17 @@ can be overridden by the local file. Execute the (one-line) *statement* in the context of the current stack frame. The exclamation point can be omitted unless the first word of the statement - resembles a debugger command. To set a global variable, you can prefix the - assignment command with a :keyword:`global` statement on the same line, - e.g.:: + resembles a debugger command, e.g.: + + .. code-block:: none + + (Pdb) ! n=42 + (Pdb) + + To set a global variable, you can prefix the assignment command with a + :keyword:`global` statement on the same line, e.g.: + + .. code-block:: none (Pdb) global list_options; list_options = ['-l'] (Pdb) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index eb13d4bf031c95..546c7147bb3b27 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -831,6 +831,9 @@ Pending Removal in Python 3.14 For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. (Contributed by Shantanu Jain in :gh:`91896`.) +* :class:`typing.ByteString`, deprecated since Python 3.9, now causes an + :exc:`DeprecationWarning` to be emitted when it is used or accessed. + * Creating immutable types (:data:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable bases using the C API. diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 3d3cbbff7aae81..a72e03f1438fc8 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -19,7 +19,6 @@ struct _frame { struct _PyInterpreterFrame *f_frame; /* points to the frame data */ PyObject *f_trace; /* Trace function */ int f_lineno; /* Current line number. Only valid if non-zero */ - int f_last_traced_line; /* The last line traced for this frame */ char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ char f_fast_as_locals; /* Have the fast locals of this frame been converted to a dict? */ diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index e94d8755546efd..9fb3952227af18 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -69,13 +69,13 @@ _Py_call_instrumentation(PyThreadState *tstate, int event, extern int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, - _Py_CODEUNIT *instr); + _Py_CODEUNIT *instr, _Py_CODEUNIT *prev); extern int _Py_call_instrumentation_instruction( PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr); -int +_Py_CODEUNIT * _Py_call_instrumentation_jump( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target); @@ -100,6 +100,7 @@ extern int _Py_Instrumentation_GetLine(PyCodeObject *code, int index); extern PyObject _PyInstrumentation_MISSING; +extern PyObject _PyInstrumentation_DISABLE; #ifdef __cplusplus } diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py index 86ca8b8a8414b3..60b1eb60fa6ae1 100644 --- a/Lib/collections/abc.py +++ b/Lib/collections/abc.py @@ -1,3 +1,12 @@ from _collections_abc import * from _collections_abc import __all__ from _collections_abc import _CallableGenericAlias + +_deprecated_ByteString = globals().pop("ByteString") + +def __getattr__(attr): + if attr == "ByteString": + import warnings + warnings._deprecated("collections.abc.ByteString", remove=(3, 14)) + return _deprecated_ByteString + raise AttributeError(f"module 'collections.abc' has no attribute {attr!r}") diff --git a/Lib/pdb.py b/Lib/pdb.py index b3dc5a455e56b9..6b6feac1ddead1 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -440,7 +440,7 @@ def displayhook(self, obj): self.message(repr(obj)) def default(self, line): - if line[:1] == '!': line = line[1:] + if line[:1] == '!': line = line[1:].strip() locals = self.curframe_locals globals = self.curframe.f_globals try: @@ -1642,9 +1642,12 @@ def help_exec(self): Execute the (one-line) statement in the context of the current stack frame. The exclamation point can be omitted unless the - first word of the statement resembles a debugger command. To - assign to a global variable you must always prefix the command - with a 'global' command, e.g.: + first word of the statement resembles a debugger command, e.g.: + (Pdb) ! n=42 + (Pdb) + + To assign to a global variable you must always prefix the command with + a 'global' command, e.g.: (Pdb) global list_options; list_options = ['-l'] (Pdb) """ diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 1babb5ce9476c9..3aaaee67fa35a6 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -5283,11 +5283,14 @@ 'current\n' ' stack frame. The exclamation point can be omitted unless the ' 'first\n' - ' word of the statement resembles a debugger command. To set ' - 'a\n' - ' global variable, you can prefix the assignment command with ' - 'a\n' - ' "global" statement on the same line, e.g.:\n' + ' word of the statement resembles a debugger command, e.g.:' + '\n' + ' (Pdb) ! n=42\n' + ' (Pdb)\n' + '\n' + ' To set a global variable, you can prefix the assignment command ' + ' with \n' + ' a "global" statement on the same line, e.g.:\n' '\n' " (Pdb) global list_options; list_options = ['-l']\n" ' (Pdb)\n' diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 2de8c6cfbc61a1..776a9e9b587d32 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -48,11 +48,13 @@ def dash_R(ns, test_name, test_func): else: zdc = zipimport._zip_directory_cache.copy() abcs = {} - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue - for obj in abc.__subclasses__() + [abc]: - abcs[obj] = _get_dump(obj)[0] + # catch and ignore collections.abc.ByteString deprecation + with warnings.catch_warnings(action='ignore', category=DeprecationWarning): + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: + if not isabstract(abc): + continue + for obj in abc.__subclasses__() + [abc]: + abcs[obj] = _get_dump(obj)[0] # bpo-31217: Integer pool to get a single integer object for the same # value. The pool is used to prevent false alarm when checking for memory @@ -173,7 +175,9 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): zipimport._zip_directory_cache.update(zdc) # Clear ABC registries, restoring previously saved ABC registries. - abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] + # ignore deprecation warning for collections.abc.ByteString + with warnings.catch_warnings(action='ignore', category=DeprecationWarning): + abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] abs_classes = filter(isabstract, abs_classes) for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index cc5870ab2b833e..c7801b767c590c 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -257,8 +257,10 @@ def restore_sysconfig__INSTALL_SCHEMES(self, saved): sysconfig._INSTALL_SCHEMES.update(saved[2]) def get_files(self): + # XXX: Maybe add an allow-list here? return sorted(fn + ('/' if os.path.isdir(fn) else '') - for fn in os.listdir()) + for fn in os.listdir() + if not fn.startswith(".hypothesis")) def restore_files(self, saved_value): fn = os_helper.TESTFN if fn not in saved_value and (fn + '/') not in saved_value: diff --git a/Lib/test/support/_hypothesis_stubs/__init__.py b/Lib/test/support/_hypothesis_stubs/__init__.py new file mode 100644 index 00000000000000..6ba5bb814b92f7 --- /dev/null +++ b/Lib/test/support/_hypothesis_stubs/__init__.py @@ -0,0 +1,111 @@ +from enum import Enum +import functools +import unittest + +__all__ = [ + "given", + "example", + "assume", + "reject", + "register_random", + "strategies", + "HealthCheck", + "settings", + "Verbosity", +] + +from . import strategies + + +def given(*_args, **_kwargs): + def decorator(f): + if examples := getattr(f, "_examples", []): + + @functools.wraps(f) + def test_function(self): + for example_args, example_kwargs in examples: + with self.subTest(*example_args, **example_kwargs): + f(self, *example_args, **example_kwargs) + + else: + # If we have found no examples, we must skip the test. If @example + # is applied after @given, it will re-wrap the test to remove the + # skip decorator. + test_function = unittest.skip( + "Hypothesis required for property test with no " + + "specified examples" + )(f) + + test_function._given = True + return test_function + + return decorator + + +def example(*args, **kwargs): + if bool(args) == bool(kwargs): + raise ValueError("Must specify exactly one of *args or **kwargs") + + def decorator(f): + base_func = getattr(f, "__wrapped__", f) + if not hasattr(base_func, "_examples"): + base_func._examples = [] + + base_func._examples.append((args, kwargs)) + + if getattr(f, "_given", False): + # If the given decorator is below all the example decorators, + # it would be erroneously skipped, so we need to re-wrap the new + # base function. + f = given()(base_func) + + return f + + return decorator + + +def assume(condition): + if not condition: + raise unittest.SkipTest("Unsatisfied assumption") + return True + + +def reject(): + assume(False) + + +def register_random(*args, **kwargs): + pass # pragma: no cover + + +def settings(*args, **kwargs): + return lambda f: f # pragma: nocover + + +class HealthCheck(Enum): + data_too_large = 1 + filter_too_much = 2 + too_slow = 3 + return_value = 5 + large_base_example = 7 + not_a_test_method = 8 + + @classmethod + def all(cls): + return list(cls) + + +class Verbosity(Enum): + quiet = 0 + normal = 1 + verbose = 2 + debug = 3 + + +class Phase(Enum): + explicit = 0 + reuse = 1 + generate = 2 + target = 3 + shrink = 4 + explain = 5 diff --git a/Lib/test/support/_hypothesis_stubs/_helpers.py b/Lib/test/support/_hypothesis_stubs/_helpers.py new file mode 100644 index 00000000000000..3f6244e4dbc0b1 --- /dev/null +++ b/Lib/test/support/_hypothesis_stubs/_helpers.py @@ -0,0 +1,43 @@ +# Stub out only the subset of the interface that we actually use in our tests. +class StubClass: + def __init__(self, *args, **kwargs): + self.__stub_args = args + self.__stub_kwargs = kwargs + self.__repr = None + + def _with_repr(self, new_repr): + new_obj = self.__class__(*self.__stub_args, **self.__stub_kwargs) + new_obj.__repr = new_repr + return new_obj + + def __repr__(self): + if self.__repr is not None: + return self.__repr + + argstr = ", ".join(self.__stub_args) + kwargstr = ", ".join(f"{kw}={val}" for kw, val in self.__stub_kwargs.items()) + + in_parens = argstr + if kwargstr: + in_parens += ", " + kwargstr + + return f"{self.__class__.__qualname__}({in_parens})" + + +def stub_factory(klass, name, *, with_repr=None, _seen={}): + if (klass, name) not in _seen: + + class Stub(klass): + def __init__(self, *args, **kwargs): + super().__init__() + self.__stub_args = args + self.__stub_kwargs = kwargs + + Stub.__name__ = name + Stub.__qualname__ = name + if with_repr is not None: + Stub._repr = None + + _seen.setdefault((klass, name, with_repr), Stub) + + return _seen[(klass, name, with_repr)] diff --git a/Lib/test/support/_hypothesis_stubs/strategies.py b/Lib/test/support/_hypothesis_stubs/strategies.py new file mode 100644 index 00000000000000..d2b885d41e16d0 --- /dev/null +++ b/Lib/test/support/_hypothesis_stubs/strategies.py @@ -0,0 +1,91 @@ +import functools + +from ._helpers import StubClass, stub_factory + + +class StubStrategy(StubClass): + def __make_trailing_repr(self, transformation_name, func): + func_name = func.__name__ or repr(func) + return f"{self!r}.{transformation_name}({func_name})" + + def map(self, pack): + return self._with_repr(self.__make_trailing_repr("map", pack)) + + def flatmap(self, expand): + return self._with_repr(self.__make_trailing_repr("flatmap", expand)) + + def filter(self, condition): + return self._with_repr(self.__make_trailing_repr("filter", condition)) + + def __or__(self, other): + new_repr = f"one_of({self!r}, {other!r})" + return self._with_repr(new_repr) + + +_STRATEGIES = { + "binary", + "booleans", + "builds", + "characters", + "complex_numbers", + "composite", + "data", + "dates", + "datetimes", + "decimals", + "deferred", + "dictionaries", + "emails", + "fixed_dictionaries", + "floats", + "fractions", + "from_regex", + "from_type", + "frozensets", + "functions", + "integers", + "iterables", + "just", + "lists", + "none", + "nothing", + "one_of", + "permutations", + "random_module", + "randoms", + "recursive", + "register_type_strategy", + "runner", + "sampled_from", + "sets", + "shared", + "slices", + "timedeltas", + "times", + "text", + "tuples", + "uuids", +} + +__all__ = sorted(_STRATEGIES) + + +def composite(f): + strategy = stub_factory(StubStrategy, f.__name__) + + @functools.wraps(f) + def inner(*args, **kwargs): + return strategy(*args, **kwargs) + + return inner + + +def __getattr__(name): + if name not in _STRATEGIES: + raise AttributeError(f"Unknown attribute {name}") + + return stub_factory(StubStrategy, f"hypothesis.strategies.{name}") + + +def __dir__(): + return __all__ diff --git a/Lib/test/support/hypothesis_helper.py b/Lib/test/support/hypothesis_helper.py new file mode 100644 index 00000000000000..76bd2490fe6e3d --- /dev/null +++ b/Lib/test/support/hypothesis_helper.py @@ -0,0 +1,4 @@ +try: + import hypothesis +except ImportError: + from . import _hypothesis_stubs as hypothesis diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 2c65ae8114818f..94fc9d4436b717 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4749,6 +4749,19 @@ def __buffer__(self, flags): c.clear() self.assertIs(c.buffer, None) + def test_release_buffer_with_exception_set(self): + class A: + def __buffer__(self, flags): + return memoryview(bytes(8)) + def __release_buffer__(self, view): + pass + + b = bytearray(8) + with memoryview(b): + # now b.extend will raise an exception due to exports + with self.assertRaises(BufferError): + b.extend(A()) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 6aaf4d1ed8d560..28d9f650926427 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -774,6 +774,44 @@ def test_legacy_converters(self): module, function = block.signatures self.assertIsInstance((function.parameters['path']).converter, clinic.str_converter) + def test_unused_param(self): + block = self.parse(""" + module foo + foo.func + fn: object + k: float + i: float(unused=True) + / + * + flag: bool(unused=True) = False + """) + sig = block.signatures[1] # Function index == 1 + params = sig.parameters + conv = lambda fn: params[fn].converter + dataset = ( + {"name": "fn", "unused": False}, + {"name": "k", "unused": False}, + {"name": "i", "unused": True}, + {"name": "flag", "unused": True}, + ) + for param in dataset: + name, unused = param.values() + with self.subTest(name=name, unused=unused): + p = conv(name) + # Verify that the unused flag is parsed correctly. + self.assertEqual(unused, p.unused) + + # Now, check that we'll produce correct code. + decl = p.simple_declaration(in_parser=False) + if unused: + self.assertIn("Py_UNUSED", decl) + else: + self.assertNotIn("Py_UNUSED", decl) + + # Make sure the Py_UNUSED macro is not used in the parser body. + parser_decl = p.simple_declaration(in_parser=True) + self.assertNotIn("Py_UNUSED", parser_decl) + def parse(self, text): c = FakeClinic() parser = DSLParser(c) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index bb8b352518ef3e..f0736b8081feac 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -11,6 +11,7 @@ import string import sys from test import support +from test.support.import_helper import import_fresh_module import types import unittest @@ -25,7 +26,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import ByteString, Buffer +from collections.abc import Buffer class TestUserObjects(unittest.TestCase): @@ -1939,6 +1940,8 @@ def assert_index_same(seq1, seq2, index_args): nativeseq, seqseq, (letter, start, stop)) def test_ByteString(self): + with self.assertWarns(DeprecationWarning): + from collections.abc import ByteString for sample in [bytes, bytearray]: with self.assertWarns(DeprecationWarning): self.assertIsInstance(sample(), ByteString) @@ -1960,6 +1963,11 @@ class X(ByteString): pass # No metaclass conflict class Z(ByteString, Awaitable): pass + def test_ByteString_attribute_access(self): + collections_abc = import_fresh_module("collections.abc") + with self.assertWarns(DeprecationWarning): + collections_abc.ByteString + def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: self.assertIsInstance(sample(b"x"), Buffer) diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 08cb5eb0c2bbab..73d758a3631b3a 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -359,6 +359,15 @@ def test_sha256_update_over_4gb(self): h.update(b"hello world") self.assertEqual(h.hexdigest(), "a5364f7a52ebe2e25f1838a4ca715a893b6fd7a23f2a0d9e9762120da8b1bf53") + @requires_resource('cpu') + def test_sha3_256_update_over_4gb(self): + zero_1mb = b"\0" * 1024 * 1024 + h = hashlib.sha3_256() + for i in range(0, 4096): + h.update(zero_1mb) + h.update(b"hello world") + self.assertEqual(h.hexdigest(), "e2d4535e3b613135c14f2fe4e026d7ad8d569db44901740beffa30d430acb038") + def check(self, name, data, hexdigest, shake=False, **kwargs): length = len(hexdigest)//2 hexdigest = hexdigest.lower() diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 1cc202bb599ae6..b2a3b7ea3e49b6 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -117,15 +117,15 @@ def get_output(moddict, name): newcode = code def get_output(moddict, name): return moddict[name] - ns = ns or {} + newns = ns.copy() if ns else {} try: - exec(newcode, ns) + exec(newcode, newns) except raises as e: # We care about e.g. NameError vs UnboundLocalError self.assertIs(type(e), raises) else: for k, v in (outputs or {}).items(): - self.assertEqual(get_output(ns, k), v) + self.assertEqual(get_output(newns, k), v) def test_lambdas_with_iteration_var_as_default(self): code = """ @@ -180,6 +180,26 @@ def test_closure_can_jump_over_comp_scope(self): z = [x() for x in items] """ outputs = {"z": [2, 2, 2, 2, 2]} + self._check_in_scopes(code, outputs, scopes=["module", "function"]) + + def test_cell_inner_free_outer(self): + code = """ + def f(): + return [lambda: x for x in (x, [1])[1]] + x = ... + y = [fn() for fn in f()] + """ + outputs = {"y": [1]} + self._check_in_scopes(code, outputs, scopes=["module", "function"]) + + def test_free_inner_cell_outer(self): + code = """ + g = 2 + def f(): + return g + y = [g for x in [1]] + """ + outputs = {"y": [2]} self._check_in_scopes(code, outputs) def test_inner_cell_shadows_outer_redefined(self): @@ -203,6 +223,37 @@ def inner(): outputs = {"x": -1} self._check_in_scopes(code, outputs, ns={"g": -1}) + def test_explicit_global(self): + code = """ + global g + x = g + g = 2 + items = [g for g in [1]] + y = g + """ + outputs = {"x": 1, "y": 2, "items": [1]} + self._check_in_scopes(code, outputs, ns={"g": 1}) + + def test_explicit_global_2(self): + code = """ + global g + x = g + g = 2 + items = [g for x in [1]] + y = g + """ + outputs = {"x": 1, "y": 2, "items": [2]} + self._check_in_scopes(code, outputs, ns={"g": 1}) + + def test_explicit_global_3(self): + code = """ + global g + fns = [lambda: g for g in [2]] + items = [fn() for fn in fns] + """ + outputs = {"items": [2]} + self._check_in_scopes(code, outputs, ns={"g": 1}) + def test_assignment_expression(self): code = """ x = -1 @@ -250,7 +301,7 @@ def g(): g() """ outputs = {"x": 1} - self._check_in_scopes(code, outputs) + self._check_in_scopes(code, outputs, scopes=["module", "function"]) def test_introspecting_frame_locals(self): code = """ diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index c678ce13418f7c..62c31e8df60368 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -525,7 +525,7 @@ def test_lines_loop(self): sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) start = LineMonitoringTest.test_lines_loop.__code__.co_firstlineno - self.assertEqual(events, [start+7, 22, 23, 23, 22, start+8]) + self.assertEqual(events, [start+7, 22, 23, 22, 23, 22, start+8]) finally: sys.monitoring.set_events(TEST_TOOL, 0) sys.monitoring.register_callback(TEST_TOOL, E.LINE, None) @@ -1051,6 +1051,8 @@ def func3(): def line_from_offset(code, offset): for start, end, line in code.co_lines(): if start <= offset < end: + if line is None: + return f"[offset={offset}]" return line - code.co_firstlineno return -1 @@ -1073,9 +1075,20 @@ class BranchRecorder(JumpRecorder): event_type = E.BRANCH name = "branch" +class ReturnRecorder: + + event_type = E.PY_RETURN + + def __init__(self, events): + self.events = events + + def __call__(self, code, offset, val): + self.events.append(("return", val)) + JUMP_AND_BRANCH_RECORDERS = JumpRecorder, BranchRecorder JUMP_BRANCH_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder +FLOW_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder, ExceptionRecorder, ReturnRecorder class TestBranchAndJumpEvents(CheckEvents): maxDiff = None @@ -1099,7 +1112,6 @@ def func(): ('jump', 'func', 4, 2), ('branch', 'func', 2, 2)]) - self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [ ('line', 'check_events', 10), ('line', 'func', 1), @@ -1109,15 +1121,62 @@ def func(): ('branch', 'func', 3, 6), ('line', 'func', 6), ('jump', 'func', 6, 2), + ('line', 'func', 2), ('branch', 'func', 2, 2), ('line', 'func', 3), ('branch', 'func', 3, 4), ('line', 'func', 4), ('jump', 'func', 4, 2), + ('line', 'func', 2), ('branch', 'func', 2, 2), + ('line', 'check_events', 11)]) + + def test_except_star(self): + + class Foo: + def meth(self): + pass + + def func(): + try: + try: + raise KeyError + except* Exception as e: + f = Foo(); f.meth() + except KeyError: + pass + + + self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('line', 'func', 1), ('line', 'func', 2), + ('line', 'func', 3), + ('line', 'func', 4), + ('branch', 'func', 4, 4), + ('line', 'func', 5), + ('line', 'meth', 1), + ('jump', 'func', 5, 5), + ('jump', 'func', 5, '[offset=114]'), + ('branch', 'func', '[offset=120]', '[offset=122]'), ('line', 'check_events', 11)]) + self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('line', 'func', 1), + ('line', 'func', 2), + ('line', 'func', 3), + ('raise', KeyError), + ('line', 'func', 4), + ('branch', 'func', 4, 4), + ('line', 'func', 5), + ('line', 'meth', 1), + ('return', None), + ('jump', 'func', 5, 5), + ('jump', 'func', 5, '[offset=114]'), + ('branch', 'func', '[offset=120]', '[offset=122]'), + ('return', None), + ('line', 'check_events', 11)]) class TestLoadSuperAttr(CheckEvents): RECORDERS = CallRecorder, LineRecorder, CRaiseRecorder, CReturnRecorder diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 482c92dbf1f6a0..83c7cdff87fd34 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1793,8 +1793,32 @@ def test_pdb_issue_gh_101517(): ... 'continue' ... ]): ... test_function() - > (5)test_function() - -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + --Return-- + > (None)test_function()->None + -> Warning: lineno is None + (Pdb) continue + """ + +def test_pdb_ambiguous_statements(): + """See GH-104301 + + Make sure that ambiguous statements prefixed by '!' are properly disambiguated + + >>> with PdbTestInput([ + ... '! n = 42', # disambiguated statement: reassign the name n + ... 'n', # advance the debugger into the print() + ... 'continue' + ... ]): + ... n = -1 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... print(f"The value of n is {n}") + > (8)() + -> print(f"The value of n is {n}") + (Pdb) ! n = 42 + (Pdb) n + The value of n is 42 + > (1)() + -> with PdbTestInput([ (Pdb) continue """ diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 611cd27ecf1240..e1db450bf8a2ef 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1446,7 +1446,7 @@ class C(object): pass def func(): return sys._getframe() x = func() - check(x, size('3Pii3c7P2ic??2P')) + check(x, size('3Pi3c7P2ic??2P')) # function def func(): pass check(func, size('14Pi')) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 980321e169b9e5..4411603af18cd3 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2867,6 +2867,5 @@ def func(arg = 1): sys.settrace(None) - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f162e587810ac0..3422dc1ed3f5f8 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -8,6 +8,7 @@ import re import sys import warnings +from test.support.import_helper import import_fresh_module from unittest import TestCase, main, skipUnless, skip from unittest.mock import patch from copy import copy, deepcopy @@ -3908,7 +3909,14 @@ class MyChain(typing.ChainMap[str, T]): ... self.assertEqual(MyChain[int]().__orig_class__, MyChain[int]) def test_all_repr_eq_any(self): - objs = (getattr(typing, el) for el in typing.__all__) + typing = import_fresh_module("typing") + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings('always', '', DeprecationWarning) + objs = [getattr(typing, el) for el in typing.__all__] + self.assertEqual( + [str(w.message) for w in wlog], + ["'typing.ByteString' is deprecated and slated for removal in Python 3.14"] + ) for obj in objs: self.assertNotEqual(repr(obj), '') self.assertEqual(obj, obj) @@ -5996,8 +6004,16 @@ def test_mutablesequence(self): self.assertNotIsInstance((), typing.MutableSequence) def test_bytestring(self): - self.assertIsInstance(b'', typing.ByteString) - self.assertIsInstance(bytearray(b''), typing.ByteString) + with self.assertWarns(DeprecationWarning): + from typing import ByteString + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(b'', ByteString) + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(bytearray(b''), ByteString) + with self.assertWarns(DeprecationWarning): + class Foo(ByteString): ... + with self.assertWarns(DeprecationWarning): + class Bar(ByteString, typing.Awaitable): ... def test_list(self): self.assertIsSubclass(list, typing.List) @@ -8293,6 +8309,10 @@ def test_no_isinstance(self): class SpecialAttrsTests(BaseTestCase): def test_special_attrs(self): + with warnings.catch_warnings( + action='ignore', category=DeprecationWarning + ): + typing_ByteString = typing.ByteString cls_to_check = { # ABC classes typing.AbstractSet: 'AbstractSet', @@ -8301,7 +8321,7 @@ def test_special_attrs(self): typing.AsyncIterable: 'AsyncIterable', typing.AsyncIterator: 'AsyncIterator', typing.Awaitable: 'Awaitable', - typing.ByteString: 'ByteString', + typing_ByteString: 'ByteString', typing.Callable: 'Callable', typing.ChainMap: 'ChainMap', typing.Collection: 'Collection', @@ -8626,6 +8646,8 @@ def test_all_exported_names(self): getattr(v, '__module__', None) == typing.__name__ ) } + # Deprecated; added dynamically via module __getattr__ + computed_all.add("ByteString") self.assertSetEqual(computed_all, actual_all) diff --git a/Lib/test/test_zoneinfo/__init__.py b/Lib/test/test_zoneinfo/__init__.py index 98cc4412ae16c2..c3ea567103275d 100644 --- a/Lib/test/test_zoneinfo/__init__.py +++ b/Lib/test/test_zoneinfo/__init__.py @@ -1 +1,2 @@ from .test_zoneinfo import * +from .test_zoneinfo_property import * diff --git a/Lib/test/test_zoneinfo/test_zoneinfo_property.py b/Lib/test/test_zoneinfo/test_zoneinfo_property.py new file mode 100644 index 00000000000000..feaa77f3e7f0b9 --- /dev/null +++ b/Lib/test/test_zoneinfo/test_zoneinfo_property.py @@ -0,0 +1,368 @@ +import contextlib +import datetime +import os +import pickle +import unittest +import zoneinfo + +from test.support.hypothesis_helper import hypothesis + +import test.test_zoneinfo._support as test_support + +ZoneInfoTestBase = test_support.ZoneInfoTestBase + +py_zoneinfo, c_zoneinfo = test_support.get_modules() + +UTC = datetime.timezone.utc +MIN_UTC = datetime.datetime.min.replace(tzinfo=UTC) +MAX_UTC = datetime.datetime.max.replace(tzinfo=UTC) +ZERO = datetime.timedelta(0) + + +def _valid_keys(): + """Get available time zones, including posix/ and right/ directories.""" + from importlib import resources + + available_zones = sorted(zoneinfo.available_timezones()) + TZPATH = zoneinfo.TZPATH + + def valid_key(key): + for root in TZPATH: + key_file = os.path.join(root, key) + if os.path.exists(key_file): + return True + + components = key.split("/") + package_name = ".".join(["tzdata.zoneinfo"] + components[:-1]) + resource_name = components[-1] + + try: + return resources.files(package_name).joinpath(resource_name).is_file() + except ModuleNotFoundError: + return False + + # This relies on the fact that dictionaries maintain insertion order — for + # shrinking purposes, it is preferable to start with the standard version, + # then move to the posix/ version, then to the right/ version. + out_zones = {"": available_zones} + for prefix in ["posix", "right"]: + prefix_out = [] + for key in available_zones: + prefix_key = f"{prefix}/{key}" + if valid_key(prefix_key): + prefix_out.append(prefix_key) + + out_zones[prefix] = prefix_out + + output = [] + for keys in out_zones.values(): + output.extend(keys) + + return output + + +VALID_KEYS = _valid_keys() +if not VALID_KEYS: + raise unittest.SkipTest("No time zone data available") + + +def valid_keys(): + return hypothesis.strategies.sampled_from(VALID_KEYS) + + +KEY_EXAMPLES = [ + "Africa/Abidjan", + "Africa/Casablanca", + "America/Los_Angeles", + "America/Santiago", + "Asia/Tokyo", + "Australia/Sydney", + "Europe/Dublin", + "Europe/Lisbon", + "Europe/London", + "Pacific/Kiritimati", + "UTC", +] + + +def add_key_examples(f): + for key in KEY_EXAMPLES: + f = hypothesis.example(key)(f) + return f + + +class ZoneInfoTest(ZoneInfoTestBase): + module = py_zoneinfo + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_str(self, key): + zi = self.klass(key) + self.assertEqual(str(zi), key) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_key(self, key): + zi = self.klass(key) + + self.assertEqual(zi.key, key) + + @hypothesis.given( + dt=hypothesis.strategies.one_of( + hypothesis.strategies.datetimes(), hypothesis.strategies.times() + ) + ) + @hypothesis.example(dt=datetime.datetime.min) + @hypothesis.example(dt=datetime.datetime.max) + @hypothesis.example(dt=datetime.datetime(1970, 1, 1)) + @hypothesis.example(dt=datetime.datetime(2039, 1, 1)) + @hypothesis.example(dt=datetime.time(0)) + @hypothesis.example(dt=datetime.time(12, 0)) + @hypothesis.example(dt=datetime.time(23, 59, 59, 999999)) + def test_utc(self, dt): + zi = self.klass("UTC") + dt_zi = dt.replace(tzinfo=zi) + + self.assertEqual(dt_zi.utcoffset(), ZERO) + self.assertEqual(dt_zi.dst(), ZERO) + self.assertEqual(dt_zi.tzname(), "UTC") + + +class CZoneInfoTest(ZoneInfoTest): + module = c_zoneinfo + + +class ZoneInfoPickleTest(ZoneInfoTestBase): + module = py_zoneinfo + + def setUp(self): + with contextlib.ExitStack() as stack: + stack.enter_context(test_support.set_zoneinfo_module(self.module)) + self.addCleanup(stack.pop_all().close) + + super().setUp() + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_pickle_unpickle_cache(self, key): + zi = self.klass(key) + pkl_str = pickle.dumps(zi) + zi_rt = pickle.loads(pkl_str) + + self.assertIs(zi, zi_rt) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_pickle_unpickle_no_cache(self, key): + zi = self.klass.no_cache(key) + pkl_str = pickle.dumps(zi) + zi_rt = pickle.loads(pkl_str) + + self.assertIsNot(zi, zi_rt) + self.assertEqual(str(zi), str(zi_rt)) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_pickle_unpickle_cache_multiple_rounds(self, key): + """Test that pickle/unpickle is idempotent.""" + zi_0 = self.klass(key) + pkl_str_0 = pickle.dumps(zi_0) + zi_1 = pickle.loads(pkl_str_0) + pkl_str_1 = pickle.dumps(zi_1) + zi_2 = pickle.loads(pkl_str_1) + pkl_str_2 = pickle.dumps(zi_2) + + self.assertEqual(pkl_str_0, pkl_str_1) + self.assertEqual(pkl_str_1, pkl_str_2) + + self.assertIs(zi_0, zi_1) + self.assertIs(zi_0, zi_2) + self.assertIs(zi_1, zi_2) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_pickle_unpickle_no_cache_multiple_rounds(self, key): + """Test that pickle/unpickle is idempotent.""" + zi_cache = self.klass(key) + + zi_0 = self.klass.no_cache(key) + pkl_str_0 = pickle.dumps(zi_0) + zi_1 = pickle.loads(pkl_str_0) + pkl_str_1 = pickle.dumps(zi_1) + zi_2 = pickle.loads(pkl_str_1) + pkl_str_2 = pickle.dumps(zi_2) + + self.assertEqual(pkl_str_0, pkl_str_1) + self.assertEqual(pkl_str_1, pkl_str_2) + + self.assertIsNot(zi_0, zi_1) + self.assertIsNot(zi_0, zi_2) + self.assertIsNot(zi_1, zi_2) + + self.assertIsNot(zi_0, zi_cache) + self.assertIsNot(zi_1, zi_cache) + self.assertIsNot(zi_2, zi_cache) + + +class CZoneInfoPickleTest(ZoneInfoPickleTest): + module = c_zoneinfo + + +class ZoneInfoCacheTest(ZoneInfoTestBase): + module = py_zoneinfo + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_cache(self, key): + zi_0 = self.klass(key) + zi_1 = self.klass(key) + + self.assertIs(zi_0, zi_1) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_no_cache(self, key): + zi_0 = self.klass.no_cache(key) + zi_1 = self.klass.no_cache(key) + + self.assertIsNot(zi_0, zi_1) + + +class CZoneInfoCacheTest(ZoneInfoCacheTest): + klass = c_zoneinfo.ZoneInfo + + +class PythonCConsistencyTest(unittest.TestCase): + """Tests that the C and Python versions do the same thing.""" + + def _is_ambiguous(self, dt): + return dt.replace(fold=not dt.fold).utcoffset() == dt.utcoffset() + + @hypothesis.given(dt=hypothesis.strategies.datetimes(), key=valid_keys()) + @hypothesis.example(dt=datetime.datetime.min, key="America/New_York") + @hypothesis.example(dt=datetime.datetime.max, key="America/New_York") + @hypothesis.example(dt=datetime.datetime(1970, 1, 1), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2020, 1, 1), key="Europe/Paris") + @hypothesis.example(dt=datetime.datetime(2020, 6, 1), key="Europe/Paris") + def test_same_str(self, dt, key): + py_dt = dt.replace(tzinfo=py_zoneinfo.ZoneInfo(key)) + c_dt = dt.replace(tzinfo=c_zoneinfo.ZoneInfo(key)) + + self.assertEqual(str(py_dt), str(c_dt)) + + @hypothesis.given(dt=hypothesis.strategies.datetimes(), key=valid_keys()) + @hypothesis.example(dt=datetime.datetime(1970, 1, 1), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2020, 2, 5), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2020, 8, 12), key="America/New_York") + @hypothesis.example(dt=datetime.datetime(2040, 1, 1), key="Africa/Casablanca") + @hypothesis.example(dt=datetime.datetime(1970, 1, 1), key="Europe/Paris") + @hypothesis.example(dt=datetime.datetime(2040, 1, 1), key="Europe/Paris") + @hypothesis.example(dt=datetime.datetime.min, key="Asia/Tokyo") + @hypothesis.example(dt=datetime.datetime.max, key="Asia/Tokyo") + def test_same_offsets_and_names(self, dt, key): + py_dt = dt.replace(tzinfo=py_zoneinfo.ZoneInfo(key)) + c_dt = dt.replace(tzinfo=c_zoneinfo.ZoneInfo(key)) + + self.assertEqual(py_dt.tzname(), c_dt.tzname()) + self.assertEqual(py_dt.utcoffset(), c_dt.utcoffset()) + self.assertEqual(py_dt.dst(), c_dt.dst()) + + @hypothesis.given( + dt=hypothesis.strategies.datetimes(timezones=hypothesis.strategies.just(UTC)), + key=valid_keys(), + ) + @hypothesis.example(dt=MIN_UTC, key="Asia/Tokyo") + @hypothesis.example(dt=MAX_UTC, key="Asia/Tokyo") + @hypothesis.example(dt=MIN_UTC, key="America/New_York") + @hypothesis.example(dt=MAX_UTC, key="America/New_York") + @hypothesis.example( + dt=datetime.datetime(2006, 10, 29, 5, 15, tzinfo=UTC), + key="America/New_York", + ) + def test_same_from_utc(self, dt, key): + py_zi = py_zoneinfo.ZoneInfo(key) + c_zi = c_zoneinfo.ZoneInfo(key) + + # Convert to UTC: This can overflow, but we just care about consistency + py_overflow_exc = None + c_overflow_exc = None + try: + py_dt = dt.astimezone(py_zi) + except OverflowError as e: + py_overflow_exc = e + + try: + c_dt = dt.astimezone(c_zi) + except OverflowError as e: + c_overflow_exc = e + + if (py_overflow_exc is not None) != (c_overflow_exc is not None): + raise py_overflow_exc or c_overflow_exc # pragma: nocover + + if py_overflow_exc is not None: + return # Consistently raises the same exception + + # PEP 495 says that an inter-zone comparison between ambiguous + # datetimes is always False. + if py_dt != c_dt: + self.assertEqual( + self._is_ambiguous(py_dt), + self._is_ambiguous(c_dt), + (py_dt, c_dt), + ) + + self.assertEqual(py_dt.tzname(), c_dt.tzname()) + self.assertEqual(py_dt.utcoffset(), c_dt.utcoffset()) + self.assertEqual(py_dt.dst(), c_dt.dst()) + + @hypothesis.given(dt=hypothesis.strategies.datetimes(), key=valid_keys()) + @hypothesis.example(dt=datetime.datetime.max, key="America/New_York") + @hypothesis.example(dt=datetime.datetime.min, key="America/New_York") + @hypothesis.example(dt=datetime.datetime.min, key="Asia/Tokyo") + @hypothesis.example(dt=datetime.datetime.max, key="Asia/Tokyo") + def test_same_to_utc(self, dt, key): + py_dt = dt.replace(tzinfo=py_zoneinfo.ZoneInfo(key)) + c_dt = dt.replace(tzinfo=c_zoneinfo.ZoneInfo(key)) + + # Convert from UTC: Overflow OK if it happens in both implementations + py_overflow_exc = None + c_overflow_exc = None + try: + py_utc = py_dt.astimezone(UTC) + except OverflowError as e: + py_overflow_exc = e + + try: + c_utc = c_dt.astimezone(UTC) + except OverflowError as e: + c_overflow_exc = e + + if (py_overflow_exc is not None) != (c_overflow_exc is not None): + raise py_overflow_exc or c_overflow_exc # pragma: nocover + + if py_overflow_exc is not None: + return # Consistently raises the same exception + + self.assertEqual(py_utc, c_utc) + + @hypothesis.given(key=valid_keys()) + @add_key_examples + def test_cross_module_pickle(self, key): + py_zi = py_zoneinfo.ZoneInfo(key) + c_zi = c_zoneinfo.ZoneInfo(key) + + with test_support.set_zoneinfo_module(py_zoneinfo): + py_pkl = pickle.dumps(py_zi) + + with test_support.set_zoneinfo_module(c_zoneinfo): + c_pkl = pickle.dumps(c_zi) + + with test_support.set_zoneinfo_module(c_zoneinfo): + # Python → C + py_to_c_zi = pickle.loads(py_pkl) + self.assertIs(py_to_c_zi, c_zi) + + with test_support.set_zoneinfo_module(py_zoneinfo): + # C → Python + c_to_py_zi = pickle.loads(c_pkl) + self.assertIs(c_to_py_zi, py_zi) diff --git a/Lib/typing.py b/Lib/typing.py index 4f8cba88632d6a..513d4d96dd6e1d 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -775,8 +775,7 @@ def Concatenate(self, parameters): "ParamSpec variable or ellipsis.") msg = "Concatenate[arg, ...]: each arg must be a type." parameters = (*(_type_check(p, msg) for p in parameters[:-1]), parameters[-1]) - return _ConcatenateGenericAlias(self, parameters, - _paramspec_tvars=True) + return _ConcatenateGenericAlias(self, parameters) @_SpecialForm @@ -1307,8 +1306,7 @@ def __getattr__(self, attr): raise AttributeError(attr) def __setattr__(self, attr, val): - if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams', - '_paramspec_tvars'}: + if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams'}: super().__setattr__(attr, val) else: setattr(self.__origin__, attr, val) @@ -1362,15 +1360,13 @@ class _GenericAlias(_BaseGenericAlias, _root=True): # ClassVar[float] # TypeVar[bool] - def __init__(self, origin, args, *, inst=True, name=None, - _paramspec_tvars=False): + def __init__(self, origin, args, *, inst=True, name=None): super().__init__(origin, inst=inst, name=name) if not isinstance(args, tuple): args = (args,) self.__args__ = tuple(... if a is _TypingEllipsis else a for a in args) self.__parameters__ = _collect_parameters(args) - self._paramspec_tvars = _paramspec_tvars if not name: self.__module__ = origin.__module__ @@ -1513,8 +1509,7 @@ def _make_substitution(self, args, new_arg_by_param): return new_args def copy_with(self, args): - return self.__class__(self.__origin__, args, name=self._name, inst=self._inst, - _paramspec_tvars=self._paramspec_tvars) + return self.__class__(self.__origin__, args, name=self._name, inst=self._inst) def __repr__(self): if self._name: @@ -1604,6 +1599,22 @@ def __or__(self, right): def __ror__(self, left): return Union[left, self] + +class _DeprecatedGenericAlias(_SpecialGenericAlias, _root=True): + def __init__( + self, origin, nparams, *, removal_version, inst=True, name=None + ): + super().__init__(origin, nparams, inst=inst, name=name) + self._removal_version = removal_version + + def __instancecheck__(self, inst): + import warnings + warnings._deprecated( + f"{self.__module__}.{self._name}", remove=self._removal_version + ) + return super().__instancecheck__(inst) + + class _CallableGenericAlias(_NotIterable, _GenericAlias, _root=True): def __repr__(self): assert self._name == 'Callable' @@ -1624,8 +1635,7 @@ def __reduce__(self): class _CallableType(_SpecialGenericAlias, _root=True): def copy_with(self, params): return _CallableGenericAlias(self.__origin__, params, - name=self._name, inst=self._inst, - _paramspec_tvars=True) + name=self._name, inst=self._inst) def __getitem__(self, params): if not isinstance(params, tuple) or len(params) != 2: @@ -1869,8 +1879,7 @@ def __class_getitem__(cls, params): new_args.append(new_arg) params = tuple(new_args) - return _GenericAlias(cls, params, - _paramspec_tvars=True) + return _GenericAlias(cls, params) def __init_subclass__(cls, *args, **kwargs): super().__init_subclass__(*args, **kwargs) @@ -2763,7 +2772,6 @@ class Other(Leaf): # Error reported by type checker MutableMapping = _alias(collections.abc.MutableMapping, 2) Sequence = _alias(collections.abc.Sequence, 1) MutableSequence = _alias(collections.abc.MutableSequence, 1) -ByteString = _alias(collections.abc.ByteString, 0) # Not generic # Tuple accepts variable number of parameters. Tuple = _TupleType(tuple, -1, inst=False, name='Tuple') Tuple.__doc__ = \ @@ -3563,3 +3571,27 @@ def method(self) -> None: # read-only property, TypeError if it's a builtin class. pass return method + + +def __getattr__(attr): + if attr == "ByteString": + import warnings + warnings._deprecated("typing.ByteString", remove=(3, 14)) + with warnings.catch_warnings( + action="ignore", category=DeprecationWarning + ): + # Not generic + ByteString = globals()["ByteString"] = _DeprecatedGenericAlias( + collections.abc.ByteString, 0, removal_version=(3, 14) + ) + return ByteString + raise AttributeError(f"module 'typing' has no attribute {attr!r}") + + +def _remove_cached_ByteString_from_globals(): + try: + del globals()["ByteString"] + except KeyError: + pass + +_cleanups.append(_remove_cached_ByteString_from_globals) diff --git a/Lib/uuid.py b/Lib/uuid.py index 697f3b45597023..470bc0d68597ab 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -53,7 +53,7 @@ __author__ = 'Ka-Ping Yee ' # The recognized platforms - known behaviors -if sys.platform in ('win32', 'darwin'): +if sys.platform in ('win32', 'darwin', 'emscripten', 'wasi'): _AIX = _LINUX = False else: import platform diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst new file mode 100644 index 00000000000000..40eee64dfa717d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst @@ -0,0 +1,5 @@ +Change behavior of ``sys.monitoring.events.LINE`` events in +``sys.monitoring``: Line events now occur when a new line is reached +dynamically, instead of using a static approximation, as before. This makes +the behavior very similar to that of "line" events in ``sys.settrace``. This +should ease porting of tools from 3.11 to 3.12. diff --git a/Misc/NEWS.d/next/Library/2023-05-09-18-46-24.gh-issue-104301.gNnbId.rst b/Misc/NEWS.d/next/Library/2023-05-09-18-46-24.gh-issue-104301.gNnbId.rst new file mode 100644 index 00000000000000..a44ad765e62458 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-09-18-46-24.gh-issue-104301.gNnbId.rst @@ -0,0 +1 @@ +Allow leading whitespace in disambiguated statements in :mod:`pdb`. diff --git a/Misc/NEWS.d/next/Library/2023-05-11-07-50-00.gh-issue-104392.YSllzt.rst b/Misc/NEWS.d/next/Library/2023-05-11-07-50-00.gh-issue-104392.YSllzt.rst new file mode 100644 index 00000000000000..b441b2acd19f1b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-11-07-50-00.gh-issue-104392.YSllzt.rst @@ -0,0 +1,2 @@ +Remove undocumented and unused ``_paramspec_tvars`` attribute from some +classes in :mod:`typing`. diff --git a/Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst b/Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst new file mode 100644 index 00000000000000..37ab74ffc441ed --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-04-25-12-19-37.gh-issue-86275.-RoLIt.rst @@ -0,0 +1,2 @@ +Added property-based tests to the :mod:`zoneinfo` tests, along with stubs +for the ``hypothesis`` interface. (Patch by Paul Ganssle) diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-05-11-15-12-11.gh-issue-104389.EiOhB3.rst b/Misc/NEWS.d/next/Tools-Demos/2023-05-11-15-12-11.gh-issue-104389.EiOhB3.rst new file mode 100644 index 00000000000000..854e1cca967c42 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-05-11-15-12-11.gh-issue-104389.EiOhB3.rst @@ -0,0 +1,2 @@ +Argument Clinic C converters now accept the ``unused`` keyword, for wrapping +a parameter with :c:macro:`Py_UNUSED`. Patch by Erlend E. Aasland. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 8e66576b5c5f00..95409d48c0da02 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -169,7 +169,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_hacl/Hacl_Hash_SHA3.c b/Modules/_hacl/Hacl_Hash_SHA3.c index 100afe7c2c6d1f..58eb436881d45c 100644 --- a/Modules/_hacl/Hacl_Hash_SHA3.c +++ b/Modules/_hacl/Hacl_Hash_SHA3.c @@ -244,7 +244,7 @@ Hacl_Streaming_Keccak_update(Hacl_Streaming_Keccak_state *p, uint8_t *data, uint Hacl_Streaming_Keccak_hash_buf block_state = s.block_state; uint64_t total_len = s.total_len; Spec_Hash_Definitions_hash_alg i = block_state.fst; - if ((uint64_t)len > (uint64_t)0xffffffffU - total_len) + if ((uint64_t)len > (uint64_t)0xffffffffffffffffU - total_len) { return (uint32_t)1U; } diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index 220ebbe5561341..d2ba05f30d860f 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=363eae2c2eb60e46f182ddd4bd1cd3f1d00b35c9 +expected_hacl_star_rev=b6903a3e6458000730c3d83174d4b08d6d3e2ece hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index ee4eca70e2af8f..2457cb124036fe 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -582,6 +582,7 @@ iomodule_traverse(PyObject *mod, visitproc visit, void *arg) { return 0; Py_VISIT(state->unsupported_operation); + Py_VISIT(state->PyIOBase_Type); Py_VISIT(state->PyIncrementalNewlineDecoder_Type); Py_VISIT(state->PyRawIOBase_Type); Py_VISIT(state->PyBufferedIOBase_Type); @@ -609,6 +610,7 @@ iomodule_clear(PyObject *mod) { return 0; Py_CLEAR(state->unsupported_operation); + Py_CLEAR(state->PyIOBase_Type); Py_CLEAR(state->PyIncrementalNewlineDecoder_Type); Py_CLEAR(state->PyRawIOBase_Type); Py_CLEAR(state->PyBufferedIOBase_Type); @@ -751,6 +753,7 @@ PyInit__io(void) } // Base classes + state->PyIOBase_Type = (PyTypeObject *)Py_NewRef(&PyIOBase_Type); ADD_TYPE(m, state->PyIncrementalNewlineDecoder_Type, &nldecoder_spec, NULL); ADD_TYPE(m, state->PyBytesIOBuffer_Type, &bytesiobuf_spec, NULL); diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 44d651338e6975..ae06fecc48b450 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -149,6 +149,7 @@ struct _io_state { PyObject *unsupported_operation; /* Types */ + PyTypeObject *PyIOBase_Type; PyTypeObject *PyIncrementalNewlineDecoder_Type; PyTypeObject *PyRawIOBase_Type; PyTypeObject *PyBufferedIOBase_Type; diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index 727398800bec01..7e6b3b5b78e8f6 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -231,20 +231,24 @@ PyDoc_STRVAR(_io__IOBase_fileno__doc__, "fileno($self, /)\n" "--\n" "\n" -"Returns underlying file descriptor if one exists.\n" +"Return underlying file descriptor if one exists.\n" "\n" -"OSError is raised if the IO object does not use a file descriptor."); +"Raise OSError if the IO object does not use a file descriptor."); #define _IO__IOBASE_FILENO_METHODDEF \ - {"fileno", (PyCFunction)_io__IOBase_fileno, METH_NOARGS, _io__IOBase_fileno__doc__}, + {"fileno", _PyCFunction_CAST(_io__IOBase_fileno), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__IOBase_fileno__doc__}, static PyObject * -_io__IOBase_fileno_impl(PyObject *self); +_io__IOBase_fileno_impl(PyObject *self, PyTypeObject *cls); static PyObject * -_io__IOBase_fileno(PyObject *self, PyObject *Py_UNUSED(ignored)) +_io__IOBase_fileno(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _io__IOBase_fileno_impl(self); + if (nargs) { + PyErr_SetString(PyExc_TypeError, "fileno() takes no arguments"); + return NULL; + } + return _io__IOBase_fileno_impl(self, cls); } PyDoc_STRVAR(_io__IOBase_isatty__doc__, @@ -416,4 +420,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=b6d4845254da1da2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=63bc25a5bfcecaf0 input=a9049054013a1b77]*/ diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index a74e46cc8dc536..26f2a3155bda62 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -511,15 +511,17 @@ iobase_exit(PyObject *self, PyObject *args) /*[clinic input] _io._IOBase.fileno + cls: defining_class + / -Returns underlying file descriptor if one exists. +Return underlying file descriptor if one exists. -OSError is raised if the IO object does not use a file descriptor. +Raise OSError if the IO object does not use a file descriptor. [clinic start generated code]*/ static PyObject * -_io__IOBase_fileno_impl(PyObject *self) -/*[clinic end generated code: output=7cc0973f0f5f3b73 input=4e37028947dc1cc8]*/ +_io__IOBase_fileno_impl(PyObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=7caaa32a6f4ada3d input=1927c8bea5c85099]*/ { _PyIO_State *state = IO_STATE(); return iobase_unsupported(state, "fileno"); diff --git a/Modules/_testcapi/gc.c b/Modules/_testcapi/gc.c new file mode 100644 index 00000000000000..829200ad12cd3c --- /dev/null +++ b/Modules/_testcapi/gc.c @@ -0,0 +1,344 @@ +#include "parts.h" + +static PyObject* +test_gc_control(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + int orig_enabled = PyGC_IsEnabled(); + const char* msg = "ok"; + int old_state; + + old_state = PyGC_Enable(); + msg = "Enable(1)"; + if (old_state != orig_enabled) { + goto failed; + } + msg = "IsEnabled(1)"; + if (!PyGC_IsEnabled()) { + goto failed; + } + + old_state = PyGC_Disable(); + msg = "disable(2)"; + if (!old_state) { + goto failed; + } + msg = "IsEnabled(2)"; + if (PyGC_IsEnabled()) { + goto failed; + } + + old_state = PyGC_Enable(); + msg = "enable(3)"; + if (old_state) { + goto failed; + } + msg = "IsEnabled(3)"; + if (!PyGC_IsEnabled()) { + goto failed; + } + + if (!orig_enabled) { + old_state = PyGC_Disable(); + msg = "disable(4)"; + if (old_state) { + goto failed; + } + msg = "IsEnabled(4)"; + if (PyGC_IsEnabled()) { + goto failed; + } + } + + Py_RETURN_NONE; + +failed: + /* Try to clean up if we can. */ + if (orig_enabled) { + PyGC_Enable(); + } else { + PyGC_Disable(); + } + PyErr_Format(PyExc_ValueError, "GC control failed in %s", msg); + return NULL; +} + +static PyObject * +without_gc(PyObject *Py_UNUSED(self), PyObject *obj) +{ + PyTypeObject *tp = (PyTypeObject*)obj; + if (!PyType_Check(obj) || !PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + return PyErr_Format(PyExc_TypeError, "heap type expected, got %R", obj); + } + if (PyType_IS_GC(tp)) { + // Don't try this at home, kids: + tp->tp_flags -= Py_TPFLAGS_HAVE_GC; + tp->tp_free = PyObject_Del; + tp->tp_traverse = NULL; + tp->tp_clear = NULL; + } + assert(!PyType_IS_GC(tp)); + return Py_NewRef(obj); +} + +static void +slot_tp_del(PyObject *self) +{ + PyObject *del, *res; + + /* Temporarily resurrect the object. */ + assert(Py_REFCNT(self) == 0); + Py_SET_REFCNT(self, 1); + + /* Save the current exception, if any. */ + PyObject *exc = PyErr_GetRaisedException(); + + PyObject *tp_del = PyUnicode_InternFromString("__tp_del__"); + if (tp_del == NULL) { + PyErr_WriteUnraisable(NULL); + PyErr_SetRaisedException(exc); + return; + } + /* Execute __del__ method, if any. */ + del = _PyType_Lookup(Py_TYPE(self), tp_del); + Py_DECREF(tp_del); + if (del != NULL) { + res = PyObject_CallOneArg(del, self); + if (res == NULL) + PyErr_WriteUnraisable(del); + else + Py_DECREF(res); + } + + /* Restore the saved exception. */ + PyErr_SetRaisedException(exc); + + /* Undo the temporary resurrection; can't use DECREF here, it would + * cause a recursive call. + */ + assert(Py_REFCNT(self) > 0); + Py_SET_REFCNT(self, Py_REFCNT(self) - 1); + if (Py_REFCNT(self) == 0) { + /* this is the normal path out */ + return; + } + + /* __del__ resurrected it! Make it look like the original Py_DECREF + * never happened. + */ + { + Py_ssize_t refcnt = Py_REFCNT(self); + _Py_NewReferenceNoTotal(self); + Py_SET_REFCNT(self, refcnt); + } + assert(!PyType_IS_GC(Py_TYPE(self)) || PyObject_GC_IsTracked(self)); +} + +static PyObject * +with_tp_del(PyObject *self, PyObject *args) +{ + PyObject *obj; + PyTypeObject *tp; + + if (!PyArg_ParseTuple(args, "O:with_tp_del", &obj)) + return NULL; + tp = (PyTypeObject *) obj; + if (!PyType_Check(obj) || !PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + PyErr_Format(PyExc_TypeError, + "heap type expected, got %R", obj); + return NULL; + } + tp->tp_del = slot_tp_del; + return Py_NewRef(obj); +} + + +struct gc_visit_state_basic { + PyObject *target; + int found; +}; + +static int +gc_visit_callback_basic(PyObject *obj, void *arg) +{ + struct gc_visit_state_basic *state = (struct gc_visit_state_basic *)arg; + if (obj == state->target) { + state->found = 1; + return 0; + } + return 1; +} + +static PyObject * +test_gc_visit_objects_basic(PyObject *Py_UNUSED(self), + PyObject *Py_UNUSED(ignored)) +{ + PyObject *obj; + struct gc_visit_state_basic state; + + obj = PyList_New(0); + if (obj == NULL) { + return NULL; + } + state.target = obj; + state.found = 0; + + PyUnstable_GC_VisitObjects(gc_visit_callback_basic, &state); + Py_DECREF(obj); + if (!state.found) { + PyErr_SetString( + PyExc_AssertionError, + "test_gc_visit_objects_basic: Didn't find live list"); + return NULL; + } + Py_RETURN_NONE; +} + +static int +gc_visit_callback_exit_early(PyObject *obj, void *arg) + { + int *visited_i = (int *)arg; + (*visited_i)++; + if (*visited_i == 2) { + return 0; + } + return 1; +} + +static PyObject * +test_gc_visit_objects_exit_early(PyObject *Py_UNUSED(self), + PyObject *Py_UNUSED(ignored)) +{ + int visited_i = 0; + PyUnstable_GC_VisitObjects(gc_visit_callback_exit_early, &visited_i); + if (visited_i != 2) { + PyErr_SetString( + PyExc_AssertionError, + "test_gc_visit_objects_exit_early: did not exit when expected"); + } + Py_RETURN_NONE; +} + +typedef struct { + PyObject_HEAD +} ObjExtraData; + +static PyObject * +obj_extra_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + size_t extra_size = sizeof(PyObject *); + PyObject *obj = PyUnstable_Object_GC_NewWithExtraData(type, extra_size); + if (obj == NULL) { + return PyErr_NoMemory(); + } + PyObject_GC_Track(obj); + return obj; +} + +static PyObject ** +obj_extra_data_get_extra_storage(PyObject *self) +{ + return (PyObject **)((char *)self + Py_TYPE(self)->tp_basicsize); +} + +static PyObject * +obj_extra_data_get(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + PyObject *value = *extra_storage; + if (!value) { + Py_RETURN_NONE; + } + return Py_NewRef(value); +} + +static int +obj_extra_data_set(PyObject *self, PyObject *newval, void *Py_UNUSED(ignored)) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + Py_CLEAR(*extra_storage); + if (newval) { + *extra_storage = Py_NewRef(newval); + } + return 0; +} + +static PyGetSetDef obj_extra_data_getset[] = { + {"extra", (getter)obj_extra_data_get, (setter)obj_extra_data_set, NULL}, + {NULL} +}; + +static int +obj_extra_data_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + PyObject *value = *extra_storage; + Py_VISIT(value); + return 0; +} + +static int +obj_extra_data_clear(PyObject *self) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + Py_CLEAR(*extra_storage); + return 0; +} + +static void +obj_extra_data_dealloc(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + obj_extra_data_clear(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyType_Slot ObjExtraData_Slots[] = { + {Py_tp_getset, obj_extra_data_getset}, + {Py_tp_dealloc, obj_extra_data_dealloc}, + {Py_tp_traverse, obj_extra_data_traverse}, + {Py_tp_clear, obj_extra_data_clear}, + {Py_tp_new, obj_extra_data_new}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +static PyType_Spec ObjExtraData_TypeSpec = { + .name = "_testcapi.ObjExtraData", + .basicsize = sizeof(ObjExtraData), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .slots = ObjExtraData_Slots, +}; + +static PyMethodDef test_methods[] = { + {"test_gc_control", test_gc_control, METH_NOARGS}, + {"test_gc_visit_objects_basic", test_gc_visit_objects_basic, METH_NOARGS, NULL}, + {"test_gc_visit_objects_exit_early", test_gc_visit_objects_exit_early, METH_NOARGS, NULL}, + {"without_gc", without_gc, METH_O, NULL}, + {"with_tp_del", with_tp_del, METH_VARARGS, NULL}, + {NULL} +}; + +int _PyTestCapi_Init_GC(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + + PyObject *ObjExtraData_Type = PyType_FromModuleAndSpec( + mod, &ObjExtraData_TypeSpec, NULL); + if (ObjExtraData_Type == 0) { + return -1; + } + int ret = PyModule_AddType(mod, (PyTypeObject*)ObjExtraData_Type); + Py_DECREF(ObjExtraData_Type); + if (ret < 0) { + return ret; + } + + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 663d4f2255de88..d1991ac6b464f2 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -41,6 +41,7 @@ int _PyTestCapi_Init_Code(PyObject *module); int _PyTestCapi_Init_Buffer(PyObject *module); int _PyTestCapi_Init_PyOS(PyObject *module); int _PyTestCapi_Init_Immortal(PyObject *module); +int _PyTestCapi_Init_GC(PyObject *mod); #ifdef LIMITED_API_AVAILABLE int _PyTestCapi_Init_VectorcallLimited(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 79ab7d3f5555c2..c29d29c4791134 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -154,68 +154,6 @@ test_sizeof_c_types(PyObject *self, PyObject *Py_UNUSED(ignored)) #endif } -static PyObject* -test_gc_control(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - int orig_enabled = PyGC_IsEnabled(); - const char* msg = "ok"; - int old_state; - - old_state = PyGC_Enable(); - msg = "Enable(1)"; - if (old_state != orig_enabled) { - goto failed; - } - msg = "IsEnabled(1)"; - if (!PyGC_IsEnabled()) { - goto failed; - } - - old_state = PyGC_Disable(); - msg = "disable(2)"; - if (!old_state) { - goto failed; - } - msg = "IsEnabled(2)"; - if (PyGC_IsEnabled()) { - goto failed; - } - - old_state = PyGC_Enable(); - msg = "enable(3)"; - if (old_state) { - goto failed; - } - msg = "IsEnabled(3)"; - if (!PyGC_IsEnabled()) { - goto failed; - } - - if (!orig_enabled) { - old_state = PyGC_Disable(); - msg = "disable(4)"; - if (old_state) { - goto failed; - } - msg = "IsEnabled(4)"; - if (PyGC_IsEnabled()) { - goto failed; - } - } - - Py_RETURN_NONE; - -failed: - /* Try to clean up if we can. */ - if (orig_enabled) { - PyGC_Enable(); - } else { - PyGC_Disable(); - } - PyErr_Format(TestError, "GC control failed in %s", msg); - return NULL; -} - static PyObject* test_list_api(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -1627,95 +1565,6 @@ restore_crossinterp_data(PyObject *self, PyObject *args) return _PyCrossInterpreterData_NewObject(data); } -static void -slot_tp_del(PyObject *self) -{ - PyObject *del, *res; - - /* Temporarily resurrect the object. */ - assert(Py_REFCNT(self) == 0); - Py_SET_REFCNT(self, 1); - - /* Save the current exception, if any. */ - PyObject *exc = PyErr_GetRaisedException(); - - PyObject *tp_del = PyUnicode_InternFromString("__tp_del__"); - if (tp_del == NULL) { - PyErr_WriteUnraisable(NULL); - PyErr_SetRaisedException(exc); - return; - } - /* Execute __del__ method, if any. */ - del = _PyType_Lookup(Py_TYPE(self), tp_del); - Py_DECREF(tp_del); - if (del != NULL) { - res = PyObject_CallOneArg(del, self); - if (res == NULL) - PyErr_WriteUnraisable(del); - else - Py_DECREF(res); - } - - /* Restore the saved exception. */ - PyErr_SetRaisedException(exc); - - /* Undo the temporary resurrection; can't use DECREF here, it would - * cause a recursive call. - */ - assert(Py_REFCNT(self) > 0); - Py_SET_REFCNT(self, Py_REFCNT(self) - 1); - if (Py_REFCNT(self) == 0) { - /* this is the normal path out */ - return; - } - - /* __del__ resurrected it! Make it look like the original Py_DECREF - * never happened. - */ - { - Py_ssize_t refcnt = Py_REFCNT(self); - _Py_NewReferenceNoTotal(self); - Py_SET_REFCNT(self, refcnt); - } - assert(!PyType_IS_GC(Py_TYPE(self)) || PyObject_GC_IsTracked(self)); -} - -static PyObject * -with_tp_del(PyObject *self, PyObject *args) -{ - PyObject *obj; - PyTypeObject *tp; - - if (!PyArg_ParseTuple(args, "O:with_tp_del", &obj)) - return NULL; - tp = (PyTypeObject *) obj; - if (!PyType_Check(obj) || !PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { - PyErr_Format(PyExc_TypeError, - "heap type expected, got %R", obj); - return NULL; - } - tp->tp_del = slot_tp_del; - return Py_NewRef(obj); -} - -static PyObject * -without_gc(PyObject *Py_UNUSED(self), PyObject *obj) -{ - PyTypeObject *tp = (PyTypeObject*)obj; - if (!PyType_Check(obj) || !PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { - return PyErr_Format(PyExc_TypeError, "heap type expected, got %R", obj); - } - if (PyType_IS_GC(tp)) { - // Don't try this at home, kids: - tp->tp_flags -= Py_TPFLAGS_HAVE_GC; - tp->tp_free = PyObject_Del; - tp->tp_traverse = NULL; - tp->tp_clear = NULL; - } - assert(!PyType_IS_GC(tp)); - return Py_NewRef(obj); -} - static PyMethodDef ml; static PyObject * @@ -3342,165 +3191,6 @@ function_set_kw_defaults(PyObject *self, PyObject *args) Py_RETURN_NONE; } -struct gc_visit_state_basic { - PyObject *target; - int found; -}; - -static int -gc_visit_callback_basic(PyObject *obj, void *arg) -{ - struct gc_visit_state_basic *state = (struct gc_visit_state_basic *)arg; - if (obj == state->target) { - state->found = 1; - return 0; - } - return 1; -} - -static PyObject * -test_gc_visit_objects_basic(PyObject *Py_UNUSED(self), - PyObject *Py_UNUSED(ignored)) -{ - PyObject *obj; - struct gc_visit_state_basic state; - - obj = PyList_New(0); - if (obj == NULL) { - return NULL; - } - state.target = obj; - state.found = 0; - - PyUnstable_GC_VisitObjects(gc_visit_callback_basic, &state); - Py_DECREF(obj); - if (!state.found) { - PyErr_SetString( - PyExc_AssertionError, - "test_gc_visit_objects_basic: Didn't find live list"); - return NULL; - } - Py_RETURN_NONE; -} - -static int -gc_visit_callback_exit_early(PyObject *obj, void *arg) - { - int *visited_i = (int *)arg; - (*visited_i)++; - if (*visited_i == 2) { - return 0; - } - return 1; -} - -static PyObject * -test_gc_visit_objects_exit_early(PyObject *Py_UNUSED(self), - PyObject *Py_UNUSED(ignored)) -{ - int visited_i = 0; - PyUnstable_GC_VisitObjects(gc_visit_callback_exit_early, &visited_i); - if (visited_i != 2) { - PyErr_SetString( - PyExc_AssertionError, - "test_gc_visit_objects_exit_early: did not exit when expected"); - } - Py_RETURN_NONE; -} - -typedef struct { - PyObject_HEAD -} ObjExtraData; - -static PyObject * -obj_extra_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - size_t extra_size = sizeof(PyObject *); - PyObject *obj = PyUnstable_Object_GC_NewWithExtraData(type, extra_size); - if (obj == NULL) { - return PyErr_NoMemory(); - } - PyObject_GC_Track(obj); - return obj; -} - -static PyObject ** -obj_extra_data_get_extra_storage(PyObject *self) -{ - return (PyObject **)((char *)self + Py_TYPE(self)->tp_basicsize); -} - -static PyObject * -obj_extra_data_get(PyObject *self, void *Py_UNUSED(ignored)) -{ - PyObject **extra_storage = obj_extra_data_get_extra_storage(self); - PyObject *value = *extra_storage; - if (!value) { - Py_RETURN_NONE; - } - return Py_NewRef(value); -} - -static int -obj_extra_data_set(PyObject *self, PyObject *newval, void *Py_UNUSED(ignored)) -{ - PyObject **extra_storage = obj_extra_data_get_extra_storage(self); - Py_CLEAR(*extra_storage); - if (newval) { - *extra_storage = Py_NewRef(newval); - } - return 0; -} - -static PyGetSetDef obj_extra_data_getset[] = { - {"extra", (getter)obj_extra_data_get, (setter)obj_extra_data_set, NULL}, - {NULL} -}; - -static int -obj_extra_data_traverse(PyObject *self, visitproc visit, void *arg) -{ - PyObject **extra_storage = obj_extra_data_get_extra_storage(self); - PyObject *value = *extra_storage; - Py_VISIT(value); - return 0; -} - -static int -obj_extra_data_clear(PyObject *self) -{ - PyObject **extra_storage = obj_extra_data_get_extra_storage(self); - Py_CLEAR(*extra_storage); - return 0; -} - -static void -obj_extra_data_dealloc(PyObject *self) -{ - PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); - obj_extra_data_clear(self); - tp->tp_free(self); - Py_DECREF(tp); -} - -static PyType_Slot ObjExtraData_Slots[] = { - {Py_tp_getset, obj_extra_data_getset}, - {Py_tp_dealloc, obj_extra_data_dealloc}, - {Py_tp_traverse, obj_extra_data_traverse}, - {Py_tp_clear, obj_extra_data_clear}, - {Py_tp_new, obj_extra_data_new}, - {Py_tp_free, PyObject_GC_Del}, - {0, NULL}, -}; - -static PyType_Spec ObjExtraData_TypeSpec = { - .name = "_testcapi.ObjExtraData", - .basicsize = sizeof(ObjExtraData), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - .slots = ObjExtraData_Slots, -}; - struct atexit_data { int called; }; @@ -3538,7 +3228,6 @@ static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, {"test_sizeof_c_types", test_sizeof_c_types, METH_NOARGS}, - {"test_gc_control", test_gc_control, METH_NOARGS}, {"test_list_api", test_list_api, METH_NOARGS}, {"test_dict_iteration", test_dict_iteration, METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, @@ -3590,7 +3279,6 @@ static PyMethodDef TestMethods[] = { METH_VARARGS | METH_KEYWORDS}, {"get_crossinterp_data", get_crossinterp_data, METH_VARARGS}, {"restore_crossinterp_data", restore_crossinterp_data, METH_VARARGS}, - {"with_tp_del", with_tp_del, METH_VARARGS}, {"create_cfunction", create_cfunction, METH_NOARGS}, {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_VARARGS, PyDoc_STR("set_error_class(error_class) -> None")}, @@ -3641,7 +3329,6 @@ static PyMethodDef TestMethods[] = { {"meth_fastcall", _PyCFunction_CAST(meth_fastcall), METH_FASTCALL}, {"meth_fastcall_keywords", _PyCFunction_CAST(meth_fastcall_keywords), METH_FASTCALL|METH_KEYWORDS}, {"pynumber_tobase", pynumber_tobase, METH_VARARGS}, - {"without_gc", without_gc, METH_O}, {"test_set_type_size", test_set_type_size, METH_NOARGS}, {"test_py_clear", test_py_clear, METH_NOARGS}, {"test_py_setref", test_py_setref, METH_NOARGS}, @@ -3675,8 +3362,6 @@ static PyMethodDef TestMethods[] = { {"function_set_defaults", function_set_defaults, METH_VARARGS, NULL}, {"function_get_kw_defaults", function_get_kw_defaults, METH_O, NULL}, {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, - {"test_gc_visit_objects_basic", test_gc_visit_objects_basic, METH_NOARGS, NULL}, - {"test_gc_visit_objects_exit_early", test_gc_visit_objects_exit_early, METH_NOARGS, NULL}, {"test_atexit", test_atexit, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; @@ -4223,17 +3908,6 @@ PyInit__testcapi(void) Py_INCREF(&MethStatic_Type); PyModule_AddObject(m, "MethStatic", (PyObject *)&MethStatic_Type); - PyObject *ObjExtraData_Type = PyType_FromModuleAndSpec( - m, &ObjExtraData_TypeSpec, NULL); - if (ObjExtraData_Type == 0) { - return NULL; - } - int ret = PyModule_AddType(m, (PyTypeObject*)ObjExtraData_Type); - Py_DECREF(ObjExtraData_Type); - if (ret < 0) { - return NULL; - } - PyModule_AddObject(m, "CHAR_MAX", PyLong_FromLong(CHAR_MAX)); PyModule_AddObject(m, "CHAR_MIN", PyLong_FromLong(CHAR_MIN)); PyModule_AddObject(m, "UCHAR_MAX", PyLong_FromLong(UCHAR_MAX)); @@ -4327,6 +4001,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Immortal(m) < 0) { return NULL; } + if (_PyTestCapi_Init_GC(m) < 0) { + return NULL; + } #ifndef LIMITED_API_AVAILABLE PyModule_AddObjectRef(m, "LIMITED_API_AVAILABLE", Py_False); diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 3b2d282d65cab9..c8c791b6d7c0d8 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -1709,11 +1709,11 @@ static Py_ssize_t parse_abbr(const char *const p, PyObject **abbr) { const char *ptr = p; - char buff = *ptr; const char *str_start; const char *str_end; if (*ptr == '<') { + char buff; ptr++; str_start = ptr; while ((buff = *ptr) != '>') { diff --git a/Objects/frameobject.c b/Objects/frameobject.c index d9aaea7831a380..2c90a6b71311ca 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -831,7 +831,6 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore start_stack = pop_value(start_stack); } /* Finally set the new lasti and return OK. */ - f->f_last_traced_line = new_lineno; f->f_lineno = 0; f->f_frame->prev_instr = _PyCode_CODE(f->f_frame->f_code) + best_addr; return 0; @@ -854,7 +853,6 @@ frame_settrace(PyFrameObject *f, PyObject* v, void *closure) } if (v != f->f_trace) { Py_XSETREF(f->f_trace, Py_XNewRef(v)); - f->f_last_traced_line = -1; } return 0; } @@ -1056,7 +1054,6 @@ _PyFrame_New_NoTrack(PyCodeObject *code) f->f_trace_opcodes = 0; f->f_fast_as_locals = 0; f->f_lineno = 0; - f->f_last_traced_line = -1; return f; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index f1745c91967358..c1525320a7cbd5 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -9117,6 +9117,12 @@ releasebuffer_maybe_call_super(PyObject *self, Py_buffer *buffer) static void releasebuffer_call_python(PyObject *self, Py_buffer *buffer) { + // bf_releasebuffer may be called while an exception is already active. + // We have no way to report additional errors up the stack, because + // this slot returns void, so we simply stash away the active exception + // and restore it after the call to Python returns. + PyObject *exc = PyErr_GetRaisedException(); + PyObject *mv; bool is_buffer_wrapper = Py_TYPE(buffer->obj) == &_PyBufferWrapper_Type; if (is_buffer_wrapper) { @@ -9124,7 +9130,7 @@ releasebuffer_call_python(PyObject *self, Py_buffer *buffer) // __release_buffer__() that __buffer__() returned. PyBufferWrapper *bw = (PyBufferWrapper *)buffer->obj; if (bw->mv == NULL) { - return; + goto end; } mv = Py_NewRef(bw->mv); } @@ -9134,7 +9140,7 @@ releasebuffer_call_python(PyObject *self, Py_buffer *buffer) mv = PyMemoryView_FromBuffer(buffer); if (mv == NULL) { PyErr_WriteUnraisable(self); - return; + goto end; } // Set the memoryview to restricted mode, which forbids // users from saving any reference to the underlying buffer @@ -9152,9 +9158,19 @@ releasebuffer_call_python(PyObject *self, Py_buffer *buffer) Py_DECREF(ret); } if (!is_buffer_wrapper) { - PyObject_CallMethodNoArgs(mv, &_Py_ID(release)); + PyObject *res = PyObject_CallMethodNoArgs(mv, &_Py_ID(release)); + if (res == NULL) { + PyErr_WriteUnraisable(self); + } + else { + Py_DECREF(res); + } } Py_DECREF(mv); +end: + assert(!PyErr_Occurred()); + + PyErr_SetRaisedException(exc); } /* @@ -10267,8 +10283,10 @@ super_descr_get(PyObject *self, PyObject *obj, PyObject *type) return NULL; newobj = (superobject *)PySuper_Type.tp_new(&PySuper_Type, NULL, NULL); - if (newobj == NULL) + if (newobj == NULL) { + Py_DECREF(obj_type); return NULL; + } newobj->type = (PyTypeObject*)Py_NewRef(su->type); newobj->obj = Py_NewRef(obj); newobj->obj_type = obj_type; diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 350f97f8ff41aa..3db9426d1a25ff 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -113,6 +113,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index af80f1eebb3c4d..8df4874659fa1c 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -66,6 +66,9 @@ Source Files + + Source Files + diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 987052bffe8edd..bdfd963dfb7579 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1630,8 +1630,8 @@ dummy_func( STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - ERROR_IF(res == NULL, error); DECREF_INPUTS(); + ERROR_IF(res == NULL, error); } inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super, class, self -- res2, res)) { @@ -3320,28 +3320,6 @@ dummy_func( assert(oparg >= 2); } - inst(INSTRUMENTED_LINE, ( -- )) { - _Py_CODEUNIT *here = next_instr-1; - _PyFrame_SetStackPointer(frame, stack_pointer); - int original_opcode = _Py_call_instrumentation_line( - tstate, frame, here); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = here+1; - goto error; - } - next_instr = frame->prev_instr; - if (next_instr != here) { - DISPATCH(); - } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - INCREMENT_ADAPTIVE_COUNTER(cache->counter); - } - opcode = original_opcode; - DISPATCH_GOTO(); - } - inst(INSTRUMENTED_INSTRUCTION, ( -- )) { int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); diff --git a/Python/ceval.c b/Python/ceval.c index 56a3b123f46331..e8534ec6598e2b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -775,6 +775,41 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #include "generated_cases.c.h" + /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c, + * because it needs to capture frame->prev_instr before it is updated, + * as happens in the standard instruction prologue. + */ +#if USE_COMPUTED_GOTOS + TARGET_INSTRUMENTED_LINE: +#else + case INSTRUMENTED_LINE: +#endif + { + _Py_CODEUNIT *prev = frame->prev_instr; + _Py_CODEUNIT *here = frame->prev_instr = next_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + int original_opcode = _Py_call_instrumentation_line( + tstate, frame, here, prev); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->prev_instr; + if (next_instr != here) { + DISPATCH(); + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + /* Prevent the underlying instruction from specializing + * and overwriting the instrumentation. */ + INCREMENT_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } + + #if USE_COMPUTED_GOTOS _unknown_opcode: #else diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 485771ac65a767..f5515d0d32dedf 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -334,11 +334,10 @@ do { \ #define INSTRUMENTED_JUMP(src, dest, event) \ do { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - int err = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ + next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (err) { \ + if (next_instr == NULL) { \ next_instr = (dest)+1; \ goto error; \ } \ - next_instr = frame->prev_instr; \ } while (0); diff --git a/Python/compile.c b/Python/compile.c index 1fdbf1b85b4295..823de2aa34bc1b 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5030,14 +5030,19 @@ push_inlined_comprehension_state(struct compiler *c, location loc, long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; PyObject *outv = PyDict_GetItemWithError(c->u->u_ste->ste_symbols, k); if (outv == NULL) { + assert(PyErr_Occurred()); return ERROR; } assert(PyLong_Check(outv)); long outsc = (PyLong_AS_LONG(outv) >> SCOPE_OFFSET) & SCOPE_MASK; - if (scope != outsc) { + if (scope != outsc && !(scope == CELL && outsc == FREE)) { // If a name has different scope inside than outside the // comprehension, we need to temporarily handle it with the - // right scope while compiling the comprehension. + // right scope while compiling the comprehension. (If it's free + // in outer scope and cell in inner scope, we can't treat it as + // both cell and free in the same function, but treating it as + // free throughout is fine; it's *_DEREF either way.) + if (state->temp_symbols == NULL) { state->temp_symbols = PyDict_New(); if (state->temp_symbols == NULL) { @@ -5073,7 +5078,11 @@ push_inlined_comprehension_state(struct compiler *c, location loc, // comprehension and restore the original one after ADDOP_NAME(c, loc, LOAD_FAST_AND_CLEAR, k, varnames); if (scope == CELL) { - ADDOP_NAME(c, loc, MAKE_CELL, k, cellvars); + if (outsc == FREE) { + ADDOP_NAME(c, loc, MAKE_CELL, k, freevars); + } else { + ADDOP_NAME(c, loc, MAKE_CELL, k, cellvars); + } } if (PyList_Append(state->pushed_locals, k) < 0) { return ERROR; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2bcf9aad025dff..7d47ca0b9ca980 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2271,11 +2271,13 @@ STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(frame->f_code->co_names, oparg >> 2); res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - if (res == NULL) goto pop_3_error; - #line 2276 "Python/generated_cases.c.h" + #line 2275 "Python/generated_cases.c.h" Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); + #line 1634 "Python/bytecodes.c" + if (res == NULL) goto pop_3_error; + #line 2281 "Python/generated_cases.c.h" STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2311,7 +2313,7 @@ res = res2; res2 = NULL; } - #line 2315 "Python/generated_cases.c.h" + #line 2317 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; stack_pointer[-2] = res2; @@ -2359,7 +2361,7 @@ NULL | meth | arg1 | ... | argN */ - #line 2363 "Python/generated_cases.c.h" + #line 2365 "Python/generated_cases.c.h" Py_DECREF(owner); #line 1709 "Python/bytecodes.c" if (meth == NULL) goto pop_1_error; @@ -2370,12 +2372,12 @@ else { /* Classic, pushes one value. */ res = PyObject_GetAttr(owner, name); - #line 2374 "Python/generated_cases.c.h" + #line 2376 "Python/generated_cases.c.h" Py_DECREF(owner); #line 1718 "Python/bytecodes.c" if (res == NULL) goto pop_1_error; } - #line 2379 "Python/generated_cases.c.h" + #line 2381 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -2402,7 +2404,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2406 "Python/generated_cases.c.h" + #line 2408 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2430,7 +2432,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2434 "Python/generated_cases.c.h" + #line 2436 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2472,7 +2474,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2476 "Python/generated_cases.c.h" + #line 2478 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2497,7 +2499,7 @@ STAT_INC(LOAD_ATTR, hit); Py_INCREF(res); res2 = NULL; - #line 2501 "Python/generated_cases.c.h" + #line 2503 "Python/generated_cases.c.h" Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2524,7 +2526,7 @@ res = descr; assert(res != NULL); Py_INCREF(res); - #line 2528 "Python/generated_cases.c.h" + #line 2530 "Python/generated_cases.c.h" Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; @@ -2562,7 +2564,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 2566 "Python/generated_cases.c.h" + #line 2568 "Python/generated_cases.c.h" } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2596,7 +2598,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 2600 "Python/generated_cases.c.h" + #line 2602 "Python/generated_cases.c.h" } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -2622,7 +2624,7 @@ Py_DECREF(old_value); } Py_DECREF(owner); - #line 2626 "Python/generated_cases.c.h" + #line 2628 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2672,7 +2674,7 @@ /* PEP 509 */ dict->ma_version_tag = new_version; Py_DECREF(owner); - #line 2676 "Python/generated_cases.c.h" + #line 2678 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2693,7 +2695,7 @@ *(PyObject **)addr = value; Py_XDECREF(old_value); Py_DECREF(owner); - #line 2697 "Python/generated_cases.c.h" + #line 2699 "Python/generated_cases.c.h" STACK_SHRINK(2); next_instr += 4; DISPATCH(); @@ -2718,12 +2720,12 @@ #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 4) <= Py_GE); res = PyObject_RichCompare(left, right, oparg>>4); - #line 2722 "Python/generated_cases.c.h" + #line 2724 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); #line 1960 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2727 "Python/generated_cases.c.h" + #line 2729 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2746,7 +2748,7 @@ _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2750 "Python/generated_cases.c.h" + #line 2752 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2773,7 +2775,7 @@ _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); res = (sign_ish & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2777 "Python/generated_cases.c.h" + #line 2779 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2797,7 +2799,7 @@ assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False; Py_INCREF(res); - #line 2801 "Python/generated_cases.c.h" + #line 2803 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -2810,12 +2812,12 @@ PyObject *b; #line 2013 "Python/bytecodes.c" int res = Py_Is(left, right) ^ oparg; - #line 2814 "Python/generated_cases.c.h" + #line 2816 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); #line 2015 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2819 "Python/generated_cases.c.h" + #line 2821 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2827,13 +2829,13 @@ PyObject *b; #line 2019 "Python/bytecodes.c" int res = PySequence_Contains(right, left); - #line 2831 "Python/generated_cases.c.h" + #line 2833 "Python/generated_cases.c.h" Py_DECREF(left); Py_DECREF(right); #line 2021 "Python/bytecodes.c" if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); - #line 2837 "Python/generated_cases.c.h" + #line 2839 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = b; DISPATCH(); @@ -2846,7 +2848,7 @@ PyObject *match; #line 2026 "Python/bytecodes.c" if (check_except_star_type_valid(tstate, match_type) < 0) { - #line 2850 "Python/generated_cases.c.h" + #line 2852 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); #line 2028 "Python/bytecodes.c" @@ -2857,7 +2859,7 @@ rest = NULL; int res = exception_group_match(exc_value, match_type, &match, &rest); - #line 2861 "Python/generated_cases.c.h" + #line 2863 "Python/generated_cases.c.h" Py_DECREF(exc_value); Py_DECREF(match_type); #line 2036 "Python/bytecodes.c" @@ -2869,7 +2871,7 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - #line 2873 "Python/generated_cases.c.h" + #line 2875 "Python/generated_cases.c.h" stack_pointer[-1] = match; stack_pointer[-2] = rest; DISPATCH(); @@ -2882,18 +2884,18 @@ #line 2047 "Python/bytecodes.c" assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { - #line 2886 "Python/generated_cases.c.h" + #line 2888 "Python/generated_cases.c.h" Py_DECREF(right); #line 2050 "Python/bytecodes.c" if (true) goto pop_1_error; } int res = PyErr_GivenExceptionMatches(left, right); - #line 2893 "Python/generated_cases.c.h" + #line 2895 "Python/generated_cases.c.h" Py_DECREF(right); #line 2055 "Python/bytecodes.c" b = Py_NewRef(res ? Py_True : Py_False); - #line 2897 "Python/generated_cases.c.h" + #line 2899 "Python/generated_cases.c.h" stack_pointer[-1] = b; DISPATCH(); } @@ -2905,12 +2907,12 @@ #line 2059 "Python/bytecodes.c" PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_name(tstate, frame, name, fromlist, level); - #line 2909 "Python/generated_cases.c.h" + #line 2911 "Python/generated_cases.c.h" Py_DECREF(level); Py_DECREF(fromlist); #line 2062 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 2914 "Python/generated_cases.c.h" + #line 2916 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -2923,7 +2925,7 @@ PyObject *name = GETITEM(frame->f_code->co_names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; - #line 2927 "Python/generated_cases.c.h" + #line 2929 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -2932,7 +2934,7 @@ TARGET(JUMP_FORWARD) { #line 2072 "Python/bytecodes.c" JUMPBY(oparg); - #line 2936 "Python/generated_cases.c.h" + #line 2938 "Python/generated_cases.c.h" DISPATCH(); } @@ -2941,7 +2943,7 @@ #line 2076 "Python/bytecodes.c" assert(oparg < INSTR_OFFSET()); JUMPBY(-oparg); - #line 2945 "Python/generated_cases.c.h" + #line 2947 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -2959,7 +2961,7 @@ } else { int err = PyObject_IsTrue(cond); - #line 2963 "Python/generated_cases.c.h" + #line 2965 "Python/generated_cases.c.h" Py_DECREF(cond); #line 2092 "Python/bytecodes.c" if (err == 0) { @@ -2969,7 +2971,7 @@ if (err < 0) goto pop_1_error; } } - #line 2973 "Python/generated_cases.c.h" + #line 2975 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -2986,7 +2988,7 @@ } else { int err = PyObject_IsTrue(cond); - #line 2990 "Python/generated_cases.c.h" + #line 2992 "Python/generated_cases.c.h" Py_DECREF(cond); #line 2112 "Python/bytecodes.c" if (err > 0) { @@ -2996,7 +2998,7 @@ if (err < 0) goto pop_1_error; } } - #line 3000 "Python/generated_cases.c.h" + #line 3002 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -3005,7 +3007,7 @@ PyObject *value = stack_pointer[-1]; #line 2122 "Python/bytecodes.c" if (!Py_IsNone(value)) { - #line 3009 "Python/generated_cases.c.h" + #line 3011 "Python/generated_cases.c.h" Py_DECREF(value); #line 2124 "Python/bytecodes.c" JUMPBY(oparg); @@ -3013,7 +3015,7 @@ else { _Py_DECREF_NO_DEALLOC(value); } - #line 3017 "Python/generated_cases.c.h" + #line 3019 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -3026,11 +3028,11 @@ JUMPBY(oparg); } else { - #line 3030 "Python/generated_cases.c.h" + #line 3032 "Python/generated_cases.c.h" Py_DECREF(value); #line 2138 "Python/bytecodes.c" } - #line 3034 "Python/generated_cases.c.h" + #line 3036 "Python/generated_cases.c.h" STACK_SHRINK(1); DISPATCH(); } @@ -3043,7 +3045,7 @@ * (see bpo-30039). */ JUMPBY(-oparg); - #line 3047 "Python/generated_cases.c.h" + #line 3049 "Python/generated_cases.c.h" DISPATCH(); } @@ -3056,7 +3058,7 @@ if (len_i < 0) goto error; len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; - #line 3060 "Python/generated_cases.c.h" + #line 3062 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = len_o; DISPATCH(); @@ -3072,7 +3074,7 @@ // None on failure. assert(PyTuple_CheckExact(names)); attrs = match_class(tstate, subject, type, oparg, names); - #line 3076 "Python/generated_cases.c.h" + #line 3078 "Python/generated_cases.c.h" Py_DECREF(subject); Py_DECREF(type); Py_DECREF(names); @@ -3084,7 +3086,7 @@ if (_PyErr_Occurred(tstate)) goto pop_3_error; attrs = Py_NewRef(Py_None); // Failure! } - #line 3088 "Python/generated_cases.c.h" + #line 3090 "Python/generated_cases.c.h" STACK_SHRINK(2); stack_pointer[-1] = attrs; DISPATCH(); @@ -3096,7 +3098,7 @@ #line 2174 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); - #line 3100 "Python/generated_cases.c.h" + #line 3102 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3109,7 +3111,7 @@ #line 2180 "Python/bytecodes.c" int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); - #line 3113 "Python/generated_cases.c.h" + #line 3115 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); @@ -3124,7 +3126,7 @@ // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; - #line 3128 "Python/generated_cases.c.h" + #line 3130 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = values_or_none; DISPATCH(); @@ -3136,11 +3138,11 @@ #line 2192 "Python/bytecodes.c" /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); - #line 3140 "Python/generated_cases.c.h" + #line 3142 "Python/generated_cases.c.h" Py_DECREF(iterable); #line 2195 "Python/bytecodes.c" if (iter == NULL) goto pop_1_error; - #line 3144 "Python/generated_cases.c.h" + #line 3146 "Python/generated_cases.c.h" stack_pointer[-1] = iter; DISPATCH(); } @@ -3171,11 +3173,11 @@ if (iter == NULL) { goto error; } - #line 3175 "Python/generated_cases.c.h" + #line 3177 "Python/generated_cases.c.h" Py_DECREF(iterable); #line 2222 "Python/bytecodes.c" } - #line 3179 "Python/generated_cases.c.h" + #line 3181 "Python/generated_cases.c.h" stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); @@ -3217,7 +3219,7 @@ DISPATCH(); } // Common case: no jump, leave it to the code generator - #line 3221 "Python/generated_cases.c.h" + #line 3223 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3251,7 +3253,7 @@ target = next_instr + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1; } INSTRUMENTED_JUMP(here, target, PY_MONITORING_EVENT_BRANCH); - #line 3255 "Python/generated_cases.c.h" + #line 3257 "Python/generated_cases.c.h" DISPATCH(); } @@ -3278,7 +3280,7 @@ DISPATCH(); end_for_iter_list: // Common case: no jump, leave it to the code generator - #line 3282 "Python/generated_cases.c.h" + #line 3284 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3308,7 +3310,7 @@ DISPATCH(); end_for_iter_tuple: // Common case: no jump, leave it to the code generator - #line 3312 "Python/generated_cases.c.h" + #line 3314 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3336,7 +3338,7 @@ if (next == NULL) { goto error; } - #line 3340 "Python/generated_cases.c.h" + #line 3342 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = next; next_instr += 1; @@ -3360,7 +3362,7 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); - #line 3364 "Python/generated_cases.c.h" + #line 3366 "Python/generated_cases.c.h" } TARGET(BEFORE_ASYNC_WITH) { @@ -3390,7 +3392,7 @@ Py_DECREF(enter); goto error; } - #line 3394 "Python/generated_cases.c.h" + #line 3396 "Python/generated_cases.c.h" Py_DECREF(mgr); #line 2406 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); @@ -3399,7 +3401,7 @@ Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3403 "Python/generated_cases.c.h" + #line 3405 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3437,7 +3439,7 @@ Py_DECREF(enter); goto error; } - #line 3441 "Python/generated_cases.c.h" + #line 3443 "Python/generated_cases.c.h" Py_DECREF(mgr); #line 2442 "Python/bytecodes.c" res = _PyObject_CallNoArgs(enter); @@ -3446,7 +3448,7 @@ Py_DECREF(exit); if (true) goto pop_1_error; } - #line 3450 "Python/generated_cases.c.h" + #line 3452 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; stack_pointer[-2] = exit; @@ -3479,7 +3481,7 @@ res = PyObject_Vectorcall(exit_func, stack + 1, 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; - #line 3483 "Python/generated_cases.c.h" + #line 3485 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = res; DISPATCH(); @@ -3498,7 +3500,7 @@ } assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); - #line 3502 "Python/generated_cases.c.h" + #line 3504 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; @@ -3529,7 +3531,7 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; assert(oparg & 1); - #line 3533 "Python/generated_cases.c.h" + #line 3535 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3553,7 +3555,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3557 "Python/generated_cases.c.h" + #line 3559 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3581,7 +3583,7 @@ res2 = Py_NewRef(descr); res = self; assert(oparg & 1); - #line 3585 "Python/generated_cases.c.h" + #line 3587 "Python/generated_cases.c.h" STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1] = res; if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } @@ -3594,7 +3596,7 @@ assert(kwnames == NULL); assert(oparg < PyTuple_GET_SIZE(frame->f_code->co_consts)); kwnames = GETITEM(frame->f_code->co_consts, oparg); - #line 3598 "Python/generated_cases.c.h" + #line 3600 "Python/generated_cases.c.h" DISPATCH(); } @@ -3612,7 +3614,7 @@ _PyCallCache *cache = (_PyCallCache *)next_instr; INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(CALL); - #line 3616 "Python/generated_cases.c.h" + #line 3618 "Python/generated_cases.c.h" } TARGET(CALL) { @@ -3704,7 +3706,7 @@ Py_DECREF(args[i]); } if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3708 "Python/generated_cases.c.h" + #line 3710 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3726,7 +3728,7 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); - #line 3730 "Python/generated_cases.c.h" + #line 3732 "Python/generated_cases.c.h" } TARGET(CALL_PY_EXACT_ARGS) { @@ -3761,7 +3763,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3765 "Python/generated_cases.c.h" + #line 3767 "Python/generated_cases.c.h" } TARGET(CALL_PY_WITH_DEFAULTS) { @@ -3805,7 +3807,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - #line 3809 "Python/generated_cases.c.h" + #line 3811 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3823,7 +3825,7 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3827 "Python/generated_cases.c.h" + #line 3829 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3847,7 +3849,7 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3851 "Python/generated_cases.c.h" + #line 3853 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3872,7 +3874,7 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3876 "Python/generated_cases.c.h" + #line 3878 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3908,7 +3910,7 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3912 "Python/generated_cases.c.h" + #line 3914 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3950,7 +3952,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3954 "Python/generated_cases.c.h" + #line 3956 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3996,7 +3998,7 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 4000 "Python/generated_cases.c.h" + #line 4002 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4042,7 +4044,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4046 "Python/generated_cases.c.h" + #line 4048 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4081,7 +4083,7 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4085 "Python/generated_cases.c.h" + #line 4087 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4121,7 +4123,7 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4125 "Python/generated_cases.c.h" + #line 4127 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4151,7 +4153,7 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 4155 "Python/generated_cases.c.h" + #line 4157 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { @@ -4189,7 +4191,7 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4193 "Python/generated_cases.c.h" + #line 4195 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4231,7 +4233,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4235 "Python/generated_cases.c.h" + #line 4237 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4273,7 +4275,7 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4277 "Python/generated_cases.c.h" + #line 4279 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4314,7 +4316,7 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 4318 "Python/generated_cases.c.h" + #line 4320 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -4326,7 +4328,7 @@ TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { #line 3122 "Python/bytecodes.c" GO_TO_INSTRUCTION(CALL_FUNCTION_EX); - #line 4330 "Python/generated_cases.c.h" + #line 4332 "Python/generated_cases.c.h" } TARGET(CALL_FUNCTION_EX) { @@ -4397,14 +4399,14 @@ } result = PyObject_Call(func, callargs, kwargs); } - #line 4401 "Python/generated_cases.c.h" + #line 4403 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); #line 3188 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4408 "Python/generated_cases.c.h" + #line 4410 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4448,7 +4450,7 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4452 "Python/generated_cases.c.h" + #line 4454 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); @@ -4476,7 +4478,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4480 "Python/generated_cases.c.h" + #line 4482 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4486,13 +4488,13 @@ PyObject *slice; #line 3252 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4490 "Python/generated_cases.c.h" + #line 4492 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); #line 3254 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4496 "Python/generated_cases.c.h" + #line 4498 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4538,7 +4540,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4542 "Python/generated_cases.c.h" + #line 4544 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4550,7 +4552,7 @@ #line 3295 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4554 "Python/generated_cases.c.h" + #line 4556 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4577,12 +4579,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4581 "Python/generated_cases.c.h" + #line 4583 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); #line 3315 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4586 "Python/generated_cases.c.h" + #line 4588 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4594,38 +4596,14 @@ PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; #line 3320 "Python/bytecodes.c" assert(oparg >= 2); - #line 4598 "Python/generated_cases.c.h" + #line 4600 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } - TARGET(INSTRUMENTED_LINE) { - #line 3324 "Python/bytecodes.c" - _Py_CODEUNIT *here = next_instr-1; - _PyFrame_SetStackPointer(frame, stack_pointer); - int original_opcode = _Py_call_instrumentation_line( - tstate, frame, here); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = here+1; - goto error; - } - next_instr = frame->prev_instr; - if (next_instr != here) { - DISPATCH(); - } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - INCREMENT_ADAPTIVE_COUNTER(cache->counter); - } - opcode = original_opcode; - DISPATCH_GOTO(); - #line 4625 "Python/generated_cases.c.h" - } - TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3346 "Python/bytecodes.c" + #line 3324 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4637,26 +4615,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4641 "Python/generated_cases.c.h" + #line 4619 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3360 "Python/bytecodes.c" + #line 3338 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4647 "Python/generated_cases.c.h" + #line 4625 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3364 "Python/bytecodes.c" + #line 3342 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4654 "Python/generated_cases.c.h" + #line 4632 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3369 "Python/bytecodes.c" + #line 3347 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4665,12 +4643,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4669 "Python/generated_cases.c.h" + #line 4647 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3380 "Python/bytecodes.c" + #line 3358 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4679,12 +4657,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4683 "Python/generated_cases.c.h" + #line 4661 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3391 "Python/bytecodes.c" + #line 3369 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4697,12 +4675,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4701 "Python/generated_cases.c.h" + #line 4679 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3406 "Python/bytecodes.c" + #line 3384 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4715,30 +4693,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4719 "Python/generated_cases.c.h" + #line 4697 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3421 "Python/bytecodes.c" + #line 3399 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4730 "Python/generated_cases.c.h" + #line 4708 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3429 "Python/bytecodes.c" + #line 3407 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4737 "Python/generated_cases.c.h" + #line 4715 "Python/generated_cases.c.h" } TARGET(RESERVED) { - #line 3434 "Python/bytecodes.c" + #line 3412 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4744 "Python/generated_cases.c.h" + #line 4722 "Python/generated_cases.c.h" } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index ca9951fceaa13b..c70668e8295ae5 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -14,7 +14,7 @@ /* Uncomment this to dump debugging output when assertions fail */ // #define INSTRUMENT_DEBUG 1 -static PyObject DISABLE = +PyObject _PyInstrumentation_DISABLE = { .ob_refcnt = _Py_IMMORTAL_REFCNT, .ob_type = &PyBaseObject_Type @@ -864,7 +864,7 @@ call_one_instrument( return -1; } Py_DECREF(res); - return (res == &DISABLE); + return (res == &_PyInstrumentation_DISABLE); } static const int8_t MOST_SIGNIFICANT_BITS[16] = { @@ -1007,7 +1007,7 @@ _Py_call_instrumentation_2args( return call_instrumentation_vector(tstate, event, frame, instr, 4, args); } -int +_Py_CODEUNIT * _Py_call_instrumentation_jump( PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target) @@ -1015,17 +1015,27 @@ _Py_call_instrumentation_jump( assert(event == PY_MONITORING_EVENT_JUMP || event == PY_MONITORING_EVENT_BRANCH); assert(frame->prev_instr == instr); + /* Event should occur after the jump */ frame->prev_instr = target; PyCodeObject *code = frame->f_code; int to = (int)(target - _PyCode_CODE(code)); PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT)); if (to_obj == NULL) { - return -1; + return NULL; } PyObject *args[4] = { NULL, NULL, NULL, to_obj }; int err = call_instrumentation_vector(tstate, event, frame, instr, 3, args); Py_DECREF(to_obj); - return err; + if (err) { + return NULL; + } + if (frame->prev_instr != target) { + /* The callback has caused a jump (by setting the line number) */ + return frame->prev_instr; + } + /* Reset prev_instr for INSTRUMENTED_LINE */ + frame->prev_instr = instr; + return target; } static void @@ -1081,13 +1091,14 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index) } int -_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) +_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) { frame->prev_instr = instr; PyCodeObject *code = frame->f_code; assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); int i = (int)(instr - _PyCode_CODE(code)); + _PyCoMonitoringData *monitoring = code->_co_monitoring; _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; uint8_t original_opcode = line_data->original_opcode; @@ -1097,6 +1108,18 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; int line = compute_line(code, i, line_delta); + assert(line >= 0); + int prev_index = (int)(prev - _PyCode_CODE(code)); + int prev_line = _Py_Instrumentation_GetLine(code, prev_index); + if (prev_line == line) { + int prev_opcode = _PyCode_CODE(code)[prev_index].op.code; + /* RESUME and INSTRUMENTED_RESUME are needed for the operation of + * instrumentation, so must never be hidden by an INSTRUMENTED_LINE. + */ + if (prev_opcode != RESUME && prev_opcode != INSTRUMENTED_RESUME) { + goto done; + } + } uint8_t tools = code->_co_monitoring->line_tools != NULL ? code->_co_monitoring->line_tools[i] : (interp->monitors.tools[PY_MONITORING_EVENT_LINE] | @@ -1280,29 +1303,91 @@ initialize_lines(PyCodeObject *code) line_data[i].original_opcode = 0; break; default: + /* Set original_opcode to the opcode iff the instruction + * starts a line, and thus should be instrumented. + * This saves having to perform this check every time the + * we turn instrumentation on or off, and serves as a sanity + * check when debugging. + */ if (line != current_line && line >= 0) { line_data[i].original_opcode = opcode; } else { line_data[i].original_opcode = 0; } - if (line >= 0) { - current_line = line; - } + current_line = line; } for (int j = 1; j < length; j++) { line_data[i+j].original_opcode = 0; line_data[i+j].line_delta = NO_LINE; } + i += length; + } + for (int i = code->_co_firsttraceable; i < code_len; ) { + int opcode = _Py_GetBaseOpcode(code, i); + int oparg = 0; + while (opcode == EXTENDED_ARG) { + oparg = (oparg << 8) | _PyCode_CODE(code)[i].op.arg; + i++; + opcode = _Py_GetBaseOpcode(code, i); + } + oparg = (oparg << 8) | _PyCode_CODE(code)[i].op.arg; + i += instruction_length(code, i); + int target = -1; switch (opcode) { - case RETURN_VALUE: - case RAISE_VARARGS: - case RERAISE: - /* Blocks of code after these terminators - * should be treated as different lines */ - current_line = -1; + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: + case JUMP_FORWARD: + { + target = i + oparg; + break; + } + case FOR_ITER: + case SEND: + { + /* Skip over END_FOR/END_SEND */ + target = i + oparg + 1; + break; + } + case JUMP_BACKWARD: + case JUMP_BACKWARD_NO_INTERRUPT: + { + target = i - oparg; + break; + } + default: + continue; + } + assert(target >= 0); + if (line_data[target].line_delta != NO_LINE) { + line_data[target].original_opcode = _Py_GetBaseOpcode(code, target); + } + } + /* Scan exception table */ + unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable); + unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable); + unsigned char *scan = start; + while (scan < end) { + int start_offset, size, handler; + scan = parse_varint(scan, &start_offset); + assert(start_offset >= 0 && start_offset < code_len); + scan = parse_varint(scan, &size); + assert(size >= 0 && start_offset+size <= code_len); + scan = parse_varint(scan, &handler); + assert(handler >= 0 && handler < code_len); + int depth_and_lasti; + scan = parse_varint(scan, &depth_and_lasti); + int original_opcode = _Py_GetBaseOpcode(code, handler); + /* Skip if not the start of a line. + * END_ASYNC_FOR is a bit special as it marks the end of + * an `async for` loop, which should not generate its own + * line event. */ + if (line_data[handler].line_delta != NO_LINE && + original_opcode != END_ASYNC_FOR) { + line_data[handler].original_opcode = original_opcode; } - i += length; } } @@ -2015,7 +2100,7 @@ PyObject *_Py_CreateMonitoringObject(void) if (mod == NULL) { return NULL; } - if (PyObject_SetAttrString(mod, "DISABLE", &DISABLE)) { + if (PyObject_SetAttrString(mod, "DISABLE", &_PyInstrumentation_DISABLE)) { goto error; } if (PyObject_SetAttrString(mod, "MISSING", &_PyInstrumentation_MISSING)) { diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index e509e63a087a52..5143b79b0864d8 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -4,6 +4,7 @@ #include #include "Python.h" +#include "opcode.h" #include "pycore_ceval.h" #include "pycore_object.h" #include "pycore_sysmodule.h" @@ -213,7 +214,6 @@ trace_line( if (line < 0) { Py_RETURN_NONE; } - frame ->f_last_traced_line = line; Py_INCREF(frame); frame->f_lineno = line; int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None); @@ -245,14 +245,12 @@ sys_trace_line_func( return NULL; } assert(args[0] == (PyObject *)frame->f_frame->f_code); - if (frame ->f_last_traced_line == line) { - /* Already traced this line */ - Py_RETURN_NONE; - } return trace_line(tstate, self, frame, line); } - +/* sys.settrace generates line events for all backward + * edges, even if on the same line. + * Handle that case here */ static PyObject * sys_trace_jump_func( _PyLegacyEventHandler *self, PyObject *const *args, @@ -268,61 +266,33 @@ sys_trace_jump_func( assert(from >= 0); int to = _PyLong_AsInt(args[2])/sizeof(_Py_CODEUNIT); assert(to >= 0); - PyFrameObject *frame = PyEval_GetFrame(); - if (frame == NULL) { - PyErr_SetString(PyExc_SystemError, - "Missing frame when calling trace function."); - return NULL; - } - if (!frame->f_trace_lines) { - Py_RETURN_NONE; + if (to > from) { + /* Forward jump */ + return &_PyInstrumentation_DISABLE; } PyCodeObject *code = (PyCodeObject *)args[0]; assert(PyCode_Check(code)); - assert(code == frame->f_frame->f_code); /* We can call _Py_Instrumentation_GetLine because we always set * line events for tracing */ int to_line = _Py_Instrumentation_GetLine(code, to); - /* Backward jump: Always generate event - * Forward jump: Only generate event if jumping to different line. */ - if (to > from && frame->f_last_traced_line == to_line) { - /* Already traced this line */ - Py_RETURN_NONE; + int from_line = _Py_Instrumentation_GetLine(code, from); + if (to_line != from_line) { + /* Will be handled by target INSTRUMENTED_LINE */ + return &_PyInstrumentation_DISABLE; } - return trace_line(tstate, self, frame, to_line); -} - -/* We don't care about the exception here, - * we just treat it as a possible new line - */ -static PyObject * -sys_trace_exception_handled( - _PyLegacyEventHandler *self, PyObject *const *args, - size_t nargsf, PyObject *kwnames -) { - assert(kwnames == NULL); - PyThreadState *tstate = _PyThreadState_GET(); - if (tstate->c_tracefunc == NULL) { - Py_RETURN_NONE; - } - assert(PyVectorcall_NARGS(nargsf) == 3); PyFrameObject *frame = PyEval_GetFrame(); - PyCodeObject *code = (PyCodeObject *)args[0]; - assert(PyCode_Check(code)); + if (frame == NULL) { + PyErr_SetString(PyExc_SystemError, + "Missing frame when calling trace function."); + return NULL; + } assert(code == frame->f_frame->f_code); - assert(PyLong_Check(args[1])); - int offset = _PyLong_AsInt(args[1])/sizeof(_Py_CODEUNIT); - /* We can call _Py_Instrumentation_GetLine because we always set - * line events for tracing */ - int line = _Py_Instrumentation_GetLine(code, offset); - if (frame->f_last_traced_line == line) { - /* Already traced this line */ + if (!frame->f_trace_lines) { Py_RETURN_NONE; } - return trace_line(tstate, self, frame, line); + return trace_line(tstate, self, frame, to_line); } - PyTypeObject _PyLegacyEventHandler_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "sys.legacy_event_handler", @@ -487,7 +457,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) } if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, (vectorcallfunc)sys_trace_jump_func, PyTrace_LINE, - PY_MONITORING_EVENT_JUMP, PY_MONITORING_EVENT_BRANCH)) { + PY_MONITORING_EVENT_JUMP, -1)) { return -1; } if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, @@ -495,11 +465,6 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) PY_MONITORING_EVENT_INSTRUCTION, -1)) { return -1; } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - (vectorcallfunc)sys_trace_exception_handled, PyTrace_LINE, - PY_MONITORING_EVENT_EXCEPTION_HANDLED, -1)) { - return -1; - } } int delta = (func != NULL) - (tstate->c_tracefunc != NULL); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index dc7bb65ff0b867..1047f4c4d417e4 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -369,8 +369,6 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 2; case SWAP: return (oparg-2) + 2; - case INSTRUMENTED_LINE: - return 0; case INSTRUMENTED_INSTRUCTION: return 0; case INSTRUMENTED_JUMP_FORWARD: @@ -763,8 +761,6 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case SWAP: return (oparg-2) + 2; - case INSTRUMENTED_LINE: - return 0; case INSTRUMENTED_INSTRUCTION: return 0; case INSTRUMENTED_JUMP_FORWARD: @@ -981,7 +977,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [COPY] = { true, INSTR_FMT_IB }, [BINARY_OP] = { true, INSTR_FMT_IBC }, [SWAP] = { true, INSTR_FMT_IB }, - [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IB }, diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 453f63ec3f1cfb..8afa92ef25d376 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -301,7 +301,7 @@ Objects/object.c - _Py_NotImplementedStruct - Objects/setobject.c - _dummy_struct - Objects/setobject.c - _PySet_Dummy - Objects/sliceobject.c - _Py_EllipsisObject - -Python/instrumentation.c - DISABLE - +Python/instrumentation.c - _PyInstrumentation_DISABLE - Python/instrumentation.c - _PyInstrumentation_MISSING - diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 2ae8a02aa6d285..19c4cd299f0bbb 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2599,6 +2599,9 @@ class CConverter(metaclass=CConverterAutoRegister): # Every non-abstract subclass should supply a valid value. c_ignored_default = 'NULL' + # If true, wrap with Py_UNUSED. + unused = False + # The C converter *function* to be used, if any. # (If this is not None, format_unit must be 'O&'.) converter = None @@ -2651,9 +2654,22 @@ class CConverter(metaclass=CConverterAutoRegister): signature_name = None # keep in sync with self_converter.__init__! - def __init__(self, name, py_name, function, default=unspecified, *, c_default=None, py_default=None, annotation=unspecified, **kwargs): + def __init__(self, + # Positional args: + name, + py_name, + function, + default=unspecified, + *, # Keyword only args: + c_default=None, + py_default=None, + annotation=unspecified, + unused=False, + **kwargs + ): self.name = ensure_legal_c_identifier(name) self.py_name = py_name + self.unused = unused if default is not unspecified: if self.default_type and not isinstance(default, (self.default_type, Unknown)): @@ -2800,6 +2816,8 @@ def simple_declaration(self, by_reference=False, *, in_parser=False): name = self.parser_name else: name = self.name + if self.unused: + name = f"Py_UNUSED({name})" prototype.append(name) return "".join(prototype)