From webhook-mailer at python.org Tue Aug 1 02:45:55 2023 From: webhook-mailer at python.org (corona10) Date: Tue, 01 Aug 2023 06:45:55 -0000 Subject: [Python-checkins] gh-89013: Improve the performance of methodcaller (lazy version) (gh-107201) Message-ID: https://github.com/python/cpython/commit/f7c9144c2c64d9c28d0524dcd2684c70e8113077 commit: f7c9144c2c64d9c28d0524dcd2684c70e8113077 branch: main author: Pieter Eendebak committer: corona10 date: 2023-08-01T15:45:51+09:00 summary: gh-89013: Improve the performance of methodcaller (lazy version) (gh-107201) files: A Misc/NEWS.d/next/Library/2021-08-16-17-52-26.bpo-44850.r8jx5u.rst M Modules/_operator.c diff --git a/Misc/NEWS.d/next/Library/2021-08-16-17-52-26.bpo-44850.r8jx5u.rst b/Misc/NEWS.d/next/Library/2021-08-16-17-52-26.bpo-44850.r8jx5u.rst new file mode 100644 index 0000000000000..1fe5497f856e9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-16-17-52-26.bpo-44850.r8jx5u.rst @@ -0,0 +1,2 @@ +Improve performance of :func:`operator.methodcaller` using the :pep:`590` ``vectorcall`` convention. +Patch by Anthony Lee and Pieter Eendebak. diff --git a/Modules/_operator.c b/Modules/_operator.c index b59bfe9ac3217..1f6496d381ada 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1549,10 +1549,77 @@ static PyType_Spec attrgetter_type_spec = { typedef struct { PyObject_HEAD PyObject *name; - PyObject *args; + PyObject *xargs; // reference to arguments passed in constructor PyObject *kwds; + PyObject **vectorcall_args; /* Borrowed references */ + PyObject *vectorcall_kwnames; + vectorcallfunc vectorcall; } methodcallerobject; +static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) +{ + PyObject* args = mc->xargs; + PyObject* kwds = mc->kwds; + + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + assert(nargs > 0); + mc->vectorcall_args = PyMem_Calloc( + nargs + (kwds ? PyDict_Size(kwds) : 0), + sizeof(PyObject*)); + if (!mc->vectorcall_args) { + PyErr_NoMemory(); + return -1; + } + /* The first item of vectorcall_args will be filled with obj later */ + if (nargs > 1) { + memcpy(mc->vectorcall_args, PySequence_Fast_ITEMS(args), + nargs * sizeof(PyObject*)); + } + if (kwds) { + const Py_ssize_t nkwds = PyDict_Size(kwds); + + mc->vectorcall_kwnames = PyTuple_New(nkwds); + if (!mc->vectorcall_kwnames) { + return -1; + } + Py_ssize_t i = 0, ppos = 0; + PyObject* key, * value; + while (PyDict_Next(kwds, &ppos, &key, &value)) { + PyTuple_SET_ITEM(mc->vectorcall_kwnames, i, Py_NewRef(key)); + mc->vectorcall_args[nargs + i] = value; // borrowed reference + ++i; + } + } + else { + mc->vectorcall_kwnames = NULL; + } + return 1; +} + + +static PyObject * +methodcaller_vectorcall( + methodcallerobject *mc, PyObject *const *args, size_t nargsf, PyObject* kwnames) +{ + if (!_PyArg_CheckPositional("methodcaller", PyVectorcall_NARGS(nargsf), 1, 1) + || !_PyArg_NoKwnames("methodcaller", kwnames)) { + return NULL; + } + if (mc->vectorcall_args == NULL) { + if (_methodcaller_initialize_vectorcall(mc) < 0) { + return NULL; + } + } + + assert(mc->vectorcall_args != 0); + mc->vectorcall_args[0] = args[0]; + return PyObject_VectorcallMethod( + mc->name, mc->vectorcall_args, + (PyTuple_GET_SIZE(mc->xargs)) | PY_VECTORCALL_ARGUMENTS_OFFSET, + mc->vectorcall_kwnames); +} + + /* AC 3.5: variable number of arguments, not currently support by AC */ static PyObject * methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) @@ -1580,30 +1647,32 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } - name = PyTuple_GET_ITEM(args, 0); Py_INCREF(name); PyUnicode_InternInPlace(&name); mc->name = name; + mc->xargs = Py_XNewRef(args); // allows us to use borrowed references mc->kwds = Py_XNewRef(kwds); + mc->vectorcall_args = 0; - mc->args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); - if (mc->args == NULL) { - Py_DECREF(mc); - return NULL; - } + + mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; PyObject_GC_Track(mc); return (PyObject *)mc; } -static int +static void methodcaller_clear(methodcallerobject *mc) { Py_CLEAR(mc->name); - Py_CLEAR(mc->args); + Py_CLEAR(mc->xargs); Py_CLEAR(mc->kwds); - return 0; + if (mc->vectorcall_args != NULL) { + PyMem_Free(mc->vectorcall_args); + mc->vectorcall_args = 0; + Py_CLEAR(mc->vectorcall_kwnames); + } } static void @@ -1611,7 +1680,7 @@ methodcaller_dealloc(methodcallerobject *mc) { PyTypeObject *tp = Py_TYPE(mc); PyObject_GC_UnTrack(mc); - (void)methodcaller_clear(mc); + methodcaller_clear(mc); tp->tp_free(mc); Py_DECREF(tp); } @@ -1620,7 +1689,7 @@ static int methodcaller_traverse(methodcallerobject *mc, visitproc visit, void *arg) { Py_VISIT(mc->name); - Py_VISIT(mc->args); + Py_VISIT(mc->xargs); Py_VISIT(mc->kwds); Py_VISIT(Py_TYPE(mc)); return 0; @@ -1639,7 +1708,16 @@ methodcaller_call(methodcallerobject *mc, PyObject *args, PyObject *kw) method = PyObject_GetAttr(obj, mc->name); if (method == NULL) return NULL; - result = PyObject_Call(method, mc->args, mc->kwds); + + + PyObject *cargs = PyTuple_GetSlice(mc->xargs, 1, PyTuple_GET_SIZE(mc->xargs)); + if (cargs == NULL) { + Py_DECREF(method); + return NULL; + } + + result = PyObject_Call(method, cargs, mc->kwds); + Py_DECREF(cargs); Py_DECREF(method); return result; } @@ -1657,7 +1735,7 @@ methodcaller_repr(methodcallerobject *mc) } numkwdargs = mc->kwds != NULL ? PyDict_GET_SIZE(mc->kwds) : 0; - numposargs = PyTuple_GET_SIZE(mc->args); + numposargs = PyTuple_GET_SIZE(mc->xargs) - 1; numtotalargs = numposargs + numkwdargs; if (numtotalargs == 0) { @@ -1673,7 +1751,7 @@ methodcaller_repr(methodcallerobject *mc) } for (i = 0; i < numposargs; ++i) { - PyObject *onerepr = PyObject_Repr(PyTuple_GET_ITEM(mc->args, i)); + PyObject *onerepr = PyObject_Repr(PyTuple_GET_ITEM(mc->xargs, i+1)); if (onerepr == NULL) goto done; PyTuple_SET_ITEM(argreprs, i, onerepr); @@ -1723,17 +1801,16 @@ methodcaller_repr(methodcallerobject *mc) static PyObject * methodcaller_reduce(methodcallerobject *mc, PyObject *Py_UNUSED(ignored)) { - PyObject *newargs; if (!mc->kwds || PyDict_GET_SIZE(mc->kwds) == 0) { Py_ssize_t i; - Py_ssize_t callargcount = PyTuple_GET_SIZE(mc->args); - newargs = PyTuple_New(1 + callargcount); + Py_ssize_t newarg_size = PyTuple_GET_SIZE(mc->xargs); + PyObject *newargs = PyTuple_New(newarg_size); if (newargs == NULL) return NULL; PyTuple_SET_ITEM(newargs, 0, Py_NewRef(mc->name)); - for (i = 0; i < callargcount; ++i) { - PyObject *arg = PyTuple_GET_ITEM(mc->args, i); - PyTuple_SET_ITEM(newargs, i + 1, Py_NewRef(arg)); + for (i = 1; i < newarg_size; ++i) { + PyObject *arg = PyTuple_GET_ITEM(mc->xargs, i); + PyTuple_SET_ITEM(newargs, i, Py_NewRef(arg)); } return Py_BuildValue("ON", Py_TYPE(mc), newargs); } @@ -1751,7 +1828,12 @@ methodcaller_reduce(methodcallerobject *mc, PyObject *Py_UNUSED(ignored)) constructor = PyObject_VectorcallDict(partial, newargs, 2, mc->kwds); Py_DECREF(partial); - return Py_BuildValue("NO", constructor, mc->args); + PyObject *args = PyTuple_GetSlice(mc->xargs, 1, PyTuple_GET_SIZE(mc->xargs)); + if (!args) { + Py_DECREF(constructor); + return NULL; + } + return Py_BuildValue("NO", constructor, args); } } @@ -1760,6 +1842,12 @@ static PyMethodDef methodcaller_methods[] = { reduce_doc}, {NULL} }; + +static PyMemberDef methodcaller_members[] = { + {"__vectorcalloffset__", Py_T_PYSSIZET, offsetof(methodcallerobject, vectorcall), Py_READONLY}, + {NULL} +}; + PyDoc_STRVAR(methodcaller_doc, "methodcaller(name, /, *args, **kwargs)\n--\n\n\ Return a callable object that calls the given method on its operand.\n\ @@ -1774,6 +1862,7 @@ static PyType_Slot methodcaller_type_slots[] = { {Py_tp_traverse, methodcaller_traverse}, {Py_tp_clear, methodcaller_clear}, {Py_tp_methods, methodcaller_methods}, + {Py_tp_members, methodcaller_members}, {Py_tp_new, methodcaller_new}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_repr, methodcaller_repr}, @@ -1785,7 +1874,7 @@ static PyType_Spec methodcaller_type_spec = { .basicsize = sizeof(methodcallerobject), .itemsize = 0, .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_IMMUTABLETYPE), .slots = methodcaller_type_slots, }; From webhook-mailer at python.org Tue Aug 1 03:47:28 2023 From: webhook-mailer at python.org (hugovk) Date: Tue, 01 Aug 2023 07:47:28 -0000 Subject: [Python-checkins] Remove newlines from bug template (#107525) Message-ID: https://github.com/python/cpython/commit/052a0d1106fa3ee0c955a3b7ba48e82c49424e20 commit: 052a0d1106fa3ee0c955a3b7ba48e82c49424e20 branch: main author: Hugo van Kemenade committer: hugovk date: 2023-08-01T01:47:24-06:00 summary: Remove newlines from bug template (#107525) files: M .github/ISSUE_TEMPLATE/bug.md diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md index 737eb6dfebb77..47037cd319e7e 100644 --- a/.github/ISSUE_TEMPLATE/bug.md +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -19,10 +19,8 @@ labels: "type-bug" -- [ ] I am confident this is a bug in CPython, - not a bug in a third-party project -- [ ] I have searched the CPython issue tracker, - and am confident this bug has not been reported before +- [ ] I am confident this is a bug in CPython, not a bug in a third-party project +- [ ] I have searched the CPython issue tracker, and am confident this bug has not been reported before ## A clear and concise description of the bug From webhook-mailer at python.org Tue Aug 1 05:32:22 2023 From: webhook-mailer at python.org (markshannon) Date: Tue, 01 Aug 2023 09:32:22 -0000 Subject: [Python-checkins] gh-106092: Fix use-after-free crash in frame_dealloc (#106875) Message-ID: https://github.com/python/cpython/commit/557b05c7a5334de5da3dc94c108c0121f10b9191 commit: 557b05c7a5334de5da3dc94c108c0121f10b9191 branch: main author: Anders Kaseorg committer: markshannon date: 2023-08-01T10:32:18+01:00 summary: gh-106092: Fix use-after-free crash in frame_dealloc (#106875) It was possible for the trashcan to delay the deallocation of a PyFrameObject until after its corresponding _PyInterpreterFrame has already been freed. So frame_dealloc needs to avoid dereferencing the f_frame pointer unless it first checks that the pointer still points to the interpreter frame within the frame object. Signed-off-by: Anders Kaseorg files: A Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst M Objects/frameobject.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst new file mode 100644 index 0000000000000..7fb5b45c763e4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst @@ -0,0 +1,2 @@ +Fix a segmentation fault caused by a use-after-free bug in ``frame_dealloc`` +when the trashcan delays the deallocation of a ``PyFrameObject``. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index cc9ac4b42f479..17571535048e2 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -879,9 +879,6 @@ frame_dealloc(PyFrameObject *f) /* It is the responsibility of the owning generator/coroutine * to have cleared the generator pointer */ - assert(f->f_frame->owner != FRAME_OWNED_BY_GENERATOR || - _PyFrame_GetGenerator(f->f_frame)->gi_frame_state == FRAME_CLEARED); - if (_PyObject_GC_IS_TRACKED(f)) { _PyObject_GC_UNTRACK(f); } @@ -889,10 +886,14 @@ frame_dealloc(PyFrameObject *f) Py_TRASHCAN_BEGIN(f, frame_dealloc); PyObject *co = NULL; + /* GH-106092: If f->f_frame was on the stack and we reached the maximum + * nesting depth for deallocations, the trashcan may have delayed this + * deallocation until after f->f_frame is freed. Avoid dereferencing + * f->f_frame unless we know it still points to valid memory. */ + _PyInterpreterFrame *frame = (_PyInterpreterFrame *)f->_f_frame_data; + /* Kill all local variables including specials, if we own them */ - if (f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { - assert(f->f_frame == (_PyInterpreterFrame *)f->_f_frame_data); - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)f->_f_frame_data; + if (f->f_frame == frame && frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { /* Don't clear code object until the end */ co = frame->f_executable; frame->f_executable = NULL; From webhook-mailer at python.org Tue Aug 1 06:21:11 2023 From: webhook-mailer at python.org (pablogsal) Date: Tue, 01 Aug 2023 10:21:11 -0000 Subject: [Python-checkins] [3.11] gh-106092: Fix use-after-free crash in frame_dealloc (GH-106875) (#107533) Message-ID: https://github.com/python/cpython/commit/46cae02085311481dc8b1ea9a5110969d9325bc7 commit: 46cae02085311481dc8b1ea9a5110969d9325bc7 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: pablogsal date: 2023-08-01T11:21:08+01:00 summary: [3.11] gh-106092: Fix use-after-free crash in frame_dealloc (GH-106875) (#107533) files: A Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst M Objects/frameobject.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst new file mode 100644 index 0000000000000..7fb5b45c763e4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst @@ -0,0 +1,2 @@ +Fix a segmentation fault caused by a use-after-free bug in ``frame_dealloc`` +when the trashcan delays the deallocation of a ``PyFrameObject``. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 1e6bdcfb592df..425749d14cf01 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -851,9 +851,6 @@ frame_dealloc(PyFrameObject *f) /* It is the responsibility of the owning generator/coroutine * to have cleared the generator pointer */ - assert(f->f_frame->owner != FRAME_OWNED_BY_GENERATOR || - _PyFrame_GetGenerator(f->f_frame)->gi_frame_state == FRAME_CLEARED); - if (_PyObject_GC_IS_TRACKED(f)) { _PyObject_GC_UNTRACK(f); } @@ -861,10 +858,14 @@ frame_dealloc(PyFrameObject *f) Py_TRASHCAN_BEGIN(f, frame_dealloc); PyCodeObject *co = NULL; + /* GH-106092: If f->f_frame was on the stack and we reached the maximum + * nesting depth for deallocations, the trashcan may have delayed this + * deallocation until after f->f_frame is freed. Avoid dereferencing + * f->f_frame unless we know it still points to valid memory. */ + _PyInterpreterFrame *frame = (_PyInterpreterFrame *)f->_f_frame_data; + /* Kill all local variables including specials, if we own them */ - if (f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { - assert(f->f_frame == (_PyInterpreterFrame *)f->_f_frame_data); - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)f->_f_frame_data; + if (f->f_frame == frame && frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { /* Don't clear code object until the end */ co = frame->f_code; frame->f_code = NULL; From webhook-mailer at python.org Tue Aug 1 06:43:00 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 01 Aug 2023 10:43:00 -0000 Subject: [Python-checkins] [3.12] gh-106092: Fix use-after-free crash in frame_dealloc (GH-106875) (#107532) Message-ID: https://github.com/python/cpython/commit/b68faa3fa3214cda35d5a34639a7a62b6a98bc6c commit: b68faa3fa3214cda35d5a34639a7a62b6a98bc6c branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-01T12:42:55+02:00 summary: [3.12] gh-106092: Fix use-after-free crash in frame_dealloc (GH-106875) (#107532) gh-106092: Fix use-after-free crash in frame_dealloc (GH-106875) It was possible for the trashcan to delay the deallocation of a PyFrameObject until after its corresponding _PyInterpreterFrame has already been freed. So frame_dealloc needs to avoid dereferencing the f_frame pointer unless it first checks that the pointer still points to the interpreter frame within the frame object. (cherry picked from commit 557b05c7a5334de5da3dc94c108c0121f10b9191) Signed-off-by: Anders Kaseorg Co-authored-by: Anders Kaseorg files: A Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst M Objects/frameobject.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst new file mode 100644 index 0000000000000..7fb5b45c763e4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst @@ -0,0 +1,2 @@ +Fix a segmentation fault caused by a use-after-free bug in ``frame_dealloc`` +when the trashcan delays the deallocation of a ``PyFrameObject``. diff --git a/Objects/frameobject.c b/Objects/frameobject.c index e62cfab80b2e3..ba365750bd4d2 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -878,9 +878,6 @@ frame_dealloc(PyFrameObject *f) /* It is the responsibility of the owning generator/coroutine * to have cleared the generator pointer */ - assert(f->f_frame->owner != FRAME_OWNED_BY_GENERATOR || - _PyFrame_GetGenerator(f->f_frame)->gi_frame_state == FRAME_CLEARED); - if (_PyObject_GC_IS_TRACKED(f)) { _PyObject_GC_UNTRACK(f); } @@ -888,10 +885,14 @@ frame_dealloc(PyFrameObject *f) Py_TRASHCAN_BEGIN(f, frame_dealloc); PyCodeObject *co = NULL; + /* GH-106092: If f->f_frame was on the stack and we reached the maximum + * nesting depth for deallocations, the trashcan may have delayed this + * deallocation until after f->f_frame is freed. Avoid dereferencing + * f->f_frame unless we know it still points to valid memory. */ + _PyInterpreterFrame *frame = (_PyInterpreterFrame *)f->_f_frame_data; + /* Kill all local variables including specials, if we own them */ - if (f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { - assert(f->f_frame == (_PyInterpreterFrame *)f->_f_frame_data); - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)f->_f_frame_data; + if (f->f_frame == frame && frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { /* Don't clear code object until the end */ co = frame->f_code; frame->f_code = NULL; From webhook-mailer at python.org Tue Aug 1 12:24:28 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 01 Aug 2023 16:24:28 -0000 Subject: [Python-checkins] gh-107467: Restructure Argument Clinic command-line interface (#107469) Message-ID: https://github.com/python/cpython/commit/49f238e78c36532bcbca7f9cd172703eb4df319b commit: 49f238e78c36532bcbca7f9cd172703eb4df319b branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-01T18:24:23+02:00 summary: gh-107467: Restructure Argument Clinic command-line interface (#107469) - Use ArgumentParser.error() to handle CLI errors - Put the entire CLI in main() - Rework ClinicExternalTest to call main() instead of using subprocesses Co-authored-by: AlexWaygood files: A Misc/NEWS.d/next/Tools-Demos/2023-07-30-23-32-16.gh-issue-107467.5O9p3G.rst M Lib/test/test_clinic.py M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 2f94f0168c916..5e74b4fb77022 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -4,14 +4,12 @@ from test import support, test_tools from test.support import os_helper -from test.support import SHORT_TIMEOUT, requires_subprocess from test.support.os_helper import TESTFN, unlink from textwrap import dedent from unittest import TestCase import collections import inspect import os.path -import subprocess import sys import unittest @@ -1411,30 +1409,26 @@ def test_scaffolding(self): class ClinicExternalTest(TestCase): maxDiff = None - clinic_py = os.path.join(test_tools.toolsdir, "clinic", "clinic.py") - - def _do_test(self, *args, expect_success=True): - with subprocess.Popen( - [sys.executable, "-Xutf8", self.clinic_py, *args], - encoding="utf-8", - bufsize=0, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) as proc: - proc.wait() - if expect_success and proc.returncode: - self.fail("".join([*proc.stdout, *proc.stderr])) - stdout = proc.stdout.read() - stderr = proc.stderr.read() - # Clinic never writes to stderr. - self.assertEqual(stderr, "") - return stdout + + def run_clinic(self, *args): + with ( + support.captured_stdout() as out, + support.captured_stderr() as err, + self.assertRaises(SystemExit) as cm + ): + clinic.main(args) + return out.getvalue(), err.getvalue(), cm.exception.code def expect_success(self, *args): - return self._do_test(*args) + out, err, code = self.run_clinic(*args) + self.assertEqual(code, 0, f"Unexpected failure: {args=}") + self.assertEqual(err, "") + return out def expect_failure(self, *args): - return self._do_test(*args, expect_success=False) + out, err, code = self.run_clinic(*args) + self.assertNotEqual(code, 0, f"Unexpected success: {args=}") + return out, err def test_external(self): CLINIC_TEST = 'clinic.test.c' @@ -1498,8 +1492,9 @@ def test_cli_force(self): # First, run the CLI without -f and expect failure. # Note, we cannot check the entire fail msg, because the path to # the tmp file will change for every run. - out = self.expect_failure(fn) - self.assertTrue(out.endswith(fail_msg)) + out, _ = self.expect_failure(fn) + self.assertTrue(out.endswith(fail_msg), + f"{out!r} does not end with {fail_msg!r}") # Then, force regeneration; success expected. out = self.expect_success("-f", fn) self.assertEqual(out, "") @@ -1641,33 +1636,30 @@ def test_cli_converters(self): ) def test_cli_fail_converters_and_filename(self): - out = self.expect_failure("--converters", "test.c") - msg = ( - "Usage error: can't specify --converters " - "and a filename at the same time" - ) - self.assertIn(msg, out) + _, err = self.expect_failure("--converters", "test.c") + msg = "can't specify --converters and a filename at the same time" + self.assertIn(msg, err) def test_cli_fail_no_filename(self): - out = self.expect_failure() - self.assertIn("usage: clinic.py", out) + _, err = self.expect_failure() + self.assertIn("no input files", err) def test_cli_fail_output_and_multiple_files(self): - out = self.expect_failure("-o", "out.c", "input.c", "moreinput.c") - msg = "Usage error: can't use -o with multiple filenames" - self.assertIn(msg, out) + _, err = self.expect_failure("-o", "out.c", "input.c", "moreinput.c") + msg = "error: can't use -o with multiple filenames" + self.assertIn(msg, err) def test_cli_fail_filename_or_output_and_make(self): + msg = "can't use -o or filenames with --make" for opts in ("-o", "out.c"), ("filename.c",): with self.subTest(opts=opts): - out = self.expect_failure("--make", *opts) - msg = "Usage error: can't use -o or filenames with --make" - self.assertIn(msg, out) + _, err = self.expect_failure("--make", *opts) + self.assertIn(msg, err) def test_cli_fail_make_without_srcdir(self): - out = self.expect_failure("--make", "--srcdir", "") - msg = "Usage error: --srcdir must not be empty with --make" - self.assertIn(msg, out) + _, err = self.expect_failure("--make", "--srcdir", "") + msg = "error: --srcdir must not be empty with --make" + self.assertIn(msg, err) try: diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-07-30-23-32-16.gh-issue-107467.5O9p3G.rst b/Misc/NEWS.d/next/Tools-Demos/2023-07-30-23-32-16.gh-issue-107467.5O9p3G.rst new file mode 100644 index 0000000000000..2996837371be0 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-07-30-23-32-16.gh-issue-107467.5O9p3G.rst @@ -0,0 +1,2 @@ +The Argument Clinic command-line tool now prints to stderr instead of stdout +on failure. diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index defe703b825b1..3501c16a567cc 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -7,6 +7,7 @@ from __future__ import annotations import abc +import argparse import ast import builtins as bltns import collections @@ -5620,10 +5621,9 @@ def state_terminal(self, line: str | None) -> None: clinic = None -def main(argv: list[str]) -> None: - import sys - import argparse +def create_cli() -> argparse.ArgumentParser: cmdline = argparse.ArgumentParser( + prog="clinic.py", description="""Preprocessor for CPython C files. The purpose of the Argument Clinic is automating all the boilerplate involved @@ -5646,14 +5646,15 @@ def main(argv: list[str]) -> None: help="the directory tree to walk in --make mode") cmdline.add_argument("filename", metavar="FILE", type=str, nargs="*", help="the list of files to process") - ns = cmdline.parse_args(argv) + return cmdline + +def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: if ns.converters: if ns.filename: - print("Usage error: can't specify --converters and a filename at the same time.") - print() - cmdline.print_usage() - sys.exit(-1) + parser.error( + "can't specify --converters and a filename at the same time" + ) converters: list[tuple[str, str]] = [] return_converters: list[tuple[str, str]] = [] ignored = set(""" @@ -5707,19 +5708,13 @@ def main(argv: list[str]) -> None: print() print("All converters also accept (c_default=None, py_default=None, annotation=None).") print("All return converters also accept (py_default=None).") - sys.exit(0) + return if ns.make: if ns.output or ns.filename: - print("Usage error: can't use -o or filenames with --make.") - print() - cmdline.print_usage() - sys.exit(-1) + parser.error("can't use -o or filenames with --make") if not ns.srcdir: - print("Usage error: --srcdir must not be empty with --make.") - print() - cmdline.print_usage() - sys.exit(-1) + parser.error("--srcdir must not be empty with --make") for root, dirs, files in os.walk(ns.srcdir): for rcs_dir in ('.svn', '.git', '.hg', 'build', 'externals'): if rcs_dir in dirs: @@ -5735,14 +5730,10 @@ def main(argv: list[str]) -> None: return if not ns.filename: - cmdline.print_usage() - sys.exit(-1) + parser.error("no input files") if ns.output and len(ns.filename) > 1: - print("Usage error: can't use -o with multiple filenames.") - print() - cmdline.print_usage() - sys.exit(-1) + parser.error("can't use -o with multiple filenames") for filename in ns.filename: if ns.verbose: @@ -5750,6 +5741,12 @@ def main(argv: list[str]) -> None: parse_file(filename, output=ns.output, verify=not ns.force) -if __name__ == "__main__": - main(sys.argv[1:]) +def main(argv: list[str] | None = None) -> NoReturn: + parser = create_cli() + args = parser.parse_args(argv) + run_clinic(parser, args) sys.exit(0) + + +if __name__ == "__main__": + main() From webhook-mailer at python.org Tue Aug 1 13:05:04 2023 From: webhook-mailer at python.org (iritkatriel) Date: Tue, 01 Aug 2023 17:05:04 -0000 Subject: [Python-checkins] gh-105481: the ENABLE_SPECIALIZATION flag does not need to be generated by the build script, or exposed in opcode.py (#107534) Message-ID: https://github.com/python/cpython/commit/6ef8f8ca88b925685c6af83a9f0a899e565e1e33 commit: 6ef8f8ca88b925685c6af83a9f0a899e565e1e33 branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-01T17:05:00Z summary: gh-105481: the ENABLE_SPECIALIZATION flag does not need to be generated by the build script, or exposed in opcode.py (#107534) files: A Misc/NEWS.d/next/Library/2023-08-01-15-17-20.gh-issue-105481.vMbmj_.rst M Doc/whatsnew/3.13.rst M Include/internal/pycore_code.h M Include/opcode.h M Lib/opcode.py M Lib/test/support/__init__.py M Modules/_opcode.c M Tools/build/generate_opcode_h.py diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 8fb4e6cfdf14c..22c1e03470f5d 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -117,6 +117,13 @@ and only logged in :ref:`Python Development Mode ` or on :ref:`Python built on debug mode `. (Contributed by Victor Stinner in :gh:`62948`.) +opcode +------ + +* Move ``opcode.ENABLE_SPECIALIZATION`` to ``_opcode.ENABLE_SPECIALIZATION``. + This field was added in 3.12, it was never documented and is not intended for + external usage. (Contributed by Irit Katriel in :gh:`105481`.) + pathlib ------- diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index bcdf8645c8557..ee1b85187cbab 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -229,6 +229,8 @@ extern void _PyLineTable_InitAddressRange( extern int _PyLineTable_NextAddressRange(PyCodeAddressRange *range); extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); +#define ENABLE_SPECIALIZATION 1 + /* Specialization functions */ extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, diff --git a/Include/opcode.h b/Include/opcode.h index 697520937d905..eb4bb85e5b667 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -256,8 +256,6 @@ extern "C" { #define NB_INPLACE_TRUE_DIVIDE 24 #define NB_INPLACE_XOR 25 -/* Defined in Lib/opcode.py */ -#define ENABLE_SPECIALIZATION 1 #ifdef __cplusplus } diff --git a/Lib/opcode.py b/Lib/opcode.py index bed922399821a..51432ab1fab3f 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -19,9 +19,6 @@ cmp_op = ('<', '<=', '==', '!=', '>', '>=') - -ENABLE_SPECIALIZATION = True - def is_pseudo(op): return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 3b332f49951f0..2a59911e54d6b 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -6,7 +6,7 @@ import contextlib import functools import getpass -import opcode +import _opcode import os import re import stat @@ -1092,7 +1092,7 @@ def requires_limited_api(test): def requires_specialization(test): return unittest.skipUnless( - opcode.ENABLE_SPECIALIZATION, "requires specialization")(test) + _opcode.ENABLE_SPECIALIZATION, "requires specialization")(test) def _filter_suite(suite, pred): """Recursively filter test cases in a suite based on a predicate.""" diff --git a/Misc/NEWS.d/next/Library/2023-08-01-15-17-20.gh-issue-105481.vMbmj_.rst b/Misc/NEWS.d/next/Library/2023-08-01-15-17-20.gh-issue-105481.vMbmj_.rst new file mode 100644 index 0000000000000..153c18a6f0095 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-01-15-17-20.gh-issue-105481.vMbmj_.rst @@ -0,0 +1 @@ +:data:`opcode.ENABLE_SPECIALIZATION` (which was added in 3.12 but never documented or intended for external usage) is moved to :data:`_opcode.ENABLE_SPECIALIZATION` where tests can access it. diff --git a/Modules/_opcode.c b/Modules/_opcode.c index b8d95a048ec2c..ad0fa736f8276 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -292,7 +292,16 @@ opcode_functions[] = { {NULL, NULL, 0, NULL} }; +int +_opcode_exec(PyObject *m) { + if (PyModule_AddIntMacro(m, ENABLE_SPECIALIZATION) < 0) { + return -1; + } + return 0; +} + static PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, _opcode_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {0, NULL} }; diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 19e5eab822a23..2259dad77869f 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -73,7 +73,6 @@ def main(opcode_py, opname = opcode['opname'] is_pseudo = opcode['is_pseudo'] - ENABLE_SPECIALIZATION = opcode["ENABLE_SPECIALIZATION"] MIN_PSEUDO_OPCODE = opcode["MIN_PSEUDO_OPCODE"] MAX_PSEUDO_OPCODE = opcode["MAX_PSEUDO_OPCODE"] MIN_INSTRUMENTED_OPCODE = opcode["MIN_INSTRUMENTED_OPCODE"] @@ -141,10 +140,6 @@ def main(opcode_py, for i, (op, _) in enumerate(opcode["_nb_ops"]): fobj.write(DEFINE.format(op, i)) - fobj.write("\n") - fobj.write("/* Defined in Lib/opcode.py */\n") - fobj.write(f"#define ENABLE_SPECIALIZATION {int(ENABLE_SPECIALIZATION)}") - iobj.write("\n") iobj.write(f"\nextern const char *const _PyOpcode_OpName[{NUM_OPCODES}];\n") iobj.write("\n#ifdef NEED_OPCODE_TABLES\n") From webhook-mailer at python.org Tue Aug 1 16:05:52 2023 From: webhook-mailer at python.org (iritkatriel) Date: Tue, 01 Aug 2023 20:05:52 -0000 Subject: [Python-checkins] gh-105481: combine regen-opcode-targets with regen-opcode to avoid calculating the specialized opcodes in two places (#107540) Message-ID: https://github.com/python/cpython/commit/2bd04d423404e5ea862c858f62af76feade4d831 commit: 2bd04d423404e5ea862c858f62af76feade4d831 branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-01T21:05:48+01:00 summary: gh-105481: combine regen-opcode-targets with regen-opcode to avoid calculating the specialized opcodes in two places (#107540) files: A Misc/NEWS.d/next/Build/2023-08-01-17-12-53.gh-issue-105481.42nsDE.rst D Python/makeopcodetargets.py M Makefile.pre.in M PCbuild/regen.targets M Tools/build/generate_opcode_h.py diff --git a/Makefile.pre.in b/Makefile.pre.in index 3725feaca66ce..12409774746a3 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1316,7 +1316,7 @@ regen-limited-abi: all # Regenerate all generated files .PHONY: regen-all -regen-all: regen-cases regen-opcode regen-opcode-targets regen-typeslots \ +regen-all: regen-cases regen-opcode regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen clinic \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ regen-test-levenshtein regen-global-objects @@ -1428,8 +1428,10 @@ regen-opcode: $(srcdir)/Lib/opcode.py \ $(srcdir)/Lib/_opcode_metadata.py \ $(srcdir)/Include/opcode.h.new \ + $(srcdir)/Python/opcode_targets.h.new \ $(srcdir)/Include/internal/pycore_opcode.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new + $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new .PHONY: regen-token @@ -1531,13 +1533,6 @@ Objects/unicodeobject.o: $(srcdir)/Objects/unicodeobject.c $(UNICODE_DEPS) Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h -.PHONY: regen-opcode-targets -regen-opcode-targets: - # Regenerate Python/opcode_targets.h from Lib/opcode.py - # using Python/makeopcodetargets.py - $(PYTHON_FOR_REGEN) $(srcdir)/Python/makeopcodetargets.py \ - $(srcdir)/Python/opcode_targets.h.new - $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new .PHONY: regen-cases regen-cases: diff --git a/Misc/NEWS.d/next/Build/2023-08-01-17-12-53.gh-issue-105481.42nsDE.rst b/Misc/NEWS.d/next/Build/2023-08-01-17-12-53.gh-issue-105481.42nsDE.rst new file mode 100644 index 0000000000000..1e61c37b60946 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-08-01-17-12-53.gh-issue-105481.42nsDE.rst @@ -0,0 +1 @@ +Remove the make target ``regen-opcode-targets``, merge its work into ``regen-opcode`` which repeats most of the calculation. This simplifies the code for the build and reduces code duplication. diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 2dd786e5e82e3..99cfff5acc0ba 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -59,9 +59,7 @@ Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)" DependsOnTargets="FindPythonForBuild"> - - diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py deleted file mode 100755 index 5843079b72993..0000000000000 --- a/Python/makeopcodetargets.py +++ /dev/null @@ -1,56 +0,0 @@ -#! /usr/bin/env python -"""Generate C code for the jump table of the threaded code interpreter -(for compilers supporting computed gotos or "labels-as-values", such as gcc). -""" - -import os -import sys - - -# 2023-04-27(warsaw): Pre-Python 3.12, this would catch ImportErrors and try to -# import imp, and then use imp.load_module(). The imp module was removed in -# Python 3.12 (and long deprecated before that), and it's unclear under what -# conditions this import will now fail, so the fallback was simply removed. -from importlib.machinery import SourceFileLoader - -def find_module(modname): - """Finds and returns a module in the local dist/checkout. - """ - modpath = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "Lib", modname + ".py") - return SourceFileLoader(modname, modpath).load_module() - - -def write_contents(f): - """Write C code contents to the target file object. - """ - opcode = find_module('opcode') - _opcode_metadata = find_module('_opcode_metadata') - targets = ['_unknown_opcode'] * 256 - for opname, op in opcode.opmap.items(): - if not opcode.is_pseudo(op): - targets[op] = "TARGET_%s" % opname - next_op = 1 - for opname in _opcode_metadata._specialized_instructions: - while targets[next_op] != '_unknown_opcode': - next_op += 1 - targets[next_op] = "TARGET_%s" % opname - f.write("static void *opcode_targets[256] = {\n") - f.write(",\n".join([" &&%s" % s for s in targets])) - f.write("\n};\n") - - -def main(): - if len(sys.argv) >= 3: - sys.exit("Too many arguments") - if len(sys.argv) == 2: - target = sys.argv[1] - else: - target = "Python/opcode_targets.h" - with open(target, "w") as f: - write_contents(f) - print("Jump table written into %s" % target) - - -if __name__ == "__main__": - main() diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 2259dad77869f..16b028dc1205a 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -64,6 +64,7 @@ def get_python_module_dict(filename): def main(opcode_py, _opcode_metadata_py='Lib/_opcode_metadata.py', outfile='Include/opcode.h', + opcode_targets_h='Python/opcode_targets.h', internaloutfile='Include/internal/pycore_opcode.h'): _opcode_metadata = get_python_module_dict(_opcode_metadata_py) @@ -161,9 +162,18 @@ def main(opcode_py, fobj.write(footer) iobj.write(internal_footer) + with open(opcode_targets_h, "w") as f: + targets = ["_unknown_opcode"] * 256 + for op, name in enumerate(opname_including_specialized): + if op < 256 and not name.startswith("<"): + targets[op] = f"TARGET_{name}" + + f.write("static void *opcode_targets[256] = {\n") + f.write(",\n".join([f" &&{s}" for s in targets])) + f.write("\n};\n") print(f"{outfile} regenerated from {opcode_py}") if __name__ == '__main__': - main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) + main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5]) From webhook-mailer at python.org Tue Aug 1 16:11:10 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Tue, 01 Aug 2023 20:11:10 -0000 Subject: [Python-checkins] gh-104683: Argument clinic: remove the `LandMine` class (#107541) Message-ID: https://github.com/python/cpython/commit/030f6b1e84274616c22666f27c5695867ab85833 commit: 030f6b1e84274616c22666f27c5695867ab85833 branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-01T21:10:54+01:00 summary: gh-104683: Argument clinic: remove the `LandMine` class (#107541) files: M Lib/test/test_clinic.py M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 5e74b4fb77022..6100444e6da79 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -331,8 +331,7 @@ def converter_init(self): [clinic start generated code]*/ """) msg = ( - "Stepped on a land mine, trying to access attribute 'noaccess':\n" - "Don't access members of self.function inside converter_init!" + "accessing self.function inside converter_init is disallowed!" ) self.assertIn(msg, out) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 3501c16a567cc..ba0132fc35903 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -38,6 +38,7 @@ ) from types import FunctionType, NoneType from typing import ( + TYPE_CHECKING, Any, Final, Literal, @@ -2638,18 +2639,6 @@ def get_displayname(self, i: int) -> str: return f'"argument {i}"' - at dc.dataclass -class LandMine: - # try to access any - __message__: str - - def __getattribute__(self, name: str) -> Any: - if name in ('__repr__', '__message__'): - return super().__getattribute__(name) - # raise RuntimeError(repr(name)) - fail("Stepped on a land mine, trying to access attribute " + repr(name) + ":\n" + self.__message__) - - CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"]) def add_c_converter( @@ -2844,16 +2833,28 @@ def __init__(self, if annotation is not unspecified: fail("The 'annotation' parameter is not currently permitted.") - # this is deliberate, to prevent you from caching information - # about the function in the init. - # (that breaks if we get cloned.) - # so after this change we will noisily fail. - self.function: Function | LandMine = LandMine( - "Don't access members of self.function inside converter_init!" - ) + # Make sure not to set self.function until after converter_init() has been called. + # This prevents you from caching information + # about the function in converter_init(). + # (That breaks if we get cloned.) self.converter_init(**kwargs) self.function = function + # Add a custom __getattr__ method to improve the error message + # if somebody tries to access self.function in converter_init(). + # + # mypy will assume arbitrary access is okay for a class with a __getattr__ method, + # and that's not what we want, + # so put it inside an `if not TYPE_CHECKING` block + if not TYPE_CHECKING: + def __getattr__(self, attr): + if attr == "function": + fail( + f"{self.__class__.__name__!r} object has no attribute 'function'.\n" + f"Note: accessing self.function inside converter_init is disallowed!" + ) + return super().__getattr__(attr) + def converter_init(self) -> None: pass @@ -4005,7 +4006,6 @@ def converter_init(self, *, type: str | None = None) -> None: def pre_render(self) -> None: f = self.function - assert isinstance(f, Function) default_type, default_name = correct_name_for_self(f) self.signature_name = default_name self.type = self.specified_type or self.type or default_type @@ -4056,7 +4056,6 @@ def pre_render(self) -> None: @property def parser_type(self) -> str: assert self.type is not None - assert isinstance(self.function, Function) return required_type_for_self_for_parser(self.function) or self.type def render(self, parameter: Parameter, data: CRenderData) -> None: From webhook-mailer at python.org Tue Aug 1 16:20:30 2023 From: webhook-mailer at python.org (hauntsaninja) Date: Tue, 01 Aug 2023 20:20:30 -0000 Subject: [Python-checkins] Clarify `Self` interaction with subclasses (#107511) Message-ID: https://github.com/python/cpython/commit/c8872f4285d3b61c252e3384bec6d30618b7d698 commit: c8872f4285d3b61c252e3384bec6d30618b7d698 branch: main author: Alexandru M?r??teanu committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2023-08-01T13:20:25-07:00 summary: Clarify `Self` interaction with subclasses (#107511) files: M Doc/library/typing.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index f96a805cef7b2..fad945ffc8210 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -963,13 +963,17 @@ using ``[]``. For example:: - from typing import Self + from typing import Self, reveal_type class Foo: def return_self(self) -> Self: ... return self + class SubclassOfFoo(Foo): pass + + reveal_type(Foo().return_self()) # Revealed type is "Foo" + reveal_type(SubclassOfFoo().return_self()) # Revealed type is "SubclassOfFoo" This annotation is semantically equivalent to the following, albeit in a more succinct fashion:: @@ -983,15 +987,11 @@ using ``[]``. ... return self - In general if something currently follows the pattern of:: - - class Foo: - def return_self(self) -> "Foo": - ... - return self - - You should use :data:`Self` as calls to ``SubclassOfFoo.return_self`` would have - ``Foo`` as the return type and not ``SubclassOfFoo``. + In general, if something returns ``self``, as in the above examples, you + should use ``Self`` as the return annotation. If ``Foo.return_self`` was + annotated as returning ``"Foo"``, then the type checker would infer the + object returned from ``SubclassOfFoo.return_self`` as being of type ``Foo`` + rather than ``SubclassOfFoo``. Other common use cases include: @@ -999,6 +999,17 @@ using ``[]``. of the ``cls`` parameter. - Annotating an :meth:`~object.__enter__` method which returns self. + You should not use ``Self`` as the return annotation if the method is not + guaranteed to return an instance of a subclass when the class is + subclassed:: + + class Eggs: + # Self would be an incorrect return annotation here, + # as the object returned is always an instance of Eggs, + # even in subclasses + def returns_eggs(self) -> "Eggs": + return Eggs() + See :pep:`673` for more details. .. versionadded:: 3.11 From webhook-mailer at python.org Tue Aug 1 16:31:26 2023 From: webhook-mailer at python.org (hauntsaninja) Date: Tue, 01 Aug 2023 20:31:26 -0000 Subject: [Python-checkins] [3.11] Clarify `Self` interaction with subclasses (GH-107511) (#107549) Message-ID: https://github.com/python/cpython/commit/623b0d9c598585dd8917b794ae99ce89c6711c5e commit: 623b0d9c598585dd8917b794ae99ce89c6711c5e branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2023-08-01T20:31:22Z summary: [3.11] Clarify `Self` interaction with subclasses (GH-107511) (#107549) Co-authored-by: Alexandru M?r??teanu files: M Doc/library/typing.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index b6bb50b65db62..0419f6799c664 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -905,13 +905,17 @@ using ``[]``. For example:: - from typing import Self + from typing import Self, reveal_type class Foo: def return_self(self) -> Self: ... return self + class SubclassOfFoo(Foo): pass + + reveal_type(Foo().return_self()) # Revealed type is "Foo" + reveal_type(SubclassOfFoo().return_self()) # Revealed type is "SubclassOfFoo" This annotation is semantically equivalent to the following, albeit in a more succinct fashion:: @@ -925,15 +929,11 @@ using ``[]``. ... return self - In general if something currently follows the pattern of:: - - class Foo: - def return_self(self) -> "Foo": - ... - return self - - You should use :data:`Self` as calls to ``SubclassOfFoo.return_self`` would have - ``Foo`` as the return type and not ``SubclassOfFoo``. + In general, if something returns ``self``, as in the above examples, you + should use ``Self`` as the return annotation. If ``Foo.return_self`` was + annotated as returning ``"Foo"``, then the type checker would infer the + object returned from ``SubclassOfFoo.return_self`` as being of type ``Foo`` + rather than ``SubclassOfFoo``. Other common use cases include: @@ -941,6 +941,17 @@ using ``[]``. of the ``cls`` parameter. - Annotating an :meth:`~object.__enter__` method which returns self. + You should not use ``Self`` as the return annotation if the method is not + guaranteed to return an instance of a subclass when the class is + subclassed:: + + class Eggs: + # Self would be an incorrect return annotation here, + # as the object returned is always an instance of Eggs, + # even in subclasses + def returns_eggs(self) -> "Eggs": + return Eggs() + See :pep:`673` for more details. .. versionadded:: 3.11 From webhook-mailer at python.org Tue Aug 1 16:42:43 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Tue, 01 Aug 2023 20:42:43 -0000 Subject: [Python-checkins] gh-104683: Argument clinic: cleanup `DLSParser` `state_foo` methods (#107543) Message-ID: https://github.com/python/cpython/commit/818c83cf819244bb0e77d956d28b84dd7434cabb commit: 818c83cf819244bb0e77d956d28b84dd7434cabb branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-01T21:42:39+01:00 summary: gh-104683: Argument clinic: cleanup `DLSParser` `state_foo` methods (#107543) files: M Tools/clinic/clinic.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index ba0132fc35903..f9409c7f088d6 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -45,7 +45,6 @@ NamedTuple, NoReturn, Protocol, - TypeGuard, TypeVar, cast, overload, @@ -4388,7 +4387,7 @@ def dedent(self, line: str) -> str: return line[indent:] -StateKeeper = Callable[[str | None], None] +StateKeeper = Callable[[str], None] ConverterArgs = dict[str, Any] class ParamState(enum.IntEnum): @@ -4610,9 +4609,7 @@ def parse(self, block: Block) -> None: fail('Tab characters are illegal in the Clinic DSL.\n\t' + repr(line), line_number=block_start) self.state(line) - self.next(self.state_terminal) - self.state(None) - + self.do_post_block_processing_cleanup() block.output.extend(self.clinic.language.render(self.clinic, block.signatures)) if self.preserve_output: @@ -4621,10 +4618,7 @@ def parse(self, block: Block) -> None: block.output = self.saved_output @staticmethod - def valid_line(line: str | None) -> TypeGuard[str]: - if line is None: - return False - + def valid_line(line: str) -> bool: # ignore comment-only lines if line.lstrip().startswith('#'): return False @@ -4650,7 +4644,7 @@ def next( if line is not None: self.state(line) - def state_dsl_start(self, line: str | None) -> None: + def state_dsl_start(self, line: str) -> None: # self.block = self.ClinicOutputBlock(self) if not self.valid_line(line): return @@ -4668,7 +4662,7 @@ def state_dsl_start(self, line: str | None) -> None: self.next(self.state_modulename_name, line) - def state_modulename_name(self, line: str | None) -> None: + def state_modulename_name(self, line: str) -> None: # looking for declaration, which establishes the leftmost column # line should be # modulename.fnname [as c_basename] [-> return annotation] @@ -4857,7 +4851,7 @@ def state_modulename_name(self, line: str | None) -> None: # separate boolean state variables.) The states are defined in the # ParamState class. - def state_parameters_start(self, line: str | None) -> None: + def state_parameters_start(self, line: str) -> None: if not self.valid_line(line): return @@ -4879,7 +4873,7 @@ def to_required(self) -> None: for p in self.function.parameters.values(): p.group = -p.group - def state_parameter(self, line: str | None) -> None: + def state_parameter(self, line: str) -> None: assert isinstance(self.function, Function) if not self.valid_line(line): @@ -5262,7 +5256,7 @@ def parse_slash(self, function: Function) -> None: "positional-only parameters, which is unsupported.") p.kind = inspect.Parameter.POSITIONAL_ONLY - def state_parameter_docstring_start(self, line: str | None) -> None: + def state_parameter_docstring_start(self, line: str) -> None: assert self.indent.margin is not None, "self.margin.infer() has not yet been called to set the margin" self.parameter_docstring_indent = len(self.indent.margin) assert self.indent.depth == 3 @@ -5271,9 +5265,7 @@ def state_parameter_docstring_start(self, line: str | None) -> None: # every line of the docstring must start with at least F spaces, # where F > P. # these F spaces will be stripped. - def state_parameter_docstring(self, line: str | None) -> None: - assert line is not None - + def state_parameter_docstring(self, line: str) -> None: stripped = line.strip() if stripped.startswith('#'): return @@ -5301,9 +5293,8 @@ def state_parameter_docstring(self, line: str | None) -> None: last_parameter.docstring = new_docstring # the final stanza of the DSL is the docstring. - def state_function_docstring(self, line: str | None) -> None: + def state_function_docstring(self, line: str) -> None: assert self.function is not None - assert line is not None if self.group: fail("Function " + self.function.name + " has a ] without a matching [.") @@ -5572,12 +5563,10 @@ def add_parameter(text: str) -> None: return docstring - def state_terminal(self, line: str | None) -> None: + def do_post_block_processing_cleanup(self) -> None: """ Called when processing the block is done. """ - assert not line - if not self.function: return From webhook-mailer at python.org Tue Aug 1 19:32:31 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 01 Aug 2023 23:32:31 -0000 Subject: [Python-checkins] gh-104683: Argument Clinic: Refactor and simplify 'add docstring' states (#107550) Message-ID: https://github.com/python/cpython/commit/b4d889778198e11257c3ed7ab456539b3933d737 commit: b4d889778198e11257c3ed7ab456539b3933d737 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-01T23:32:27Z summary: gh-104683: Argument Clinic: Refactor and simplify 'add docstring' states (#107550) Introduce docstring_append() helper, and use it for both parameter and function docstrings. Remove docstring fixup from do_post_block_processing_cleanup(); instead, make sure no fixup is needed. Co-authored-by: Alex Waygood files: M Lib/test/test_clinic.py M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 6100444e6da79..6bdc571dd4d5a 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -648,6 +648,28 @@ def test_function_docstring(self): Path to be examined """) + def test_docstring_trailing_whitespace(self): + function = self.parse_function( + "module t\n" + "t.s\n" + " a: object\n" + " Param docstring with trailing whitespace \n" + "Func docstring summary with trailing whitespace \n" + " \n" + "Func docstring body with trailing whitespace \n" + ) + self.checkDocstring(function, """ + s($module, /, a) + -- + + Func docstring summary with trailing whitespace + + a + Param docstring with trailing whitespace + + Func docstring body with trailing whitespace + """) + def test_explicit_parameters_in_docstring(self): function = self.parse_function(dedent(""" module foo diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index f9409c7f088d6..642fb7e791f05 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4617,15 +4617,21 @@ def parse(self, block: Block) -> None: fail("'preserve' only works for blocks that don't produce any output!") block.output = self.saved_output - @staticmethod - def valid_line(line: str) -> bool: + def in_docstring(self) -> bool: + """Return true if we are processing a docstring.""" + return self.state in { + self.state_parameter_docstring, + self.state_function_docstring, + } + + def valid_line(self, line: str) -> bool: # ignore comment-only lines if line.lstrip().startswith('#'): return False # Ignore empty lines too # (but not in docstring sections!) - if not line.strip(): + if not self.in_docstring() and not line.strip(): return False return True @@ -5262,12 +5268,20 @@ def state_parameter_docstring_start(self, line: str) -> None: assert self.indent.depth == 3 return self.next(self.state_parameter_docstring, line) + def docstring_append(self, obj: Function | Parameter, line: str) -> None: + """Add a rstripped line to the current docstring.""" + docstring = obj.docstring + if docstring: + docstring += "\n" + if stripped := line.rstrip(): + docstring += self.indent.dedent(stripped) + obj.docstring = docstring + # every line of the docstring must start with at least F spaces, # where F > P. # these F spaces will be stripped. def state_parameter_docstring(self, line: str) -> None: - stripped = line.strip() - if stripped.startswith('#'): + if not self.valid_line(line): return indent = self.indent.measure(line) @@ -5281,16 +5295,8 @@ def state_parameter_docstring(self, line: str) -> None: return self.next(self.state_function_docstring, line) assert self.function and self.function.parameters - last_parameter = next(reversed(list(self.function.parameters.values()))) - - new_docstring = last_parameter.docstring - - if new_docstring: - new_docstring += '\n' - if stripped: - new_docstring += self.indent.dedent(line) - - last_parameter.docstring = new_docstring + last_param = next(reversed(self.function.parameters.values())) + self.docstring_append(last_param, line) # the final stanza of the DSL is the docstring. def state_function_docstring(self, line: str) -> None: @@ -5299,19 +5305,10 @@ def state_function_docstring(self, line: str) -> None: if self.group: fail("Function " + self.function.name + " has a ] without a matching [.") - stripped = line.strip() - if stripped.startswith('#'): + if not self.valid_line(line): return - new_docstring = self.function.docstring - if new_docstring: - new_docstring += "\n" - if stripped: - line = self.indent.dedent(line).rstrip() - else: - line = '' - new_docstring += line - self.function.docstring = new_docstring + self.docstring_append(self.function, line) def format_docstring(self) -> str: f = self.function @@ -5580,12 +5577,6 @@ def do_post_block_processing_cleanup(self) -> None: if no_parameter_after_star: fail("Function " + self.function.name + " specifies '*' without any parameters afterwards.") - # remove trailing whitespace from all parameter docstrings - for name, value in self.function.parameters.items(): - if not value: - continue - value.docstring = value.docstring.rstrip() - self.function.docstring = self.format_docstring() From webhook-mailer at python.org Tue Aug 1 19:52:32 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 01 Aug 2023 23:52:32 -0000 Subject: [Python-checkins] [3.12] Clarify `Self` interaction with subclasses (GH-107511) (#107548) Message-ID: https://github.com/python/cpython/commit/f7e16d74adffb8bc890530caebf0a08a6ea89d36 commit: f7e16d74adffb8bc890530caebf0a08a6ea89d36 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-02T01:52:28+02:00 summary: [3.12] Clarify `Self` interaction with subclasses (GH-107511) (#107548) Clarify `Self` interaction with subclasses (GH-107511) (cherry picked from commit c8872f4285d3b61c252e3384bec6d30618b7d698) Co-authored-by: Alexandru M?r??teanu files: M Doc/library/typing.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 60bf06e471cb2..d060066c0cdea 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -953,13 +953,17 @@ using ``[]``. For example:: - from typing import Self + from typing import Self, reveal_type class Foo: def return_self(self) -> Self: ... return self + class SubclassOfFoo(Foo): pass + + reveal_type(Foo().return_self()) # Revealed type is "Foo" + reveal_type(SubclassOfFoo().return_self()) # Revealed type is "SubclassOfFoo" This annotation is semantically equivalent to the following, albeit in a more succinct fashion:: @@ -973,15 +977,11 @@ using ``[]``. ... return self - In general if something currently follows the pattern of:: - - class Foo: - def return_self(self) -> "Foo": - ... - return self - - You should use :data:`Self` as calls to ``SubclassOfFoo.return_self`` would have - ``Foo`` as the return type and not ``SubclassOfFoo``. + In general, if something returns ``self``, as in the above examples, you + should use ``Self`` as the return annotation. If ``Foo.return_self`` was + annotated as returning ``"Foo"``, then the type checker would infer the + object returned from ``SubclassOfFoo.return_self`` as being of type ``Foo`` + rather than ``SubclassOfFoo``. Other common use cases include: @@ -989,6 +989,17 @@ using ``[]``. of the ``cls`` parameter. - Annotating an :meth:`~object.__enter__` method which returns self. + You should not use ``Self`` as the return annotation if the method is not + guaranteed to return an instance of a subclass when the class is + subclassed:: + + class Eggs: + # Self would be an incorrect return annotation here, + # as the object returned is always an instance of Eggs, + # even in subclasses + def returns_eggs(self) -> "Eggs": + return Eggs() + See :pep:`673` for more details. .. versionadded:: 3.11 From webhook-mailer at python.org Wed Aug 2 05:22:22 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 02 Aug 2023 09:22:22 -0000 Subject: [Python-checkins] gh-104146: Remove dead code from Argument Clinic (#107555) Message-ID: https://github.com/python/cpython/commit/439466a02b145b522a8969267264743706336501 commit: 439466a02b145b522a8969267264743706336501 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-02T09:22:17Z summary: gh-104146: Remove dead code from Argument Clinic (#107555) files: M Tools/clinic/clinic.py M Tools/clinic/cpp.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 642fb7e791f05..5f7d41e441551 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1098,7 +1098,6 @@ def parser_body( parsearg = p.converter.parse_arg(argname, displayname) if parsearg is None: - #print('Cannot convert %s %r for %s' % (p.converter.__class__.__name__, p.converter.format_unit, p.converter.name), file=sys.stderr) parser_code = None break if has_optional or p.is_optional(): @@ -1191,7 +1190,6 @@ def parser_body( displayname = p.get_displayname(i+1) parsearg = p.converter.parse_arg(argname_fmt % i, displayname) if parsearg is None: - #print('Cannot convert %s %r for %s' % (p.converter.__class__.__name__, p.converter.format_unit, p.converter.name), file=sys.stderr) parser_code = None break if add_label and (i == pos_only or i == max_pos): @@ -4645,13 +4643,11 @@ def next( state: StateKeeper, line: str | None = None ) -> None: - # real_print(self.state.__name__, "->", state.__name__, ", line=", line) self.state = state if line is not None: self.state(line) def state_dsl_start(self, line: str) -> None: - # self.block = self.ClinicOutputBlock(self) if not self.valid_line(line): return diff --git a/Tools/clinic/cpp.py b/Tools/clinic/cpp.py index fbac81336b545..5b7fa06494ad5 100644 --- a/Tools/clinic/cpp.py +++ b/Tools/clinic/cpp.py @@ -1,7 +1,6 @@ import dataclasses as dc import re import sys -from collections.abc import Callable from typing import NoReturn From webhook-mailer at python.org Wed Aug 2 08:40:28 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 02 Aug 2023 12:40:28 -0000 Subject: [Python-checkins] gh-107559: Argument Clinic: complain about non-ASCII chars in param docstrings (#107560) Message-ID: https://github.com/python/cpython/commit/9ff7b4af137b8028b04b52addf003c4b0607113b commit: 9ff7b4af137b8028b04b52addf003c4b0607113b branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-02T12:40:23Z summary: gh-107559: Argument Clinic: complain about non-ASCII chars in param docstrings (#107560) Previously, only function docstrings were checked for non-ASCII characters. Also, improve the warn() message. Co-authored-by: Alex Waygood files: M Lib/test/test_clinic.py M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 6bdc571dd4d5a..6f53036366891 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1427,6 +1427,25 @@ def test_scaffolding(self): actual = stdout.getvalue() self.assertEqual(actual, expected) + def test_non_ascii_character_in_docstring(self): + block = """ + module test + test.fn + a: int + ? param docstring + docstring f? b?r ba? + """ + with support.captured_stdout() as stdout: + self.parse(block) + # The line numbers are off; this is a known limitation. + expected = dedent("""\ + Warning on line 0: + Non-ascii characters are not allowed in docstrings: '?' + Warning on line 0: + Non-ascii characters are not allowed in docstrings: '?', '?', '?' + """) + self.assertEqual(stdout.getvalue(), expected) + class ClinicExternalTest(TestCase): maxDiff = None diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 5f7d41e441551..1f461665003c8 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -785,9 +785,6 @@ def docstring_for_c_string( self, f: Function ) -> str: - if re.search(r'[^\x00-\x7F]', f.docstring): - warn("Non-ascii character appear in docstring.") - text, add, output = _text_accumulator() # turn docstring into a properly quoted C string for line in f.docstring.split('\n'): @@ -5266,6 +5263,11 @@ def state_parameter_docstring_start(self, line: str) -> None: def docstring_append(self, obj: Function | Parameter, line: str) -> None: """Add a rstripped line to the current docstring.""" + matches = re.finditer(r'[^\x00-\x7F]', line) + if offending := ", ".join([repr(m[0]) for m in matches]): + warn("Non-ascii characters are not allowed in docstrings:", + offending) + docstring = obj.docstring if docstring: docstring += "\n" From webhook-mailer at python.org Wed Aug 2 09:33:14 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 02 Aug 2023 13:33:14 -0000 Subject: [Python-checkins] gh-106368: Increase test coverage for Argument Clinic (#107514) Message-ID: https://github.com/python/cpython/commit/b9c9a36c2f2edc11b9c27eb7c5810919d9da9767 commit: b9c9a36c2f2edc11b9c27eb7c5810919d9da9767 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-02T13:33:10Z summary: gh-106368: Increase test coverage for Argument Clinic (#107514) As per this commit, we've got approx. ~91% test coverage for clinic.py. files: M Lib/test/clinic.test.c M Lib/test/test_clinic.py diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index a660ccf8876e2..a49c2e7781064 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5004,3 +5004,263 @@ PyDoc_STRVAR(new_dest__doc__, "\n" "Only this docstring should be outputted to test1."); /*[clinic end generated code: output=9cac703f51d90e84 input=090db8df4945576d]*/ + + +/*[clinic input] +mangled_c_keyword_identifier + i as int: int +The 'int' param should be mangled as 'int_value' +[clinic start generated code]*/ + +PyDoc_STRVAR(mangled_c_keyword_identifier__doc__, +"mangled_c_keyword_identifier($module, /, i)\n" +"--\n" +"\n" +"The \'int\' param should be mangled as \'int_value\'"); + +#define MANGLED_C_KEYWORD_IDENTIFIER_METHODDEF \ + {"mangled_c_keyword_identifier", _PyCFunction_CAST(mangled_c_keyword_identifier), METH_FASTCALL|METH_KEYWORDS, mangled_c_keyword_identifier__doc__}, + +static PyObject * +mangled_c_keyword_identifier_impl(PyObject *module, int int_value); + +static PyObject * +mangled_c_keyword_identifier(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(i), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"i", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "mangled_c_keyword_identifier", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + int int_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + int_value = _PyLong_AsInt(args[0]); + if (int_value == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = mangled_c_keyword_identifier_impl(module, int_value); + +exit: + return return_value; +} + +static PyObject * +mangled_c_keyword_identifier_impl(PyObject *module, int int_value) +/*[clinic end generated code: output=c049d7d79be26cda input=060876448ab567a2]*/ + + +/*[clinic input] +bool_return -> bool +[clinic start generated code]*/ + +PyDoc_STRVAR(bool_return__doc__, +"bool_return($module, /)\n" +"--\n" +"\n"); + +#define BOOL_RETURN_METHODDEF \ + {"bool_return", (PyCFunction)bool_return, METH_NOARGS, bool_return__doc__}, + +static int +bool_return_impl(PyObject *module); + +static PyObject * +bool_return(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = bool_return_impl(module); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + +static int +bool_return_impl(PyObject *module) +/*[clinic end generated code: output=3a65f07830e48e98 input=93ba95d39ee98f39]*/ + + +/*[clinic input] +double_return -> double +[clinic start generated code]*/ + +PyDoc_STRVAR(double_return__doc__, +"double_return($module, /)\n" +"--\n" +"\n"); + +#define DOUBLE_RETURN_METHODDEF \ + {"double_return", (PyCFunction)double_return, METH_NOARGS, double_return__doc__}, + +static double +double_return_impl(PyObject *module); + +static PyObject * +double_return(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + double _return_value; + + _return_value = double_return_impl(module); + if ((_return_value == -1.0) && PyErr_Occurred()) { + goto exit; + } + return_value = PyFloat_FromDouble(_return_value); + +exit: + return return_value; +} + +static double +double_return_impl(PyObject *module) +/*[clinic end generated code: output=076dc72595d3f66d input=da11b6255e4cbfd7]*/ + + +/*[clinic input] +Test.__init__ + a: object + [ + b: object + ] + / +Should generate two PyArg_ParseTuple calls. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test___init____doc__, +"Test(a, [b])\n" +"Should generate two PyArg_ParseTuple calls."); + +static int +Test___init___impl(TestObj *self, PyObject *a, int group_right_1, + PyObject *b); + +static int +Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = TestType; + PyObject *a; + int group_right_1 = 0; + PyObject *b = NULL; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + switch (PyTuple_GET_SIZE(args)) { + case 1: + if (!PyArg_ParseTuple(args, "O:__init__", &a)) { + goto exit; + } + break; + case 2: + if (!PyArg_ParseTuple(args, "OO:__init__", &a, &b)) { + goto exit; + } + group_right_1 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "Test.__init__ requires 1 to 2 arguments"); + goto exit; + } + return_value = Test___init___impl((TestObj *)self, a, group_right_1, b); + +exit: + return return_value; +} + +static int +Test___init___impl(TestObj *self, PyObject *a, int group_right_1, + PyObject *b) +/*[clinic end generated code: output=2bbb8ea60e8f57a6 input=10f5d0f1e8e466ef]*/ + + +/*[clinic input] +Test._pyarg_parsestackandkeywords + cls: defining_class + key: str(accept={str, robuffer}, zeroes=True) + / +Check that _PyArg_ParseStackAndKeywords() is generated. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test__pyarg_parsestackandkeywords__doc__, +"_pyarg_parsestackandkeywords($self, key, /)\n" +"--\n" +"\n" +"Check that _PyArg_ParseStackAndKeywords() is generated."); + +#define TEST__PYARG_PARSESTACKANDKEYWORDS_METHODDEF \ + {"_pyarg_parsestackandkeywords", _PyCFunction_CAST(Test__pyarg_parsestackandkeywords), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, Test__pyarg_parsestackandkeywords__doc__}, + +static PyObject * +Test__pyarg_parsestackandkeywords_impl(TestObj *self, PyTypeObject *cls, + const char *key, + Py_ssize_t key_length); + +static PyObject * +Test__pyarg_parsestackandkeywords(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "s#:_pyarg_parsestackandkeywords", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + const char *key; + Py_ssize_t key_length; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &key, &key_length)) { + goto exit; + } + return_value = Test__pyarg_parsestackandkeywords_impl(self, cls, key, key_length); + +exit: + return return_value; +} + +static PyObject * +Test__pyarg_parsestackandkeywords_impl(TestObj *self, PyTypeObject *cls, + const char *key, + Py_ssize_t key_length) +/*[clinic end generated code: output=4fda8a7f2547137c input=fc72ef4b4cfafabc]*/ diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 6f53036366891..cdabcbaa6f03c 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -8,6 +8,7 @@ from textwrap import dedent from unittest import TestCase import collections +import contextlib import inspect import os.path import sys @@ -335,6 +336,70 @@ def converter_init(self): ) self.assertIn(msg, out) + @staticmethod + @contextlib.contextmanager + def _clinic_version(new_version): + """Helper for test_version_*() tests""" + _saved = clinic.version + clinic.version = new_version + try: + yield + finally: + clinic.version = _saved + + def test_version_directive(self): + dataset = ( + # (clinic version, required version) + ('3', '2'), # required version < clinic version + ('3.1', '3.0'), # required version < clinic version + ('1.2b0', '1.2a7'), # required version < clinic version + ('5', '5'), # required version == clinic version + ('6.1', '6.1'), # required version == clinic version + ('1.2b3', '1.2b3'), # required version == clinic version + ) + for clinic_version, required_version in dataset: + with self.subTest(clinic_version=clinic_version, + required_version=required_version): + with self._clinic_version(clinic_version): + block = dedent(f""" + /*[clinic input] + version {required_version} + [clinic start generated code]*/ + """) + self.clinic.parse(block) + + def test_version_directive_insufficient_version(self): + with self._clinic_version('4'): + err = ( + "Insufficient Clinic version!\n" + " Version: 4\n" + " Required: 5" + ) + out = self.expect_failure(""" + /*[clinic input] + version 5 + [clinic start generated code]*/ + """) + self.assertIn(err, out) + + def test_version_directive_illegal_char(self): + err = "Illegal character 'v' in version string 'v5'" + out = self.expect_failure(""" + /*[clinic input] + version v5 + [clinic start generated code]*/ + """) + self.assertIn(err, out) + + def test_version_directive_unsupported_string(self): + err = "Unsupported version string: '.-'" + out = self.expect_failure(""" + /*[clinic input] + version .- + [clinic start generated code]*/ + """) + self.assertIn(err, out) + class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): @@ -513,6 +578,22 @@ def test_clinic_1(self): class ClinicParserTest(_ParserBase): + + def parse(self, text): + c = FakeClinic() + parser = DSLParser(c) + block = clinic.Block(text) + parser.parse(block) + return block + + def parse_function(self, text, signatures_in_block=2, function_index=1): + block = self.parse(text) + s = block.signatures + self.assertEqual(len(s), signatures_in_block) + assert isinstance(s[0], clinic.Module) + assert isinstance(s[function_index], clinic.Function) + return s[function_index] + def checkDocstring(self, fn, expected): self.assertTrue(hasattr(fn, "docstring")) self.assertEqual(fn.docstring.strip(), @@ -1395,21 +1476,6 @@ def test_unused_param(self): parser_decl = p.simple_declaration(in_parser=True) self.assertNotIn("Py_UNUSED", parser_decl) - def parse(self, text): - c = FakeClinic() - parser = DSLParser(c) - block = clinic.Block(text) - parser.parse(block) - return block - - def parse_function(self, text, signatures_in_block=2, function_index=1): - block = self.parse(text) - s = block.signatures - self.assertEqual(len(s), signatures_in_block) - assert isinstance(s[0], clinic.Module) - assert isinstance(s[function_index], clinic.Function) - return s[function_index] - def test_scaffolding(self): # test repr on special values self.assertEqual(repr(clinic.unspecified), '') @@ -1446,6 +1512,15 @@ def test_non_ascii_character_in_docstring(self): """) self.assertEqual(stdout.getvalue(), expected) + def test_illegal_c_identifier(self): + err = "Illegal C identifier: 17a" + out = self.parse_function_should_fail(""" + module test + test.fn + a as 17a: int + """) + self.assertIn(err, out) + class ClinicExternalTest(TestCase): maxDiff = None From webhook-mailer at python.org Wed Aug 2 13:17:02 2023 From: webhook-mailer at python.org (iritkatriel) Date: Wed, 02 Aug 2023 17:17:02 -0000 Subject: [Python-checkins] gh-105481: simplify definition of pseudo ops in Lib/opcode.py (#107561) Message-ID: https://github.com/python/cpython/commit/dd693d6320feeca887174fa592537669d017ca9b commit: dd693d6320feeca887174fa592537669d017ca9b branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-02T18:16:57+01:00 summary: gh-105481: simplify definition of pseudo ops in Lib/opcode.py (#107561) files: A Misc/NEWS.d/next/Library/2023-08-01-21-43-58.gh-issue-105481.cl2ajS.rst M Doc/whatsnew/3.13.rst M Include/opcode.h M Lib/opcode.py M Tools/build/generate_opcode_h.py diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 22c1e03470f5d..63cdee6cf1a4f 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -124,6 +124,11 @@ opcode This field was added in 3.12, it was never documented and is not intended for external usage. (Contributed by Irit Katriel in :gh:`105481`.) +* Removed ``opcode.is_pseudo``, ``opcode.MIN_PSEUDO_OPCODE`` and + ``opcode.MAX_PSEUDO_OPCODE``, which were added in 3.12, were never + documented or exposed through ``dis``, and were not intended to be + used externally. + pathlib ------- diff --git a/Include/opcode.h b/Include/opcode.h index eb4bb85e5b667..ede1518b6fd25 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -146,7 +146,6 @@ extern "C" { #define INSTRUMENTED_END_SEND 252 #define INSTRUMENTED_INSTRUCTION 253 #define INSTRUMENTED_LINE 254 -#define MIN_PSEUDO_OPCODE 256 #define SETUP_FINALLY 256 #define SETUP_CLEANUP 257 #define SETUP_WITH 258 @@ -159,7 +158,6 @@ extern "C" { #define LOAD_ZERO_SUPER_ATTR 265 #define STORE_FAST_MAYBE_NULL 266 #define LOAD_CLOSURE 267 -#define MAX_PSEUDO_OPCODE 267 #define TO_BOOL_ALWAYS_TRUE 7 #define TO_BOOL_BOOL 8 #define TO_BOOL_INT 10 diff --git a/Lib/opcode.py b/Lib/opcode.py index 51432ab1fab3f..5a9f8ddd0738d 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -19,23 +19,11 @@ cmp_op = ('<', '<=', '==', '!=', '>', '>=') -def is_pseudo(op): - return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE - opmap = {} -# pseudo opcodes (used in the compiler) mapped to the values -#?they can become in the actual code. -_pseudo_ops = {} - def def_op(name, op): opmap[name] = op -def pseudo_op(name, op, real_ops): - def_op(name, op) - _pseudo_ops[name] = real_ops - - # Instruction opcodes for compiled code # Blank lines correspond to available opcodes @@ -212,29 +200,27 @@ def pseudo_op(name, op, real_ops): # 255 is reserved -MIN_PSEUDO_OPCODE = 256 - -pseudo_op('SETUP_FINALLY', 256, ['NOP']) -pseudo_op('SETUP_CLEANUP', 257, ['NOP']) -pseudo_op('SETUP_WITH', 258, ['NOP']) -pseudo_op('POP_BLOCK', 259, ['NOP']) +# Pseudo ops are above 255: -pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD']) -pseudo_op('JUMP_NO_INTERRUPT', 261, ['JUMP_FORWARD', 'JUMP_BACKWARD_NO_INTERRUPT']) +def_op('SETUP_FINALLY', 256) +def_op('SETUP_CLEANUP', 257) +def_op('SETUP_WITH', 258) +def_op('POP_BLOCK', 259) -pseudo_op('LOAD_METHOD', 262, ['LOAD_ATTR']) -pseudo_op('LOAD_SUPER_METHOD', 263, ['LOAD_SUPER_ATTR']) -pseudo_op('LOAD_ZERO_SUPER_METHOD', 264, ['LOAD_SUPER_ATTR']) -pseudo_op('LOAD_ZERO_SUPER_ATTR', 265, ['LOAD_SUPER_ATTR']) +def_op('JUMP', 260) +def_op('JUMP_NO_INTERRUPT', 261) -pseudo_op('STORE_FAST_MAYBE_NULL', 266, ['STORE_FAST']) -pseudo_op('LOAD_CLOSURE', 267, ['LOAD_FAST']) +def_op('LOAD_METHOD', 262) +def_op('LOAD_SUPER_METHOD', 263) +def_op('LOAD_ZERO_SUPER_METHOD', 264) +def_op('LOAD_ZERO_SUPER_ATTR', 265) -MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1 +def_op('STORE_FAST_MAYBE_NULL', 266) +def_op('LOAD_CLOSURE', 267) -del def_op, pseudo_op +del def_op -opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)] +opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] for op, i in opmap.items(): opname[i] = op diff --git a/Misc/NEWS.d/next/Library/2023-08-01-21-43-58.gh-issue-105481.cl2ajS.rst b/Misc/NEWS.d/next/Library/2023-08-01-21-43-58.gh-issue-105481.cl2ajS.rst new file mode 100644 index 0000000000000..d02f909e87018 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-01-21-43-58.gh-issue-105481.cl2ajS.rst @@ -0,0 +1,2 @@ +Remove ``opcode.is_pseudo``, ``opcode.MIN_PSEUDO_OPCODE`` and ``opcode.MAX_PSEUDO_OPCODE``, +which were added in 3.12, were never documented and were not intended to be used externally. diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 16b028dc1205a..3a817326c94cb 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -72,10 +72,7 @@ def main(opcode_py, opcode = get_python_module_dict(opcode_py) opmap = opcode['opmap'] opname = opcode['opname'] - is_pseudo = opcode['is_pseudo'] - MIN_PSEUDO_OPCODE = opcode["MIN_PSEUDO_OPCODE"] - MAX_PSEUDO_OPCODE = opcode["MAX_PSEUDO_OPCODE"] MIN_INSTRUMENTED_OPCODE = opcode["MIN_INSTRUMENTED_OPCODE"] NUM_OPCODES = len(opname) @@ -101,16 +98,11 @@ def main(opcode_py, for name in opname: if name in opmap: op = opmap[name] - if op == MIN_PSEUDO_OPCODE: - fobj.write(DEFINE.format("MIN_PSEUDO_OPCODE", MIN_PSEUDO_OPCODE)) if op == MIN_INSTRUMENTED_OPCODE: fobj.write(DEFINE.format("MIN_INSTRUMENTED_OPCODE", MIN_INSTRUMENTED_OPCODE)) fobj.write(DEFINE.format(name, op)) - if op == MAX_PSEUDO_OPCODE: - fobj.write(DEFINE.format("MAX_PSEUDO_OPCODE", MAX_PSEUDO_OPCODE)) - for name, op in specialized_opmap.items(): fobj.write(DEFINE.format(name, op)) @@ -126,7 +118,7 @@ def main(opcode_py, deoptcodes = {} for basic, op in opmap.items(): - if not is_pseudo(op): + if op < 256: deoptcodes[basic] = basic for basic, family in _opcode_metadata["_specializations"].items(): for specialized in family: From webhook-mailer at python.org Wed Aug 2 13:44:24 2023 From: webhook-mailer at python.org (markshannon) Date: Wed, 02 Aug 2023 17:44:24 -0000 Subject: [Python-checkins] GH-100964: Break cycles involving exception state when returning from generator (GH-107563) Message-ID: https://github.com/python/cpython/commit/0d30a5a40968cce19750be78154232fae25d641f commit: 0d30a5a40968cce19750be78154232fae25d641f branch: main author: Mark Shannon committer: markshannon date: 2023-08-02T18:44:20+01:00 summary: GH-100964: Break cycles involving exception state when returning from generator (GH-107563) files: A Misc/NEWS.d/next/Core and Builtins/2023-07-30-18-05-11.gh-issue-100964.HluhBJ.rst M Objects/genobject.c M Python/ceval.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-30-18-05-11.gh-issue-100964.HluhBJ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-18-05-11.gh-issue-100964.HluhBJ.rst new file mode 100644 index 0000000000000..99ebc926e2ce2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-18-05-11.gh-issue-100964.HluhBJ.rst @@ -0,0 +1,2 @@ +Clear generators' exception state after ``return`` to break reference +cycles. diff --git a/Objects/genobject.c b/Objects/genobject.c index a630f84fb5a29..65782be182cd7 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -149,14 +149,16 @@ gen_dealloc(PyGenObject *gen) gen->gi_frame_state = FRAME_CLEARED; frame->previous = NULL; _PyFrame_ClearExceptCode(frame); + _PyErr_ClearExcState(&gen->gi_exc_state); } + assert(gen->gi_exc_state.exc_value == NULL); if (_PyGen_GetCode(gen)->co_flags & CO_COROUTINE) { Py_CLEAR(((PyCoroObject *)gen)->cr_origin_or_finalizer); } Py_DECREF(_PyGen_GetCode(gen)); Py_CLEAR(gen->gi_name); Py_CLEAR(gen->gi_qualname); - _PyErr_ClearExcState(&gen->gi_exc_state); + PyObject_GC_Del(gen); } @@ -252,10 +254,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, !PyErr_ExceptionMatches(PyExc_StopAsyncIteration)); } - /* generator can't be rerun, so release the frame */ - /* first clean reference cycle through stored exception traceback */ - _PyErr_ClearExcState(&gen->gi_exc_state); - + assert(gen->gi_exc_state.exc_value == NULL); assert(gen->gi_frame_state == FRAME_CLEARED); *presult = result; return result ? PYGEN_RETURN : PYGEN_ERROR; diff --git a/Python/ceval.c b/Python/ceval.c index 17818a0d3c17e..369b9a69152a5 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1466,6 +1466,7 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) tstate->c_recursion_remaining--; assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); _PyFrame_ClearExceptCode(frame); + _PyErr_ClearExcState(&gen->gi_exc_state); tstate->c_recursion_remaining++; frame->previous = NULL; } From webhook-mailer at python.org Wed Aug 2 15:05:29 2023 From: webhook-mailer at python.org (gvanrossum) Date: Wed, 02 Aug 2023 19:05:29 -0000 Subject: [Python-checkins] Fix test_capi.test_misc when run with -R:: (#107566) Message-ID: https://github.com/python/cpython/commit/af8141cf879f606ed71da42e12f7ec0275f561db commit: af8141cf879f606ed71da42e12f7ec0275f561db branch: main author: Guido van Rossum committer: gvanrossum date: 2023-08-02T12:05:25-07:00 summary: Fix test_capi.test_misc when run with -R:: (#107566) Should fix the buildbot failures. This creates a new function each time that test is run, like Victor did for other tests. files: M Lib/test/test_capi/test_misc.py diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 0d3f93941e820..e7cdd4be002a1 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2464,36 +2464,40 @@ def testfunc(x): def test_extended_arg(self): "Check EXTENDED_ARG handling in superblock creation" - def many_vars(): - # 260 vars, so z9 should have index 259 - a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 - b0 = b1 = b2 = b3 = b4 = b5 = b6 = b7 = b8 = b9 = 42 - c0 = c1 = c2 = c3 = c4 = c5 = c6 = c7 = c8 = c9 = 42 - d0 = d1 = d2 = d3 = d4 = d5 = d6 = d7 = d8 = d9 = 42 - e0 = e1 = e2 = e3 = e4 = e5 = e6 = e7 = e8 = e9 = 42 - f0 = f1 = f2 = f3 = f4 = f5 = f6 = f7 = f8 = f9 = 42 - g0 = g1 = g2 = g3 = g4 = g5 = g6 = g7 = g8 = g9 = 42 - h0 = h1 = h2 = h3 = h4 = h5 = h6 = h7 = h8 = h9 = 42 - i0 = i1 = i2 = i3 = i4 = i5 = i6 = i7 = i8 = i9 = 42 - j0 = j1 = j2 = j3 = j4 = j5 = j6 = j7 = j8 = j9 = 42 - k0 = k1 = k2 = k3 = k4 = k5 = k6 = k7 = k8 = k9 = 42 - l0 = l1 = l2 = l3 = l4 = l5 = l6 = l7 = l8 = l9 = 42 - m0 = m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m9 = 42 - n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = n9 = 42 - o0 = o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = 42 - p0 = p1 = p2 = p3 = p4 = p5 = p6 = p7 = p8 = p9 = 42 - q0 = q1 = q2 = q3 = q4 = q5 = q6 = q7 = q8 = q9 = 42 - r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = 42 - s0 = s1 = s2 = s3 = s4 = s5 = s6 = s7 = s8 = s9 = 42 - t0 = t1 = t2 = t3 = t4 = t5 = t6 = t7 = t8 = t9 = 42 - u0 = u1 = u2 = u3 = u4 = u5 = u6 = u7 = u8 = u9 = 42 - v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = v8 = v9 = 42 - w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 - x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 - y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 - z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = 42 - while z9 > 0: - z9 = z9 - 1 + ns = {} + exec(textwrap.dedent(""" + def many_vars(): + # 260 vars, so z9 should have index 259 + a0 = a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = 42 + b0 = b1 = b2 = b3 = b4 = b5 = b6 = b7 = b8 = b9 = 42 + c0 = c1 = c2 = c3 = c4 = c5 = c6 = c7 = c8 = c9 = 42 + d0 = d1 = d2 = d3 = d4 = d5 = d6 = d7 = d8 = d9 = 42 + e0 = e1 = e2 = e3 = e4 = e5 = e6 = e7 = e8 = e9 = 42 + f0 = f1 = f2 = f3 = f4 = f5 = f6 = f7 = f8 = f9 = 42 + g0 = g1 = g2 = g3 = g4 = g5 = g6 = g7 = g8 = g9 = 42 + h0 = h1 = h2 = h3 = h4 = h5 = h6 = h7 = h8 = h9 = 42 + i0 = i1 = i2 = i3 = i4 = i5 = i6 = i7 = i8 = i9 = 42 + j0 = j1 = j2 = j3 = j4 = j5 = j6 = j7 = j8 = j9 = 42 + k0 = k1 = k2 = k3 = k4 = k5 = k6 = k7 = k8 = k9 = 42 + l0 = l1 = l2 = l3 = l4 = l5 = l6 = l7 = l8 = l9 = 42 + m0 = m1 = m2 = m3 = m4 = m5 = m6 = m7 = m8 = m9 = 42 + n0 = n1 = n2 = n3 = n4 = n5 = n6 = n7 = n8 = n9 = 42 + o0 = o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = 42 + p0 = p1 = p2 = p3 = p4 = p5 = p6 = p7 = p8 = p9 = 42 + q0 = q1 = q2 = q3 = q4 = q5 = q6 = q7 = q8 = q9 = 42 + r0 = r1 = r2 = r3 = r4 = r5 = r6 = r7 = r8 = r9 = 42 + s0 = s1 = s2 = s3 = s4 = s5 = s6 = s7 = s8 = s9 = 42 + t0 = t1 = t2 = t3 = t4 = t5 = t6 = t7 = t8 = t9 = 42 + u0 = u1 = u2 = u3 = u4 = u5 = u6 = u7 = u8 = u9 = 42 + v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = v8 = v9 = 42 + w0 = w1 = w2 = w3 = w4 = w5 = w6 = w7 = w8 = w9 = 42 + x0 = x1 = x2 = x3 = x4 = x5 = x6 = x7 = x8 = x9 = 42 + y0 = y1 = y2 = y3 = y4 = y5 = y6 = y7 = y8 = y9 = 42 + z0 = z1 = z2 = z3 = z4 = z5 = z6 = z7 = z8 = z9 = 42 + while z9 > 0: + z9 = z9 - 1 + """), ns, ns) + many_vars = ns["many_vars"] opt = _testinternalcapi.get_uop_optimizer() with temporary_optimizer(opt): From webhook-mailer at python.org Wed Aug 2 15:37:40 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 02 Aug 2023 19:37:40 -0000 Subject: [Python-checkins] gh-104683: Make Argument Clinic template strings class level members (#107556) Message-ID: https://github.com/python/cpython/commit/bcdd3072316181b49d94567bb648825a07ca9ae1 commit: bcdd3072316181b49d94567bb648825a07ca9ae1 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-02T21:37:36+02:00 summary: gh-104683: Make Argument Clinic template strings class level members (#107556) The motivation for this change is to clean up the output_templates() method a little bit, as it accounts for ~10% of the lines of code in clinic.py; removing some clutter helps readability. files: M Tools/clinic/clinic.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 1f461665003c8..ce8184753c729 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -760,6 +760,59 @@ class CLanguage(Language): stop_line = "[{dsl_name} start generated code]*/" checksum_line = "/*[{dsl_name} end generated code: {arguments}]*/" + PARSER_PROTOTYPE_KEYWORD: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) + """) + PARSER_PROTOTYPE_KEYWORD___INIT__: Final[str] = normalize_snippet(""" + static int + {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) + """) + PARSER_PROTOTYPE_VARARGS: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *args) + """) + PARSER_PROTOTYPE_FASTCALL: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) + """) + PARSER_PROTOTYPE_FASTCALL_KEYWORDS: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) + """) + PARSER_PROTOTYPE_DEF_CLASS: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) + """) + PARSER_PROTOTYPE_NOARGS: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) + """) + METH_O_PROTOTYPE: Final[str] = normalize_snippet(""" + static PyObject * + {c_basename}({impl_parameters}) + """) + DOCSTRING_PROTOTYPE_VAR: Final[str] = normalize_snippet(""" + PyDoc_VAR({c_basename}__doc__); + """) + DOCSTRING_PROTOTYPE_STRVAR: Final[str] = normalize_snippet(""" + PyDoc_STRVAR({c_basename}__doc__, + {docstring}); + """) + IMPL_DEFINITION_PROTOTYPE: Final[str] = normalize_snippet(""" + static {impl_return_type} + {c_basename}_impl({impl_parameters}) + """) + METHODDEF_PROTOTYPE_DEFINE: Final[str] = normalize_snippet(r""" + #define {methoddef_name} \ + {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, + """) + METHODDEF_PROTOTYPE_IFNDEF: Final[str] = normalize_snippet(""" + #ifndef {methoddef_name} + #define {methoddef_name} + #endif /* !defined({methoddef_name}) */ + """) + def __init__(self, filename: str) -> None: super().__init__(filename) self.cpp = cpp.Monitor(filename) @@ -862,52 +915,15 @@ def output_templates( # methoddef_ifndef return_value_declaration = "PyObject *return_value = NULL;" - - methoddef_define = normalize_snippet(""" - #define {methoddef_name} \\ - {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, - """) + methoddef_define = self.METHODDEF_PROTOTYPE_DEFINE if new_or_init and not f.docstring: docstring_prototype = docstring_definition = '' else: - docstring_prototype = normalize_snippet(""" - PyDoc_VAR({c_basename}__doc__); - """) - docstring_definition = normalize_snippet(""" - PyDoc_STRVAR({c_basename}__doc__, - {docstring}); - """) - impl_definition = normalize_snippet(""" - static {impl_return_type} - {c_basename}_impl({impl_parameters}) - """) + docstring_prototype = self.DOCSTRING_PROTOTYPE_VAR + docstring_definition = self.DOCSTRING_PROTOTYPE_STRVAR + impl_definition = self.IMPL_DEFINITION_PROTOTYPE impl_prototype = parser_prototype = parser_definition = None - parser_prototype_keyword = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) - """) - - parser_prototype_varargs = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *args) - """) - - parser_prototype_fastcall = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs) - """) - - parser_prototype_fastcall_keywords = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) - """) - - parser_prototype_def_class = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyTypeObject *{defining_class_name}, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) - """) - # parser_body_fields remembers the fields passed in to the # previous call to parser_body. this is used for an awful hack. parser_body_fields: tuple[str, ...] = () @@ -950,19 +966,13 @@ def parser_body( if not requires_defining_class: # no parameters, METH_NOARGS flags = "METH_NOARGS" - - parser_prototype = normalize_snippet(""" - static PyObject * - {c_basename}({self_type}{self_name}, PyObject *Py_UNUSED(ignored)) - """) + parser_prototype = self.PARSER_PROTOTYPE_NOARGS parser_code = [] - else: assert not new_or_init flags = "METH_METHOD|METH_FASTCALL|METH_KEYWORDS" - - parser_prototype = parser_prototype_def_class + parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS return_error = ('return NULL;' if default_return_converter else 'goto exit;') parser_code = [normalize_snippet(""" @@ -987,10 +997,7 @@ def parser_body( if (isinstance(converters[0], object_converter) and converters[0].format_unit == 'O'): - meth_o_prototype = normalize_snippet(""" - static PyObject * - {c_basename}({impl_parameters}) - """) + meth_o_prototype = self.METH_O_PROTOTYPE if default_return_converter: # maps perfectly to METH_O, doesn't need a return converter. @@ -1030,8 +1037,7 @@ def parser_body( # in a big switch statement) flags = "METH_VARARGS" - parser_prototype = parser_prototype_varargs - + parser_prototype = self.PARSER_PROTOTYPE_VARARGS parser_definition = parser_body(parser_prototype, ' {option_group_parsing}') elif not requires_defining_class and pos_only == len(parameters) - pseudo_args: @@ -1040,7 +1046,7 @@ def parser_body( # we only need one call to _PyArg_ParseStack flags = "METH_FASTCALL" - parser_prototype = parser_prototype_fastcall + parser_prototype = self.PARSER_PROTOTYPE_FASTCALL nargs = 'nargs' argname_fmt = 'args[%d]' else: @@ -1048,7 +1054,7 @@ def parser_body( # we only need one call to PyArg_ParseTuple flags = "METH_VARARGS" - parser_prototype = parser_prototype_varargs + parser_prototype = self.PARSER_PROTOTYPE_VARARGS nargs = 'PyTuple_GET_SIZE(args)' argname_fmt = 'PyTuple_GET_ITEM(args, %d)' @@ -1145,7 +1151,7 @@ def parser_body( nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" if not new_or_init: flags = "METH_FASTCALL|METH_KEYWORDS" - parser_prototype = parser_prototype_fastcall_keywords + parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS argname_fmt = 'args[%d]' declarations = declare_parser(f) declarations += "\nPyObject *argsbuf[%s];" % len(converters) @@ -1160,7 +1166,7 @@ def parser_body( else: # positional-or-keyword arguments flags = "METH_VARARGS|METH_KEYWORDS" - parser_prototype = parser_prototype_keyword + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD argname_fmt = 'fastargs[%d]' declarations = declare_parser(f) declarations += "\nPyObject *argsbuf[%s];" % len(converters) @@ -1177,7 +1183,7 @@ def parser_body( if requires_defining_class: flags = 'METH_METHOD|' + flags - parser_prototype = parser_prototype_def_class + parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS add_label: str | None = None for i, p in enumerate(parameters): @@ -1264,13 +1270,10 @@ def parser_body( methoddef_define = '' if f.kind is METHOD_NEW: - parser_prototype = parser_prototype_keyword + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD else: return_value_declaration = "int return_value = -1;" - parser_prototype = normalize_snippet(""" - static int - {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) - """) + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD___INIT__ fields = list(parser_body_fields) parses_positional = 'METH_NOARGS' not in flags @@ -1323,12 +1326,7 @@ def parser_body( if methoddef_define and f.full_name not in clinic.ifndef_symbols: clinic.ifndef_symbols.add(f.full_name) - methoddef_ifndef = normalize_snippet(""" - #ifndef {methoddef_name} - #define {methoddef_name} - #endif /* !defined({methoddef_name}) */ - """) - + methoddef_ifndef = self.METHODDEF_PROTOTYPE_IFNDEF # add ';' to the end of parser_prototype and impl_prototype # (they mustn't be None, but they could be an empty string.) From webhook-mailer at python.org Wed Aug 2 16:55:13 2023 From: webhook-mailer at python.org (ericsnowcurrently) Date: Wed, 02 Aug 2023 20:55:13 -0000 Subject: [Python-checkins] gh-107471: Fix Refleaks in test_import (gh-107569) Message-ID: https://github.com/python/cpython/commit/017f047183fa33743f7e36c5c360f5c670032be3 commit: 017f047183fa33743f7e36c5c360f5c670032be3 branch: main author: Eric Snow committer: ericsnowcurrently date: 2023-08-02T20:55:09Z summary: gh-107471: Fix Refleaks in test_import (gh-107569) gh-107184 introduced a refleak in test_import.SubinterpImportTests (specifically test_singlephase_check_with_setting_and_override and test_single_init_extension_compat). We fix it here by making sure _testsinglephase is removed from sys.modules whenever we clear the runtime's internal state for the module. The underlying problem is strictly contained in the internal function _PyImport_ClearExtension() (AKA _testinternalcapi.clear_extension()), which is only used in tests. (This also fixes an intermittent segfault introduced in the same place, in test_disallowed_reimport.) files: M Lib/test/test_import/__init__.py M Python/import.c diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 7a3fcc22be8d1..163ed824ff1fb 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -150,6 +150,7 @@ def _ready_to_import(name=None, source=""): def restore__testsinglephase(*, _orig=_testsinglephase): # We started with the module imported and want to restore # it to its nominal state. + sys.modules.pop('_testsinglephase', None) _orig._clear_globals() _testinternalcapi.clear_extension('_testsinglephase', _orig.__file__) import _testsinglephase @@ -2125,7 +2126,7 @@ def clean_up(): _interpreters.run_string(interpid, textwrap.dedent(f''' name = {self.NAME!r} if name in sys.modules: - sys.modules[name]._clear_globals() + sys.modules.pop(name)._clear_globals() _testinternalcapi.clear_extension(name, {self.FILE!r}) ''')) _interpreters.destroy(interpid) diff --git a/Python/import.c b/Python/import.c index 3be2f76c9eca7..56b2dc1a4ada2 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1073,6 +1073,7 @@ _extensions_cache_delete(PyObject *filename, PyObject *name) However, this decref would be problematic if the module def were dynamically allocated, it were the last ref, and this function were called with an interpreter other than the def's owner. */ + assert(_Py_IsImmortal(entry->value)); entry->value = NULL; finally: From webhook-mailer at python.org Wed Aug 2 17:23:19 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 02 Aug 2023 21:23:19 -0000 Subject: [Python-checkins] [3.12] gh-107471: Fix Refleaks in test_import (gh-107569) (#107571) Message-ID: https://github.com/python/cpython/commit/12d1c494ae236e1a1ce8ae8084f8f9a6b6eb8295 commit: 12d1c494ae236e1a1ce8ae8084f8f9a6b6eb8295 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-02T23:23:15+02:00 summary: [3.12] gh-107471: Fix Refleaks in test_import (gh-107569) (#107571) gh-107471: Fix Refleaks in test_import (gh-107569) gh-107184 introduced a refleak in test_import.SubinterpImportTests (specifically test_singlephase_check_with_setting_and_override and test_single_init_extension_compat). We fix it here by making sure _testsinglephase is removed from sys.modules whenever we clear the runtime's internal state for the module. The underlying problem is strictly contained in the internal function _PyImport_ClearExtension() (AKA _testinternalcapi.clear_extension()), which is only used in tests. (This also fixes an intermittent segfault introduced in the same place, in test_disallowed_reimport.) (cherry picked from commit 017f047183fa33743f7e36c5c360f5c670032be3) Co-authored-by: Eric Snow files: M Lib/test/test_import/__init__.py M Python/import.c diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index cc6e8d4e640cd..06adf01f18c0a 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -131,6 +131,7 @@ def _ready_to_import(name=None, source=""): def restore__testsinglephase(*, _orig=_testsinglephase): # We started with the module imported and want to restore # it to its nominal state. + sys.modules.pop('_testsinglephase', None) _orig._clear_globals() _testinternalcapi.clear_extension('_testsinglephase', _orig.__file__) import _testsinglephase @@ -2110,7 +2111,7 @@ def clean_up(): _interpreters.run_string(interpid, textwrap.dedent(f''' name = {self.NAME!r} if name in sys.modules: - sys.modules[name]._clear_globals() + sys.modules.pop(name)._clear_globals() _testinternalcapi.clear_extension(name, {self.FILE!r}) ''')) _interpreters.destroy(interpid) diff --git a/Python/import.c b/Python/import.c index f8f01f1bcd8c6..3f3f2a2b68135 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1080,6 +1080,7 @@ _extensions_cache_delete(PyObject *filename, PyObject *name) However, this decref would be problematic if the module def were dynamically allocated, it were the last ref, and this function were called with an interpreter other than the def's owner. */ + assert(_Py_IsImmortal(entry->value)); entry->value = NULL; finally: From webhook-mailer at python.org Wed Aug 2 20:00:10 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Thu, 03 Aug 2023 00:00:10 -0000 Subject: [Python-checkins] gh-104683: Rework Argument Clinic error handling (#107551) Message-ID: https://github.com/python/cpython/commit/1cd479c6d371605e9689c88ae1789dbcbceb2da0 commit: 1cd479c6d371605e9689c88ae1789dbcbceb2da0 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-03T00:00:06Z summary: gh-104683: Rework Argument Clinic error handling (#107551) Introduce ClinicError, and use it in fail(). The CLI runs main(), catches ClinicError, formats the error message, prints to stderr and exits with an error. As a side effect, this refactor greatly improves the accuracy of reported line numbers in case of error. Also, adapt the test suite to work with ClinicError. Co-authored-by: Alex Waygood files: M Lib/test/test_clinic.py M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index cdabcbaa6f03c..127008d443e4c 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -11,6 +11,7 @@ import contextlib import inspect import os.path +import re import sys import unittest @@ -20,17 +21,24 @@ from clinic import DSLParser -class _ParserBase(TestCase): - maxDiff = None - - def expect_parser_failure(self, parser, _input): - with support.captured_stdout() as stdout: - with self.assertRaises(SystemExit): - parser(_input) - return stdout.getvalue() +def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None): + """Helper for the parser tests. - def parse_function_should_fail(self, _input): - return self.expect_parser_failure(self.parse_function, _input) + tc: unittest.TestCase; passed self in the wrapper + parser: the clinic parser used for this test case + code: a str with input text (clinic code) + errmsg: the expected error message + filename: str, optional filename + lineno: int, optional line number + """ + code = dedent(code).strip() + errmsg = re.escape(errmsg) + with tc.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: + parser(code) + if filename is not None: + tc.assertEqual(cm.exception.filename, filename) + if lineno is not None: + tc.assertEqual(cm.exception.lineno, lineno) class FakeConverter: @@ -108,14 +116,15 @@ def __repr__(self): return "" -class ClinicWholeFileTest(_ParserBase): +class ClinicWholeFileTest(TestCase): + + def expect_failure(self, raw, errmsg, *, filename=None, lineno=None): + _expect_failure(self, self.clinic.parse, raw, errmsg, + filename=filename, lineno=lineno) + def setUp(self): self.clinic = clinic.Clinic(clinic.CLanguage(None), filename="test.c") - def expect_failure(self, raw): - _input = dedent(raw).strip() - return self.expect_parser_failure(self.clinic.parse, _input) - def test_eol(self): # regression test: # clinic's block parser didn't recognize @@ -139,12 +148,11 @@ def test_mangled_marker_line(self): [clinic start generated code]*/ /*[clinic end generated code: foo]*/ """ - msg = ( - 'Error in file "test.c" on line 3:\n' - "Mangled Argument Clinic marker line: '/*[clinic end generated code: foo]*/'\n" + err = ( + "Mangled Argument Clinic marker line: " + "'/*[clinic end generated code: foo]*/'" ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + self.expect_failure(raw, err, filename="test.c", lineno=3) def test_checksum_mismatch(self): raw = """ @@ -152,38 +160,31 @@ def test_checksum_mismatch(self): [clinic start generated code]*/ /*[clinic end generated code: output=0123456789abcdef input=fedcba9876543210]*/ """ - msg = ( - 'Error in file "test.c" on line 3:\n' + err = ( 'Checksum mismatch!\n' 'Expected: 0123456789abcdef\n' 'Computed: da39a3ee5e6b4b0d\n' ) - out = self.expect_failure(raw) - self.assertIn(msg, out) + self.expect_failure(raw, err, filename="test.c", lineno=3) def test_garbage_after_stop_line(self): raw = """ /*[clinic input] [clinic start generated code]*/foobarfoobar! """ - msg = ( - 'Error in file "test.c" on line 2:\n' - "Garbage after stop line: 'foobarfoobar!'\n" - ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + err = "Garbage after stop line: 'foobarfoobar!'" + self.expect_failure(raw, err, filename="test.c", lineno=2) def test_whitespace_before_stop_line(self): raw = """ /*[clinic input] [clinic start generated code]*/ """ - msg = ( - 'Error in file "test.c" on line 2:\n' - "Whitespace is not allowed before the stop line: ' [clinic start generated code]*/'\n" + err = ( + "Whitespace is not allowed before the stop line: " + "' [clinic start generated code]*/'" ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + self.expect_failure(raw, err, filename="test.c", lineno=2) def test_parse_with_body_prefix(self): clang = clinic.CLanguage(None) @@ -213,12 +214,8 @@ def test_cpp_monitor_fail_nested_block_comment(self): */ */ """ - msg = ( - 'Error in file "test.c" on line 2:\n' - 'Nested block comment!\n' - ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + err = 'Nested block comment!' + self.expect_failure(raw, err, filename="test.c", lineno=2) def test_cpp_monitor_fail_invalid_format_noarg(self): raw = """ @@ -226,12 +223,8 @@ def test_cpp_monitor_fail_invalid_format_noarg(self): a() #endif """ - msg = ( - 'Error in file "test.c" on line 1:\n' - 'Invalid format for #if line: no argument!\n' - ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + err = 'Invalid format for #if line: no argument!' + self.expect_failure(raw, err, filename="test.c", lineno=1) def test_cpp_monitor_fail_invalid_format_toomanyargs(self): raw = """ @@ -239,39 +232,31 @@ def test_cpp_monitor_fail_invalid_format_toomanyargs(self): a() #endif """ - msg = ( - 'Error in file "test.c" on line 1:\n' - 'Invalid format for #ifdef line: should be exactly one argument!\n' - ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + err = 'Invalid format for #ifdef line: should be exactly one argument!' + self.expect_failure(raw, err, filename="test.c", lineno=1) def test_cpp_monitor_fail_no_matching_if(self): raw = '#else' - msg = ( - 'Error in file "test.c" on line 1:\n' - '#else without matching #if / #ifdef / #ifndef!\n' - ) - out = self.expect_failure(raw) - self.assertEqual(out, msg) + err = '#else without matching #if / #ifdef / #ifndef!' + self.expect_failure(raw, err, filename="test.c", lineno=1) def test_directive_output_unknown_preset(self): - out = self.expect_failure(""" + raw = """ /*[clinic input] output preset nosuchpreset [clinic start generated code]*/ - """) - msg = "Unknown preset 'nosuchpreset'" - self.assertIn(msg, out) + """ + err = "Unknown preset 'nosuchpreset'" + self.expect_failure(raw, err) def test_directive_output_cant_pop(self): - out = self.expect_failure(""" + raw = """ /*[clinic input] output pop [clinic start generated code]*/ - """) - msg = "Can't 'output pop', stack is empty" - self.assertIn(msg, out) + """ + err = "Can't 'output pop', stack is empty" + self.expect_failure(raw, err) def test_directive_output_print(self): raw = dedent(""" @@ -309,16 +294,16 @@ def test_directive_output_print(self): ) def test_unknown_destination_command(self): - out = self.expect_failure(""" + raw = """ /*[clinic input] destination buffer nosuchcommand [clinic start generated code]*/ - """) - msg = "unknown destination command 'nosuchcommand'" - self.assertIn(msg, out) + """ + err = "unknown destination command 'nosuchcommand'" + self.expect_failure(raw, err) def test_no_access_to_members_in_converter_init(self): - out = self.expect_failure(""" + raw = """ /*[python input] class Custom_converter(CConverter): converter = "some_c_function" @@ -330,11 +315,11 @@ def converter_init(self): test.fn a: Custom [clinic start generated code]*/ - """) - msg = ( + """ + err = ( "accessing self.function inside converter_init is disallowed!" ) - self.assertIn(msg, out) + self.expect_failure(raw, err) @staticmethod @contextlib.contextmanager @@ -375,30 +360,30 @@ def test_version_directive_insufficient_version(self): " Version: 4\n" " Required: 5" ) - out = self.expect_failure(""" + block = """ /*[clinic input] version 5 [clinic start generated code]*/ - """) - self.assertIn(err, out) + """ + self.expect_failure(block, err) def test_version_directive_illegal_char(self): err = "Illegal character 'v' in version string 'v5'" - out = self.expect_failure(""" + block = """ /*[clinic input] version v5 [clinic start generated code]*/ - """) - self.assertIn(err, out) + """ + self.expect_failure(block, err) def test_version_directive_unsupported_string(self): err = "Unsupported version string: '.-'" - out = self.expect_failure(""" + block = """ /*[clinic input] version .- [clinic start generated code]*/ - """) - self.assertIn(err, out) + """ + self.expect_failure(block, err) class ClinicGroupPermuterTest(TestCase): @@ -577,7 +562,7 @@ def test_clinic_1(self): """) -class ClinicParserTest(_ParserBase): +class ClinicParserTest(TestCase): def parse(self, text): c = FakeClinic() @@ -594,6 +579,10 @@ def parse_function(self, text, signatures_in_block=2, function_index=1): assert isinstance(s[function_index], clinic.Function) return s[function_index] + def expect_failure(self, block, err, *, filename=None, lineno=None): + _expect_failure(self, self.parse_function, block, err, + filename=filename, lineno=lineno) + def checkDocstring(self, fn, expected): self.assertTrue(hasattr(fn, "docstring")) self.assertEqual(fn.docstring.strip(), @@ -663,17 +652,16 @@ def test_param_default_expression(self): self.assertEqual(sys.maxsize, p.default) self.assertEqual("MAXSIZE", p.converter.c_default) - expected_msg = ( - "Error on line 0:\n" + err = ( "When you specify a named constant ('sys.maxsize') as your default value,\n" - "you MUST specify a valid c_default.\n" + "you MUST specify a valid c_default." ) - out = self.parse_function_should_fail(""" + block = """ module os os.access follow_symlinks: int = sys.maxsize - """) - self.assertEqual(out, expected_msg) + """ + self.expect_failure(block, err, lineno=2) def test_param_no_docstring(self): function = self.parse_function(""" @@ -688,17 +676,17 @@ def test_param_no_docstring(self): self.assertIsInstance(conv, clinic.str_converter) def test_param_default_parameters_out_of_order(self): - expected_msg = ( - "Error on line 0:\n" + err = ( "Can't have a parameter without a default ('something_else')\n" - "after a parameter with a default!\n" + "after a parameter with a default!" ) - out = self.parse_function_should_fail(""" + block = """ module os os.access follow_symlinks: bool = True - something_else: str""") - self.assertEqual(out, expected_msg) + something_else: str + """ + self.expect_failure(block, err, lineno=3) def disabled_test_converter_arguments(self): function = self.parse_function(""" @@ -797,17 +785,19 @@ def test_c_name(self): self.assertEqual("os_stat_fn", function.c_basename) def test_cloning_nonexistent_function_correctly_fails(self): - stdout = self.parse_function_should_fail(""" - cloned = fooooooooooooooooooooooo + block = """ + cloned = fooooooooooooooooo This is trying to clone a nonexistent function!! + """ + err = "Couldn't find existing function 'fooooooooooooooooo'!" + with support.captured_stderr() as stderr: + self.expect_failure(block, err, lineno=0) + expected_debug_print = dedent("""\ + cls=None, module=, existing='fooooooooooooooooo' + (cls or module).functions=[] """) - expected_error = """\ -cls=None, module=, existing='fooooooooooooooooooooooo' -(cls or module).functions=[] -Error on line 0: -Couldn't find existing function 'fooooooooooooooooooooooo'! -""" - self.assertEqual(expected_error, stdout) + stderr = stderr.getvalue() + self.assertIn(expected_debug_print, stderr) def test_return_converter(self): function = self.parse_function(""" @@ -817,30 +807,28 @@ def test_return_converter(self): self.assertIsInstance(function.return_converter, clinic.int_return_converter) def test_return_converter_invalid_syntax(self): - stdout = self.parse_function_should_fail(""" + block = """ module os os.stat -> invalid syntax - """) - expected_error = "Badly formed annotation for os.stat: 'invalid syntax'" - self.assertIn(expected_error, stdout) + """ + err = "Badly formed annotation for os.stat: 'invalid syntax'" + self.expect_failure(block, err) def test_legacy_converter_disallowed_in_return_annotation(self): - stdout = self.parse_function_should_fail(""" + block = """ module os os.stat -> "s" - """) - expected_error = "Legacy converter 's' not allowed as a return converter" - self.assertIn(expected_error, stdout) + """ + err = "Legacy converter 's' not allowed as a return converter" + self.expect_failure(block, err) def test_unknown_return_converter(self): - stdout = self.parse_function_should_fail(""" + block = """ module os - os.stat -> foooooooooooooooooooooooo - """) - expected_error = ( - "No available return converter called 'foooooooooooooooooooooooo'" - ) - self.assertIn(expected_error, stdout) + os.stat -> fooooooooooooooooooooooo + """ + err = "No available return converter called 'fooooooooooooooooooooooo'" + self.expect_failure(block, err) def test_star(self): function = self.parse_function(""" @@ -985,19 +973,12 @@ def test_nested_groups(self): Attributes for the character. """) - def parse_function_should_fail(self, s): - with support.captured_stdout() as stdout: - with self.assertRaises(SystemExit): - self.parse_function(s) - return stdout.getvalue() - def test_disallowed_grouping__two_top_groups_on_left(self): - expected_msg = ( - 'Error on line 0:\n' + err = ( 'Function two_top_groups_on_left has an unsupported group ' - 'configuration. (Unexpected state 2.b)\n' + 'configuration. (Unexpected state 2.b)' ) - out = self.parse_function_should_fail(""" + block = """ module foo foo.two_top_groups_on_left [ @@ -1007,11 +988,11 @@ def test_disallowed_grouping__two_top_groups_on_left(self): group2 : int ] param: int - """) - self.assertEqual(out, expected_msg) + """ + self.expect_failure(block, err, lineno=5) def test_disallowed_grouping__two_top_groups_on_right(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.two_top_groups_on_right param: int @@ -1021,15 +1002,15 @@ def test_disallowed_grouping__two_top_groups_on_right(self): [ group2 : int ] - """) - msg = ( + """ + err = ( "Function two_top_groups_on_right has an unsupported group " "configuration. (Unexpected state 6.b)" ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_disallowed_grouping__parameter_after_group_on_right(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.parameter_after_group_on_right param: int @@ -1039,15 +1020,15 @@ def test_disallowed_grouping__parameter_after_group_on_right(self): ] group2 : int ] - """) - msg = ( + """ + err = ( "Function parameter_after_group_on_right has an unsupported group " "configuration. (Unexpected state 6.a)" ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_disallowed_grouping__group_after_parameter_on_left(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.group_after_parameter_on_left [ @@ -1057,15 +1038,15 @@ def test_disallowed_grouping__group_after_parameter_on_left(self): ] ] param: int - """) - msg = ( + """ + err = ( "Function group_after_parameter_on_left has an unsupported group " "configuration. (Unexpected state 2.b)" ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_disallowed_grouping__empty_group_on_left(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.empty_group [ @@ -1074,15 +1055,15 @@ def test_disallowed_grouping__empty_group_on_left(self): group2 : int ] param: int - """) - msg = ( + """ + err = ( "Function empty_group has an empty group.\n" "All groups must contain at least one parameter." ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_disallowed_grouping__empty_group_on_right(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.empty_group param: int @@ -1091,24 +1072,24 @@ def test_disallowed_grouping__empty_group_on_right(self): ] group2 : int ] - """) - msg = ( + """ + err = ( "Function empty_group has an empty group.\n" "All groups must contain at least one parameter." ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_disallowed_grouping__no_matching_bracket(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.empty_group param: int ] group2: int ] - """) - msg = "Function empty_group has a ] without a matching [." - self.assertIn(msg, out) + """ + err = "Function empty_group has a ] without a matching [." + self.expect_failure(block, err) def test_no_parameters(self): function = self.parse_function(""" @@ -1137,31 +1118,32 @@ class foo.Bar "unused" "notneeded" self.assertEqual(1, len(function.parameters)) def test_illegal_module_line(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar => int / - """) - msg = "Illegal function name: foo.bar => int" - self.assertIn(msg, out) + """ + err = "Illegal function name: foo.bar => int" + self.expect_failure(block, err) def test_illegal_c_basename(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar as 935 / - """) - msg = "Illegal C basename: 935" - self.assertIn(msg, out) + """ + err = "Illegal C basename: 935" + self.expect_failure(block, err) def test_single_star(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar * * - """) - self.assertIn("Function bar uses '*' more than once.", out) + """ + err = "Function bar uses '*' more than once." + self.expect_failure(block, err) def test_parameters_required_after_star(self): dataset = ( @@ -1170,39 +1152,38 @@ def test_parameters_required_after_star(self): "module foo\nfoo.bar\n this: int\n *", "module foo\nfoo.bar\n this: int\n *\nDocstring.", ) - msg = "Function bar specifies '*' without any parameters afterwards." + err = "Function bar specifies '*' without any parameters afterwards." for block in dataset: with self.subTest(block=block): - out = self.parse_function_should_fail(block) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_single_slash(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar / / - """) - msg = ( + """ + err = ( "Function bar has an unsupported group configuration. " "(Unexpected state 0.d)" ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_double_slash(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar a: int / b: int / - """) - msg = "Function bar uses '/' more than once." - self.assertIn(msg, out) + """ + err = "Function bar uses '/' more than once." + self.expect_failure(block, err) def test_mix_star_and_slash(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar x: int @@ -1210,38 +1191,35 @@ def test_mix_star_and_slash(self): * z: int / - """) - msg = ( + """ + err = ( "Function bar mixes keyword-only and positional-only parameters, " "which is unsupported." ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_parameters_not_permitted_after_slash_for_now(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar / x: int - """) - msg = ( + """ + err = ( "Function bar has an unsupported group configuration. " "(Unexpected state 0.d)" ) - self.assertIn(msg, out) + self.expect_failure(block, err) def test_parameters_no_more_than_one_vararg(self): - expected_msg = ( - "Error on line 0:\n" - "Too many var args\n" - ) - out = self.parse_function_should_fail(""" + err = "Too many var args" + block = """ module foo foo.bar *vararg1: object *vararg2: object - """) - self.assertEqual(out, expected_msg) + """ + self.expect_failure(block, err, lineno=0) def test_function_not_at_column_0(self): function = self.parse_function(""" @@ -1264,23 +1242,24 @@ def test_function_not_at_column_0(self): """) def test_indent_stack_no_tabs(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar *vararg1: object \t*vararg2: object - """) - msg = "Tab characters are illegal in the Clinic DSL." - self.assertIn(msg, out) + """ + err = "Tab characters are illegal in the Clinic DSL." + self.expect_failure(block, err) def test_indent_stack_illegal_outdent(self): - out = self.parse_function_should_fail(""" + block = """ module foo foo.bar a: object b: object - """) - self.assertIn("Illegal outdent", out) + """ + err = "Illegal outdent" + self.expect_failure(block, err) def test_directive(self): c = FakeClinic() @@ -1298,10 +1277,7 @@ def test_legacy_converters(self): self.assertIsInstance(conv, clinic.str_converter) def test_legacy_converters_non_string_constant_annotation(self): - expected_failure_message = ( - "Error on line 0:\n" - "Annotations must be either a name, a function call, or a string.\n" - ) + err = "Annotations must be either a name, a function call, or a string" dataset = ( 'module os\nos.access\n path: 42', 'module os\nos.access\n path: 42.42', @@ -1310,14 +1286,10 @@ def test_legacy_converters_non_string_constant_annotation(self): ) for block in dataset: with self.subTest(block=block): - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_failure_message) + self.expect_failure(block, err, lineno=2) def test_other_bizarre_things_in_annotations_fail(self): - expected_failure_message = ( - "Error on line 0:\n" - "Annotations must be either a name, a function call, or a string.\n" - ) + err = "Annotations must be either a name, a function call, or a string" dataset = ( 'module os\nos.access\n path: {"some": "dictionary"}', 'module os\nos.access\n path: ["list", "of", "strings"]', @@ -1325,30 +1297,24 @@ def test_other_bizarre_things_in_annotations_fail(self): ) for block in dataset: with self.subTest(block=block): - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_failure_message) + self.expect_failure(block, err, lineno=2) def test_kwarg_splats_disallowed_in_function_call_annotations(self): - expected_error_msg = ( - "Error on line 0:\n" - "Cannot use a kwarg splat in a function-call annotation\n" - ) + err = "Cannot use a kwarg splat in a function-call annotation" dataset = ( 'module fo\nfo.barbaz\n o: bool(**{None: "bang!"})', 'module fo\nfo.barbaz -> bool(**{None: "bang!"})', 'module fo\nfo.barbaz -> bool(**{"bang": 42})', 'module fo\nfo.barbaz\n o: bool(**{"bang": None})', ) - for fn in dataset: - with self.subTest(fn=fn): - out = self.parse_function_should_fail(fn) - self.assertEqual(out, expected_error_msg) + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err) def test_self_param_placement(self): - expected_error_msg = ( - "Error on line 0:\n" + err = ( "A 'self' parameter, if specified, must be the very first thing " - "in the parameter block.\n" + "in the parameter block." ) block = """ module foo @@ -1356,27 +1322,21 @@ def test_self_param_placement(self): a: int self: self(type="PyObject *") """ - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_error_msg) + self.expect_failure(block, err, lineno=3) def test_self_param_cannot_be_optional(self): - expected_error_msg = ( - "Error on line 0:\n" - "A 'self' parameter cannot be marked optional.\n" - ) + err = "A 'self' parameter cannot be marked optional." block = """ module foo foo.func self: self(type="PyObject *") = None """ - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_error_msg) + self.expect_failure(block, err, lineno=2) def test_defining_class_param_placement(self): - expected_error_msg = ( - "Error on line 0:\n" + err = ( "A 'defining_class' parameter, if specified, must either be the " - "first thing in the parameter block, or come just after 'self'.\n" + "first thing in the parameter block, or come just after 'self'." ) block = """ module foo @@ -1385,21 +1345,16 @@ def test_defining_class_param_placement(self): a: int cls: defining_class """ - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_error_msg) + self.expect_failure(block, err, lineno=4) def test_defining_class_param_cannot_be_optional(self): - expected_error_msg = ( - "Error on line 0:\n" - "A 'defining_class' parameter cannot be marked optional.\n" - ) + err = "A 'defining_class' parameter cannot be marked optional." block = """ module foo foo.func cls: defining_class(type="PyObject *") = None """ - out = self.parse_function_should_fail(block) - self.assertEqual(out, expected_error_msg) + self.expect_failure(block, err, lineno=2) def test_slot_methods_cannot_access_defining_class(self): block = """ @@ -1409,34 +1364,28 @@ class Foo "" "" cls: defining_class a: object """ - msg = "Slot methods cannot access their defining class." - with self.assertRaisesRegex(ValueError, msg): + err = "Slot methods cannot access their defining class." + with self.assertRaisesRegex(ValueError, err): self.parse_function(block) def test_new_must_be_a_class_method(self): - expected_error_msg = ( - "Error on line 0:\n" - "__new__ must be a class method!\n" - ) - out = self.parse_function_should_fail(""" + err = "__new__ must be a class method!" + block = """ module foo class Foo "" "" Foo.__new__ - """) - self.assertEqual(out, expected_error_msg) + """ + self.expect_failure(block, err, lineno=2) def test_init_must_be_a_normal_method(self): - expected_error_msg = ( - "Error on line 0:\n" - "__init__ must be a normal method, not a class or static method!\n" - ) - out = self.parse_function_should_fail(""" + err = "__init__ must be a normal method, not a class or static method!" + block = """ module foo class Foo "" "" @classmethod Foo.__init__ - """) - self.assertEqual(out, expected_error_msg) + """ + self.expect_failure(block, err, lineno=3) def test_unused_param(self): block = self.parse(""" @@ -1487,11 +1436,12 @@ def test_scaffolding(self): 'The igloos are melting!\n' ) with support.captured_stdout() as stdout: - with self.assertRaises(SystemExit): - clinic.fail('The igloos are melting!', - filename='clown.txt', line_number=69) - actual = stdout.getvalue() - self.assertEqual(actual, expected) + errmsg = 'The igloos are melting' + with self.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: + clinic.fail(errmsg, filename='clown.txt', line_number=69) + exc = cm.exception + self.assertEqual(exc.filename, 'clown.txt') + self.assertEqual(exc.lineno, 69) def test_non_ascii_character_in_docstring(self): block = """ @@ -1507,19 +1457,21 @@ def test_non_ascii_character_in_docstring(self): expected = dedent("""\ Warning on line 0: Non-ascii characters are not allowed in docstrings: '?' + Warning on line 0: Non-ascii characters are not allowed in docstrings: '?', '?', '?' + """) self.assertEqual(stdout.getvalue(), expected) def test_illegal_c_identifier(self): err = "Illegal C identifier: 17a" - out = self.parse_function_should_fail(""" + block = """ module test test.fn a as 17a: int - """) - self.assertIn(err, out) + """ + self.expect_failure(block, err) class ClinicExternalTest(TestCase): @@ -1607,9 +1559,9 @@ def test_cli_force(self): # First, run the CLI without -f and expect failure. # Note, we cannot check the entire fail msg, because the path to # the tmp file will change for every run. - out, _ = self.expect_failure(fn) - self.assertTrue(out.endswith(fail_msg), - f"{out!r} does not end with {fail_msg!r}") + _, err = self.expect_failure(fn) + self.assertTrue(err.endswith(fail_msg), + f"{err!r} does not end with {fail_msg!r}") # Then, force regeneration; success expected. out = self.expect_success("-f", fn) self.assertEqual(out, "") @@ -2231,8 +2183,11 @@ def test_gh_99233_refcount(self): self.assertEqual(arg_refcount_origin, arg_refcount_after) def test_gh_99240_double_free(self): - expected_error = r'gh_99240_double_free\(\) argument 2 must be encoded string without null bytes, not str' - with self.assertRaisesRegex(TypeError, expected_error): + err = re.escape( + "gh_99240_double_free() argument 2 must be encoded string " + "without null bytes, not str" + ) + with self.assertRaisesRegex(TypeError, err): ac_tester.gh_99240_double_free('a', '\0b') def test_cloned_func_exception_message(self): diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index ce8184753c729..1bcdb6b1c3640 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -28,7 +28,6 @@ import string import sys import textwrap -import traceback from collections.abc import ( Callable, @@ -137,6 +136,28 @@ def text_accumulator() -> TextAccumulator: text, append, output = _text_accumulator() return TextAccumulator(append, output) + + at dc.dataclass +class ClinicError(Exception): + message: str + _: dc.KW_ONLY + lineno: int | None = None + filename: str | None = None + + def __post_init__(self) -> None: + super().__init__(self.message) + + def report(self, *, warn_only: bool = False) -> str: + msg = "Warning" if warn_only else "Error" + if self.filename is not None: + msg += f" in file {self.filename!r}" + if self.lineno is not None: + msg += f" on line {self.lineno}" + msg += ":\n" + msg += f"{self.message}\n" + return msg + + @overload def warn_or_fail( *args: object, @@ -160,25 +181,16 @@ def warn_or_fail( line_number: int | None = None, ) -> None: joined = " ".join([str(a) for a in args]) - add, output = text_accumulator() - if fail: - add("Error") - else: - add("Warning") if clinic: if filename is None: filename = clinic.filename if getattr(clinic, 'block_parser', None) and (line_number is None): line_number = clinic.block_parser.line_number - if filename is not None: - add(' in file "' + filename + '"') - if line_number is not None: - add(" on line " + str(line_number)) - add(':\n') - add(joined) - print(output()) + error = ClinicError(joined, filename=filename, lineno=line_number) if fail: - sys.exit(-1) + raise error + else: + print(error.report(warn_only=True)) def warn( @@ -347,7 +359,7 @@ def version_splitter(s: str) -> tuple[int, ...]: accumulator: list[str] = [] def flush() -> None: if not accumulator: - raise ValueError('Unsupported version string: ' + repr(s)) + fail(f'Unsupported version string: {s!r}') version.append(int(''.join(accumulator))) accumulator.clear() @@ -360,7 +372,7 @@ def flush() -> None: flush() version.append('abc'.index(c) - 3) else: - raise ValueError('Illegal character ' + repr(c) + ' in version string ' + repr(s)) + fail(f'Illegal character {c!r} in version string {s!r}') flush() return tuple(version) @@ -2233,11 +2245,7 @@ def parse(self, input: str) -> str: assert dsl_name in parsers, f"No parser to handle {dsl_name!r} block." self.parsers[dsl_name] = parsers[dsl_name](self) parser = self.parsers[dsl_name] - try: - parser.parse(block) - except Exception: - fail('Exception raised during parsing:\n' + - traceback.format_exc().rstrip()) + parser.parse(block) printer.print_block(block) # these are destinations not buffers @@ -4600,7 +4608,11 @@ def parse(self, block: Block) -> None: for line_number, line in enumerate(lines, self.clinic.block_parser.block_start_line_number): if '\t' in line: fail('Tab characters are illegal in the Clinic DSL.\n\t' + repr(line), line_number=block_start) - self.state(line) + try: + self.state(line) + except ClinicError as exc: + exc.lineno = line_number + raise self.do_post_block_processing_cleanup() block.output.extend(self.clinic.language.render(self.clinic, block.signatures)) @@ -4701,8 +4713,8 @@ def state_modulename_name(self, line: str) -> None: if existing_function.name == function_name: break else: - print(f"{cls=}, {module=}, {existing=}") - print(f"{(cls or module).functions=}") + print(f"{cls=}, {module=}, {existing=}", file=sys.stderr) + print(f"{(cls or module).functions=}", file=sys.stderr) fail(f"Couldn't find existing function {existing!r}!") fields = [x.strip() for x in full_name.split('.')] @@ -5719,8 +5731,13 @@ def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: def main(argv: list[str] | None = None) -> NoReturn: parser = create_cli() args = parser.parse_args(argv) - run_clinic(parser, args) - sys.exit(0) + try: + run_clinic(parser, args) + except ClinicError as exc: + sys.stderr.write(exc.report()) + sys.exit(1) + else: + sys.exit(0) if __name__ == "__main__": From webhook-mailer at python.org Thu Aug 3 01:29:06 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 03 Aug 2023 05:29:06 -0000 Subject: [Python-checkins] gh-107455: ctypes: Improve error messages when converting to an incompatible type (#107456) Message-ID: https://github.com/python/cpython/commit/62a3a15119cf16d216a2cc7dc10d97143017ef62 commit: 62a3a15119cf16d216a2cc7dc10d97143017ef62 branch: main author: Tomas R committer: kumaraditya303 date: 2023-08-03T10:59:03+05:30 summary: gh-107455: ctypes: Improve error messages when converting to an incompatible type (#107456) files: A Misc/NEWS.d/next/Core and Builtins/2023-07-30-14-18-49.gh-issue-107455.Es53l7.rst M Lib/test/test_ctypes/test_functions.py M Modules/_ctypes/_ctypes.c diff --git a/Lib/test/test_ctypes/test_functions.py b/Lib/test/test_ctypes/test_functions.py index 9cf680f16620a..08eecbc9ea444 100644 --- a/Lib/test/test_ctypes/test_functions.py +++ b/Lib/test/test_ctypes/test_functions.py @@ -4,8 +4,8 @@ import unittest from ctypes import (CDLL, Structure, Array, CFUNCTYPE, byref, POINTER, pointer, ArgumentError, - c_char, c_wchar, c_byte, c_char_p, - c_short, c_int, c_long, c_longlong, + c_char, c_wchar, c_byte, c_char_p, c_wchar_p, + c_short, c_int, c_long, c_longlong, c_void_p, c_float, c_double, c_longdouble) from _ctypes import _Pointer, _SimpleCData @@ -92,6 +92,54 @@ def test_wchar_parm(self): "argument 2: TypeError: one character unicode string " "expected") + def test_c_char_p_parm(self): + """Test the error message when converting an incompatible type to c_char_p.""" + proto = CFUNCTYPE(c_int, c_char_p) + def callback(*args): + return 0 + + callback = proto(callback) + self.assertEqual(callback(b"abc"), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(10) + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: 'int' object cannot be " + "interpreted as ctypes.c_char_p") + + def test_c_wchar_p_parm(self): + """Test the error message when converting an incompatible type to c_wchar_p.""" + proto = CFUNCTYPE(c_int, c_wchar_p) + def callback(*args): + return 0 + + callback = proto(callback) + self.assertEqual(callback("abc"), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(10) + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: 'int' object cannot be " + "interpreted as ctypes.c_wchar_p") + + def test_c_void_p_parm(self): + """Test the error message when converting an incompatible type to c_void_p.""" + proto = CFUNCTYPE(c_int, c_void_p) + def callback(*args): + return 0 + + callback = proto(callback) + self.assertEqual(callback(5), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(2.5) + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: 'float' object cannot be " + "interpreted as ctypes.c_void_p") + def test_wchar_result(self): f = dll._testfunc_i_bhilfd f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-30-14-18-49.gh-issue-107455.Es53l7.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-14-18-49.gh-issue-107455.Es53l7.rst new file mode 100644 index 0000000000000..84a93251e799d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-14-18-49.gh-issue-107455.Es53l7.rst @@ -0,0 +1,3 @@ +Improve error messages when converting an incompatible type to +:class:`ctypes.c_char_p`, :class:`ctypes.c_wchar_p` and +:class:`ctypes.c_void_p`. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 8d4fd30150f19..9aee37a9d954e 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1728,9 +1728,9 @@ c_wchar_p_from_param(PyObject *type, PyObject *value) Py_DECREF(as_parameter); return value; } - /* XXX better message */ - PyErr_SetString(PyExc_TypeError, - "wrong type"); + PyErr_Format(PyExc_TypeError, + "'%.200s' object cannot be interpreted " + "as ctypes.c_wchar_p", Py_TYPE(value)->tp_name); return NULL; } @@ -1792,9 +1792,9 @@ c_char_p_from_param(PyObject *type, PyObject *value) Py_DECREF(as_parameter); return value; } - /* XXX better message */ - PyErr_SetString(PyExc_TypeError, - "wrong type"); + PyErr_Format(PyExc_TypeError, + "'%.200s' object cannot be interpreted " + "as ctypes.c_char_p", Py_TYPE(value)->tp_name); return NULL; } @@ -1927,9 +1927,9 @@ c_void_p_from_param(PyObject *type, PyObject *value) Py_DECREF(as_parameter); return value; } - /* XXX better message */ - PyErr_SetString(PyExc_TypeError, - "wrong type"); + PyErr_Format(PyExc_TypeError, + "'%.200s' object cannot be interpreted " + "as ctypes.c_void_p", Py_TYPE(value)->tp_name); return NULL; } From webhook-mailer at python.org Thu Aug 3 02:36:06 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 03 Aug 2023 06:36:06 -0000 Subject: [Python-checkins] GH-107458: fix test_tools refleak (#107577) Message-ID: https://github.com/python/cpython/commit/46366ca0486d07fe94c70d00771482c8ef1546fc commit: 46366ca0486d07fe94c70d00771482c8ef1546fc branch: main author: Kumar Aditya committer: kumaraditya303 date: 2023-08-03T06:36:02Z summary: GH-107458: fix test_tools refleak (#107577) files: M Lib/test/test_tools/test_sundry.py diff --git a/Lib/test/test_tools/test_sundry.py b/Lib/test/test_tools/test_sundry.py index 2f8ba272164d3..d0b702d392cdf 100644 --- a/Lib/test/test_tools/test_sundry.py +++ b/Lib/test/test_tools/test_sundry.py @@ -19,17 +19,11 @@ class TestSundryScripts(unittest.TestCase): # cleanly the logging module. @import_helper.mock_register_at_fork def test_sundry(self, mock_os): - old_modules = import_helper.modules_setup() - try: - for fn in os.listdir(scriptsdir): - if not fn.endswith('.py'): - continue - - name = fn[:-3] - import_tool(name) - finally: - # Unload all modules loaded in this test - import_helper.modules_cleanup(*old_modules) + for fn in os.listdir(scriptsdir): + if not fn.endswith('.py'): + continue + name = fn[:-3] + import_tool(name) if __name__ == '__main__': From webhook-mailer at python.org Thu Aug 3 05:35:30 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Thu, 03 Aug 2023 09:35:30 -0000 Subject: [Python-checkins] gh-106368: Increase Argument Clinic test coverage (#107582) Message-ID: https://github.com/python/cpython/commit/a73daf54ebd7bd6bf32e82766a605ebead2f128c commit: a73daf54ebd7bd6bf32e82766a605ebead2f128c branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-03T09:35:26Z summary: gh-106368: Increase Argument Clinic test coverage (#107582) Add tests for DSL parser state machine and docstring formatting files: M Lib/test/clinic.test.c M Lib/test/test_clinic.py diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index a49c2e7781064..d2ad1a0482c30 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5264,3 +5264,203 @@ Test__pyarg_parsestackandkeywords_impl(TestObj *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length) /*[clinic end generated code: output=4fda8a7f2547137c input=fc72ef4b4cfafabc]*/ + + +/*[clinic input] +Test.__init__ -> long +Test overriding the __init__ return converter +[clinic start generated code]*/ + +PyDoc_STRVAR(Test___init____doc__, +"Test()\n" +"--\n" +"\n" +"Test overriding the __init__ return converter"); + +static long +Test___init___impl(TestObj *self); + +static int +Test___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = TestType; + long _return_value; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoPositional("Test", args)) { + goto exit; + } + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Test", kwargs)) { + goto exit; + } + _return_value = Test___init___impl((TestObj *)self); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong(_return_value); + +exit: + return return_value; +} + +static long +Test___init___impl(TestObj *self) +/*[clinic end generated code: output=daf6ee12c4e443fb input=311af0dc7f17e8e9]*/ + + +/*[clinic input] +fn_with_default_binop_expr + arg: object(c_default='CONST_A + CONST_B') = a+b +[clinic start generated code]*/ + +PyDoc_STRVAR(fn_with_default_binop_expr__doc__, +"fn_with_default_binop_expr($module, /, arg=a+b)\n" +"--\n" +"\n"); + +#define FN_WITH_DEFAULT_BINOP_EXPR_METHODDEF \ + {"fn_with_default_binop_expr", _PyCFunction_CAST(fn_with_default_binop_expr), METH_FASTCALL|METH_KEYWORDS, fn_with_default_binop_expr__doc__}, + +static PyObject * +fn_with_default_binop_expr_impl(PyObject *module, PyObject *arg); + +static PyObject * +fn_with_default_binop_expr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(arg), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"arg", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "fn_with_default_binop_expr", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *arg = CONST_A + CONST_B; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + arg = args[0]; +skip_optional_pos: + return_value = fn_with_default_binop_expr_impl(module, arg); + +exit: + return return_value; +} + +static PyObject * +fn_with_default_binop_expr_impl(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=018672772e4092ff input=1b55c8ae68d89453]*/ + +/*[python input] +class Custom_converter(CConverter): + type = "str" + default = "Hello!" + converter = "c_converter_func" +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=d612708f0efb8e3c]*/ + +/*[clinic input] +docstr_fallback_to_converter_default + a: Custom +Check docstring default value fallback. + +Verify that the docstring formatter fetches the default +value from the converter if no 'py_default' is found. +The signature should have the default a='Hello!', +as given by the Custom converter. +[clinic start generated code]*/ + +PyDoc_STRVAR(docstr_fallback_to_converter_default__doc__, +"docstr_fallback_to_converter_default($module, /, a=\'Hello!\')\n" +"--\n" +"\n" +"Check docstring default value fallback.\n" +"\n" +"Verify that the docstring formatter fetches the default\n" +"value from the converter if no \'py_default\' is found.\n" +"The signature should have the default a=\'Hello!\',\n" +"as given by the Custom converter."); + +#define DOCSTR_FALLBACK_TO_CONVERTER_DEFAULT_METHODDEF \ + {"docstr_fallback_to_converter_default", _PyCFunction_CAST(docstr_fallback_to_converter_default), METH_FASTCALL|METH_KEYWORDS, docstr_fallback_to_converter_default__doc__}, + +static PyObject * +docstr_fallback_to_converter_default_impl(PyObject *module, str a); + +static PyObject * +docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "docstr_fallback_to_converter_default", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + str a; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!c_converter_func(args[0], &a)) { + goto exit; + } + return_value = docstr_fallback_to_converter_default_impl(module, a); + +exit: + return return_value; +} + +static PyObject * +docstr_fallback_to_converter_default_impl(PyObject *module, str a) +/*[clinic end generated code: output=ae24a9c6f60ee8a6 input=0cbe6a4d24bc2274]*/ diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 127008d443e4c..2f94566361872 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -385,6 +385,37 @@ def test_version_directive_unsupported_string(self): """ self.expect_failure(block, err) + def test_clone_mismatch(self): + err = "'kind' of function and cloned function don't match!" + block = """ + /*[clinic input] + module m + @classmethod + m.f1 + a: object + [clinic start generated code]*/ + /*[clinic input] + @staticmethod + m.f2 = m.f1 + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=9) + + def test_badly_formed_return_annotation(self): + err = "Badly formed annotation for m.f: 'Custom'" + block = """ + /*[python input] + class Custom_return_converter(CReturnConverter): + def __init__(self): + raise ValueError("abc") + [python start generated code]*/ + /*[clinic input] + module m + m.f -> Custom + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=8) + class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): @@ -642,7 +673,7 @@ def test_param_with_continuations(self): p = function.parameters['follow_symlinks'] self.assertEqual(True, p.default) - def test_param_default_expression(self): + def test_param_default_expr_named_constant(self): function = self.parse_function(""" module os os.access @@ -663,6 +694,17 @@ def test_param_default_expression(self): """ self.expect_failure(block, err, lineno=2) + def test_param_default_expr_binop(self): + err = ( + "When you specify an expression ('a + b') as your default value,\n" + "you MUST specify a valid c_default." + ) + block = """ + fn + follow_symlinks: int = a + b + """ + self.expect_failure(block, err, lineno=1) + def test_param_no_docstring(self): function = self.parse_function(""" module os @@ -1241,6 +1283,63 @@ def test_function_not_at_column_0(self): Nested docstring here, goeth. """) + def test_docstring_only_summary(self): + function = self.parse_function(""" + module m + m.f + summary + """) + self.checkDocstring(function, """ + f($module, /) + -- + + summary + """) + + def test_docstring_empty_lines(self): + function = self.parse_function(""" + module m + m.f + + + """) + self.checkDocstring(function, """ + f($module, /) + -- + """) + + def test_docstring_explicit_params_placement(self): + function = self.parse_function(""" + module m + m.f + a: int + Param docstring for 'a' will be included + b: int + c: int + Param docstring for 'c' will be included + This is the summary line. + + We'll now place the params section here: + {parameters} + And now for something completely different! + (Note the added newline) + """) + self.checkDocstring(function, """ + f($module, /, a, b, c) + -- + + This is the summary line. + + We'll now place the params section here: + a + Param docstring for 'a' will be included + c + Param docstring for 'c' will be included + + And now for something completely different! + (Note the added newline) + """) + def test_indent_stack_no_tabs(self): block = """ module foo @@ -1471,7 +1570,100 @@ def test_illegal_c_identifier(self): test.fn a as 17a: int """ - self.expect_failure(block, err) + self.expect_failure(block, err, lineno=2) + + def test_cannot_convert_special_method(self): + err = "__len__ is a special method and cannot be converted" + block = """ + class T "" "" + T.__len__ + """ + self.expect_failure(block, err, lineno=1) + + def test_cannot_specify_pydefault_without_default(self): + err = "You can't specify py_default without specifying a default value!" + block = """ + fn + a: object(py_default='NULL') + """ + self.expect_failure(block, err, lineno=1) + + def test_vararg_cannot_take_default_value(self): + err = "Vararg can't take a default value!" + block = """ + fn + *args: object = None + """ + self.expect_failure(block, err, lineno=1) + + def test_invalid_legacy_converter(self): + err = "fhi is not a valid legacy converter" + block = """ + fn + a: 'fhi' + """ + self.expect_failure(block, err, lineno=1) + + def test_parent_class_or_module_does_not_exist(self): + err = "Parent class or module z does not exist" + block = """ + module m + z.func + """ + self.expect_failure(block, err, lineno=1) + + def test_duplicate_param_name(self): + err = "You can't have two parameters named 'a'" + block = """ + module m + m.func + a: int + a: float + """ + self.expect_failure(block, err, lineno=3) + + def test_param_requires_custom_c_name(self): + err = "Parameter 'module' requires a custom C name" + block = """ + module m + m.func + module: int + """ + self.expect_failure(block, err, lineno=2) + + def test_state_func_docstring_assert_no_group(self): + err = "Function func has a ] without a matching [." + block = """ + module m + m.func + ] + docstring + """ + self.expect_failure(block, err, lineno=2) + + def test_state_func_docstring_no_summary(self): + err = "Docstring for m.func does not have a summary line!" + block = """ + module m + m.func + docstring1 + docstring2 + """ + self.expect_failure(block, err, lineno=0) + + def test_state_func_docstring_only_one_param_template(self): + err = "You may not specify {parameters} more than once in a docstring!" + block = """ + module m + m.func + docstring summary + + these are the params: + {parameters} + these are the params again: + {parameters} + """ + self.expect_failure(block, err, lineno=0) class ClinicExternalTest(TestCase): From webhook-mailer at python.org Thu Aug 3 08:37:18 2023 From: webhook-mailer at python.org (pablogsal) Date: Thu, 03 Aug 2023 12:37:18 -0000 Subject: [Python-checkins] gh-107077: Raise SSLCertVerificationError even if the error is set via SSL_ERROR_SYSCALL (#107586) Message-ID: https://github.com/python/cpython/commit/77e09192b5f1caf14cd5f92ccb53a4592e83e8bc commit: 77e09192b5f1caf14cd5f92ccb53a4592e83e8bc branch: main author: Pablo Galindo Salgado committer: pablogsal date: 2023-08-03T12:37:14Z summary: gh-107077: Raise SSLCertVerificationError even if the error is set via SSL_ERROR_SYSCALL (#107586) Co-authored-by: T. Wouters files: A Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst M Modules/_ssl.c diff --git a/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst new file mode 100644 index 0000000000000..ecaf437a48e0a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst @@ -0,0 +1,6 @@ +Seems that in some conditions, OpenSSL will return ``SSL_ERROR_SYSCALL`` +instead of ``SSL_ERROR_SSL`` when a certification verification has failed, +but the error parameters will still contain ``ERR_LIB_SSL`` and +``SSL_R_CERTIFICATE_VERIFY_FAILED``. We are now detecting this situation and +raising the appropiate ``ssl.SSLCertVerificationError``. Patch by Pablo +Galindo diff --git a/Modules/_ssl.c b/Modules/_ssl.c index b61d10d9f59fd..c001b875906d4 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -667,6 +667,10 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) errstr = "Some I/O error occurred"; } } else { + if (ERR_GET_LIB(e) == ERR_LIB_SSL && + ERR_GET_REASON(e) == SSL_R_CERTIFICATE_VERIFY_FAILED) { + type = state->PySSLCertVerificationErrorObject; + } p = PY_SSL_ERROR_SYSCALL; } break; From webhook-mailer at python.org Thu Aug 3 10:09:31 2023 From: webhook-mailer at python.org (pablogsal) Date: Thu, 03 Aug 2023 14:09:31 -0000 Subject: [Python-checkins] [3.11] gh-107077: Raise SSLCertVerificationError even if the error is set via SSL_ERROR_SYSCALL (GH-107586) (#107588) Message-ID: https://github.com/python/cpython/commit/4f65f03f2d597881e1a805e91579d10889299a3d commit: 4f65f03f2d597881e1a805e91579d10889299a3d branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: pablogsal date: 2023-08-03T15:09:27+01:00 summary: [3.11] gh-107077: Raise SSLCertVerificationError even if the error is set via SSL_ERROR_SYSCALL (GH-107586) (#107588) Co-authored-by: Pablo Galindo Salgado Co-authored-by: T. Wouters files: A Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst M Modules/_ssl.c diff --git a/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst new file mode 100644 index 0000000000000..ecaf437a48e0a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst @@ -0,0 +1,6 @@ +Seems that in some conditions, OpenSSL will return ``SSL_ERROR_SYSCALL`` +instead of ``SSL_ERROR_SSL`` when a certification verification has failed, +but the error parameters will still contain ``ERR_LIB_SSL`` and +``SSL_R_CERTIFICATE_VERIFY_FAILED``. We are now detecting this situation and +raising the appropiate ``ssl.SSLCertVerificationError``. Patch by Pablo +Galindo diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 0925722392446..4ccd1240bac3e 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -650,6 +650,10 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) errstr = "Some I/O error occurred"; } } else { + if (ERR_GET_LIB(e) == ERR_LIB_SSL && + ERR_GET_REASON(e) == SSL_R_CERTIFICATE_VERIFY_FAILED) { + type = state->PySSLCertVerificationErrorObject; + } p = PY_SSL_ERROR_SYSCALL; } break; From webhook-mailer at python.org Thu Aug 3 10:09:33 2023 From: webhook-mailer at python.org (pablogsal) Date: Thu, 03 Aug 2023 14:09:33 -0000 Subject: [Python-checkins] [3.12] gh-107077: Raise SSLCertVerificationError even if the error is set via SSL_ERROR_SYSCALL (GH-107586) (#107587) Message-ID: https://github.com/python/cpython/commit/93fcf7587888e93ff7e45f4422c315f8a5f1ba0d commit: 93fcf7587888e93ff7e45f4422c315f8a5f1ba0d branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: pablogsal date: 2023-08-03T15:09:29+01:00 summary: [3.12] gh-107077: Raise SSLCertVerificationError even if the error is set via SSL_ERROR_SYSCALL (GH-107586) (#107587) Co-authored-by: Pablo Galindo Salgado Co-authored-by: T. Wouters files: A Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst M Modules/_ssl.c diff --git a/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst new file mode 100644 index 0000000000000..ecaf437a48e0a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst @@ -0,0 +1,6 @@ +Seems that in some conditions, OpenSSL will return ``SSL_ERROR_SYSCALL`` +instead of ``SSL_ERROR_SSL`` when a certification verification has failed, +but the error parameters will still contain ``ERR_LIB_SSL`` and +``SSL_R_CERTIFICATE_VERIFY_FAILED``. We are now detecting this situation and +raising the appropiate ``ssl.SSLCertVerificationError``. Patch by Pablo +Galindo diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 619b4f4e94d06..e939f95048984 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -647,6 +647,10 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) errstr = "Some I/O error occurred"; } } else { + if (ERR_GET_LIB(e) == ERR_LIB_SSL && + ERR_GET_REASON(e) == SSL_R_CERTIFICATE_VERIFY_FAILED) { + type = state->PySSLCertVerificationErrorObject; + } p = PY_SSL_ERROR_SYSCALL; } break; From webhook-mailer at python.org Thu Aug 3 10:09:36 2023 From: webhook-mailer at python.org (pablogsal) Date: Thu, 03 Aug 2023 14:09:36 -0000 Subject: [Python-checkins] [3.10] gh-107077: Raise SSLCertVerificationError even if the error is set via SSL_ERROR_SYSCALL (GH-107586) (#107589) Message-ID: https://github.com/python/cpython/commit/24d54feafc28a9fb421de852d830cc370fe51db3 commit: 24d54feafc28a9fb421de852d830cc370fe51db3 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: pablogsal date: 2023-08-03T15:09:32+01:00 summary: [3.10] gh-107077: Raise SSLCertVerificationError even if the error is set via SSL_ERROR_SYSCALL (GH-107586) (#107589) Co-authored-by: Pablo Galindo Salgado Co-authored-by: T. Wouters files: A Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst M Modules/_ssl.c diff --git a/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst new file mode 100644 index 0000000000000..ecaf437a48e0a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst @@ -0,0 +1,6 @@ +Seems that in some conditions, OpenSSL will return ``SSL_ERROR_SYSCALL`` +instead of ``SSL_ERROR_SSL`` when a certification verification has failed, +but the error parameters will still contain ``ERR_LIB_SSL`` and +``SSL_R_CERTIFICATE_VERIFY_FAILED``. We are now detecting this situation and +raising the appropiate ``ssl.SSLCertVerificationError``. Patch by Pablo +Galindo diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 7a28f2d37f6c5..bb0508f9aceec 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -656,6 +656,10 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) errstr = "Some I/O error occurred"; } } else { + if (ERR_GET_LIB(e) == ERR_LIB_SSL && + ERR_GET_REASON(e) == SSL_R_CERTIFICATE_VERIFY_FAILED) { + type = state->PySSLCertVerificationErrorObject; + } p = PY_SSL_ERROR_SYSCALL; } break; From webhook-mailer at python.org Thu Aug 3 10:19:29 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Thu, 03 Aug 2023 14:19:29 -0000 Subject: [Python-checkins] gh-107576: Ensure `__orig_bases__` are our own in `get_original_bases` (#107584) Message-ID: https://github.com/python/cpython/commit/ed4a978449c856372d1a7cd389f91cafe2581c87 commit: ed4a978449c856372d1a7cd389f91cafe2581c87 branch: main author: James Hilton-Balfe committer: AlexWaygood date: 2023-08-03T14:19:24Z summary: gh-107576: Ensure `__orig_bases__` are our own in `get_original_bases` (#107584) Co-authored-by: Chris Bouchard Co-authored-by: Alex Waygood files: A Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst M Lib/test/test_types.py M Lib/types.py diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 81744940f25b8..f2efee90dc024 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1397,6 +1397,7 @@ class A: pass class B(typing.Generic[T]): pass class C(B[int]): pass class D(B[str], float): pass + self.assertEqual(types.get_original_bases(A), (object,)) self.assertEqual(types.get_original_bases(B), (typing.Generic[T],)) self.assertEqual(types.get_original_bases(C), (B[int],)) @@ -1409,6 +1410,18 @@ class F(list[int]): pass self.assertEqual(types.get_original_bases(E), (list[T],)) self.assertEqual(types.get_original_bases(F), (list[int],)) + class FirstBase(typing.Generic[T]): pass + class SecondBase(typing.Generic[T]): pass + class First(FirstBase[int]): pass + class Second(SecondBase[int]): pass + class G(First, Second): pass + self.assertEqual(types.get_original_bases(G), (First, Second)) + + class First_(typing.Generic[T]): pass + class Second_(typing.Generic[T]): pass + class H(First_, Second_): pass + self.assertEqual(types.get_original_bases(H), (First_, Second_)) + class ClassBasedNamedTuple(typing.NamedTuple): x: int diff --git a/Lib/types.py b/Lib/types.py index 6110e6e1de724..b4aa19cec40c8 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -165,14 +165,11 @@ class Baz(list[str]): ... assert get_original_bases(int) == (object,) """ try: - return cls.__orig_bases__ + return cls.__dict__.get("__orig_bases__", cls.__bases__) except AttributeError: - try: - return cls.__bases__ - except AttributeError: - raise TypeError( - f'Expected an instance of type, not {type(cls).__name__!r}' - ) from None + raise TypeError( + f"Expected an instance of type, not {type(cls).__name__!r}" + ) from None class DynamicClassAttribute: diff --git a/Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst b/Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst new file mode 100644 index 0000000000000..67677dd3c8ed2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst @@ -0,0 +1,3 @@ +Fix :func:`types.get_original_bases` to only return +:attr:`!__orig_bases__` if it is present on ``cls`` directly. Patch by +James Hilton-Balfe. From webhook-mailer at python.org Thu Aug 3 11:04:07 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 03 Aug 2023 15:04:07 -0000 Subject: [Python-checkins] gh-107446: Fix test_inspect.test_class_with_method_from_other_module when ran multiple times (#107451) Message-ID: https://github.com/python/cpython/commit/14fbd4e6b16dcbcbff448b047f7e2faa27bbedba commit: 14fbd4e6b16dcbcbff448b047f7e2faa27bbedba branch: main author: Tian Gao committer: kumaraditya303 date: 2023-08-03T15:04:03Z summary: gh-107446: Fix test_inspect.test_class_with_method_from_other_module when ran multiple times (#107451) Co-authored-by: Kumar Aditya files: M Lib/test/test_inspect.py diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 3fbfc07325553..5c31748ce2caa 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -990,6 +990,9 @@ def f(self): with DirsOnSysPath(tempdir): import inspect_actual self.assertIn("correct", inspect.getsource(inspect_actual.A)) + # Remove the module from sys.modules to force it to be reloaded. + # This is necessary when the test is run multiple times. + sys.modules.pop("inspect_actual") @unittest.skipIf( support.is_emscripten or support.is_wasi, From webhook-mailer at python.org Thu Aug 3 11:07:47 2023 From: webhook-mailer at python.org (Yhg1s) Date: Thu, 03 Aug 2023 15:07:47 -0000 Subject: [Python-checkins] [3.12] gh-107576: Ensure `__orig_bases__` are our own in `get_original_bases` (GH-107584) (#107592) Message-ID: https://github.com/python/cpython/commit/d2c7b25afba3d86ee20331d9fb722a6fe1f768ac commit: d2c7b25afba3d86ee20331d9fb722a6fe1f768ac branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-03T17:07:43+02:00 summary: [3.12] gh-107576: Ensure `__orig_bases__` are our own in `get_original_bases` (GH-107584) (#107592) gh-107576: Ensure `__orig_bases__` are our own in `get_original_bases` (GH-107584) (cherry picked from commit ed4a978449c856372d1a7cd389f91cafe2581c87) Co-authored-by: James Hilton-Balfe Co-authored-by: Chris Bouchard Co-authored-by: Alex Waygood files: A Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst M Lib/test/test_types.py M Lib/types.py diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 81744940f25b8..f2efee90dc024 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -1397,6 +1397,7 @@ class A: pass class B(typing.Generic[T]): pass class C(B[int]): pass class D(B[str], float): pass + self.assertEqual(types.get_original_bases(A), (object,)) self.assertEqual(types.get_original_bases(B), (typing.Generic[T],)) self.assertEqual(types.get_original_bases(C), (B[int],)) @@ -1409,6 +1410,18 @@ class F(list[int]): pass self.assertEqual(types.get_original_bases(E), (list[T],)) self.assertEqual(types.get_original_bases(F), (list[int],)) + class FirstBase(typing.Generic[T]): pass + class SecondBase(typing.Generic[T]): pass + class First(FirstBase[int]): pass + class Second(SecondBase[int]): pass + class G(First, Second): pass + self.assertEqual(types.get_original_bases(G), (First, Second)) + + class First_(typing.Generic[T]): pass + class Second_(typing.Generic[T]): pass + class H(First_, Second_): pass + self.assertEqual(types.get_original_bases(H), (First_, Second_)) + class ClassBasedNamedTuple(typing.NamedTuple): x: int diff --git a/Lib/types.py b/Lib/types.py index 6110e6e1de724..b4aa19cec40c8 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -165,14 +165,11 @@ class Baz(list[str]): ... assert get_original_bases(int) == (object,) """ try: - return cls.__orig_bases__ + return cls.__dict__.get("__orig_bases__", cls.__bases__) except AttributeError: - try: - return cls.__bases__ - except AttributeError: - raise TypeError( - f'Expected an instance of type, not {type(cls).__name__!r}' - ) from None + raise TypeError( + f"Expected an instance of type, not {type(cls).__name__!r}" + ) from None class DynamicClassAttribute: diff --git a/Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst b/Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst new file mode 100644 index 0000000000000..67677dd3c8ed2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst @@ -0,0 +1,3 @@ +Fix :func:`types.get_original_bases` to only return +:attr:`!__orig_bases__` if it is present on ``cls`` directly. Patch by +James Hilton-Balfe. From webhook-mailer at python.org Thu Aug 3 15:51:12 2023 From: webhook-mailer at python.org (ericsnowcurrently) Date: Thu, 03 Aug 2023 19:51:12 -0000 Subject: [Python-checkins] gh-107080: Fix Py_TRACE_REFS Crashes Under Isolated Subinterpreters (gh-107567) Message-ID: https://github.com/python/cpython/commit/58ef74186795c56e3ec86e8c8f351a1d7826638a commit: 58ef74186795c56e3ec86e8c8f351a1d7826638a branch: main author: Eric Snow committer: ericsnowcurrently date: 2023-08-03T19:51:08Z summary: gh-107080: Fix Py_TRACE_REFS Crashes Under Isolated Subinterpreters (gh-107567) The linked list of objects was a global variable, which broke isolation between interpreters, causing crashes. To solve this, we've moved the linked list to each interpreter. files: A Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst M Include/internal/pycore_object.h M Include/internal/pycore_object_state.h M Include/internal/pycore_runtime_init.h M Objects/object.c M Python/pylifecycle.c diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index b730aba991283..76b3cd69cbf5a 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -292,8 +292,8 @@ extern void _PyDebug_PrintTotalRefs(void); #ifdef Py_TRACE_REFS extern void _Py_AddToAllObjects(PyObject *op, int force); -extern void _Py_PrintReferences(FILE *); -extern void _Py_PrintReferenceAddresses(FILE *); +extern void _Py_PrintReferences(PyInterpreterState *, FILE *); +extern void _Py_PrintReferenceAddresses(PyInterpreterState *, FILE *); #endif diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index 94005d7788143..65feb5af969f8 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -11,17 +11,22 @@ extern "C" { struct _py_object_runtime_state { #ifdef Py_REF_DEBUG Py_ssize_t interpreter_leaks; -#else - int _not_used; #endif + int _not_used; }; struct _py_object_state { #ifdef Py_REF_DEBUG Py_ssize_t reftotal; -#else - int _not_used; #endif +#ifdef Py_TRACE_REFS + /* Head of circular doubly-linked list of all objects. These are linked + * together via the _ob_prev and _ob_next members of a PyObject, which + * exist only in a Py_TRACE_REFS build. + */ + PyObject refchain; +#endif + int _not_used; }; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index dcfceef4f175a..e89d368be07aa 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -157,6 +157,7 @@ extern PyTypeObject _PyExc_MemoryError; { .threshold = 10, }, \ }, \ }, \ + .object_state = _py_object_state_INIT(INTERP), \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ .dict_state = _dict_state_INIT, \ .func_state = { \ @@ -186,6 +187,16 @@ extern PyTypeObject _PyExc_MemoryError; .context_ver = 1, \ } +#ifdef Py_TRACE_REFS +# define _py_object_state_INIT(INTERP) \ + { \ + .refchain = {&INTERP.object_state.refchain, &INTERP.object_state.refchain}, \ + } +#else +# define _py_object_state_INIT(INTERP) \ + { 0 } +#endif + // global objects diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst new file mode 100644 index 0000000000000..5084c854360e3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst @@ -0,0 +1,4 @@ +Trace refs builds (``--with-trace-refs``) were crashing when used with +isolated subinterpreters. The problematic global state has been isolated to +each interpreter. Other fixing the crashes, this change does not affect +users. diff --git a/Objects/object.c b/Objects/object.c index 8caa5fd0af3cc..c4413a00ede80 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -159,11 +159,8 @@ _PyDebug_PrintTotalRefs(void) { Do not call them otherwise, they do not initialize the object! */ #ifdef Py_TRACE_REFS -/* Head of circular doubly-linked list of all objects. These are linked - * together via the _ob_prev and _ob_next members of a PyObject, which - * exist only in a Py_TRACE_REFS build. - */ -static PyObject refchain = {&refchain, &refchain}; + +#define REFCHAIN(interp) &interp->object_state.refchain /* Insert op at the front of the list of all objects. If force is true, * op is added even if _ob_prev and _ob_next are non-NULL already. If @@ -188,10 +185,11 @@ _Py_AddToAllObjects(PyObject *op, int force) } #endif if (force || op->_ob_prev == NULL) { - op->_ob_next = refchain._ob_next; - op->_ob_prev = &refchain; - refchain._ob_next->_ob_prev = op; - refchain._ob_next = op; + PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); + op->_ob_next = refchain->_ob_next; + op->_ob_prev = refchain; + refchain->_ob_next->_ob_prev = op; + refchain->_ob_next = op; } } #endif /* Py_TRACE_REFS */ @@ -2229,7 +2227,8 @@ _Py_ForgetReference(PyObject *op) _PyObject_ASSERT_FAILED_MSG(op, "negative refcnt"); } - if (op == &refchain || + PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); + if (op == refchain || op->_ob_prev->_ob_next != op || op->_ob_next->_ob_prev != op) { _PyObject_ASSERT_FAILED_MSG(op, "invalid object chain"); @@ -2237,12 +2236,12 @@ _Py_ForgetReference(PyObject *op) #ifdef SLOW_UNREF_CHECK PyObject *p; - for (p = refchain._ob_next; p != &refchain; p = p->_ob_next) { + for (p = refchain->_ob_next; p != refchain; p = p->_ob_next) { if (p == op) { break; } } - if (p == &refchain) { + if (p == refchain) { /* Not found */ _PyObject_ASSERT_FAILED_MSG(op, "object not found in the objects list"); @@ -2258,11 +2257,15 @@ _Py_ForgetReference(PyObject *op) * interpreter must be in a healthy state. */ void -_Py_PrintReferences(FILE *fp) +_Py_PrintReferences(PyInterpreterState *interp, FILE *fp) { PyObject *op; + if (interp == NULL) { + interp = _PyInterpreterState_Main(); + } fprintf(fp, "Remaining objects:\n"); - for (op = refchain._ob_next; op != &refchain; op = op->_ob_next) { + PyObject *refchain = REFCHAIN(interp); + for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) { fprintf(fp, "%p [%zd] ", (void *)op, Py_REFCNT(op)); if (PyObject_Print(op, fp, 0) != 0) { PyErr_Clear(); @@ -2274,34 +2277,42 @@ _Py_PrintReferences(FILE *fp) /* Print the addresses of all live objects. Unlike _Py_PrintReferences, this * doesn't make any calls to the Python C API, so is always safe to call. */ +// XXX This function is not safe to use if the interpreter has been +// freed or is in an unhealthy state (e.g. late in finalization). +// The call in Py_FinalizeEx() is okay since the main interpreter +// is statically allocated. void -_Py_PrintReferenceAddresses(FILE *fp) +_Py_PrintReferenceAddresses(PyInterpreterState *interp, FILE *fp) { PyObject *op; + PyObject *refchain = REFCHAIN(interp); fprintf(fp, "Remaining object addresses:\n"); - for (op = refchain._ob_next; op != &refchain; op = op->_ob_next) + for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) fprintf(fp, "%p [%zd] %s\n", (void *)op, Py_REFCNT(op), Py_TYPE(op)->tp_name); } +/* The implementation of sys.getobjects(). */ PyObject * _Py_GetObjects(PyObject *self, PyObject *args) { int i, n; PyObject *t = NULL; PyObject *res, *op; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (!PyArg_ParseTuple(args, "i|O", &n, &t)) return NULL; - op = refchain._ob_next; + PyObject *refchain = REFCHAIN(interp); + op = refchain->_ob_next; res = PyList_New(0); if (res == NULL) return NULL; - for (i = 0; (n == 0 || i < n) && op != &refchain; i++) { + for (i = 0; (n == 0 || i < n) && op != refchain; i++) { while (op == self || op == args || op == res || op == t || (t != NULL && !Py_IS_TYPE(op, (PyTypeObject *) t))) { op = op->_ob_next; - if (op == &refchain) + if (op == refchain) return res; } if (PyList_Append(res, op) < 0) { @@ -2313,6 +2324,8 @@ _Py_GetObjects(PyObject *self, PyObject *args) return res; } +#undef REFCHAIN + #endif diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ceca7776a276e..7a17f92b550c0 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1921,11 +1921,11 @@ Py_FinalizeEx(void) } if (dump_refs) { - _Py_PrintReferences(stderr); + _Py_PrintReferences(tstate->interp, stderr); } if (dump_refs_fp != NULL) { - _Py_PrintReferences(dump_refs_fp); + _Py_PrintReferences(tstate->interp, dump_refs_fp); } #endif /* Py_TRACE_REFS */ @@ -1961,11 +1961,11 @@ Py_FinalizeEx(void) */ if (dump_refs) { - _Py_PrintReferenceAddresses(stderr); + _Py_PrintReferenceAddresses(tstate->interp, stderr); } if (dump_refs_fp != NULL) { - _Py_PrintReferenceAddresses(dump_refs_fp); + _Py_PrintReferenceAddresses(tstate->interp, dump_refs_fp); fclose(dump_refs_fp); } #endif /* Py_TRACE_REFS */ From webhook-mailer at python.org Thu Aug 3 16:35:46 2023 From: webhook-mailer at python.org (pablogsal) Date: Thu, 03 Aug 2023 20:35:46 -0000 Subject: [Python-checkins] [3.10] Revert "[3.10] gh-107077: Raise SSLCertVerificationError even if the error is set via SSL_ERROR_SYSCALL (GH-107586) (#107589)" (#107602) Message-ID: https://github.com/python/cpython/commit/c32f0955b430ea5960fb293f8a6ff9263fc64e5c commit: c32f0955b430ea5960fb293f8a6ff9263fc64e5c branch: 3.10 author: Pablo Galindo Salgado committer: pablogsal date: 2023-08-03T21:35:42+01:00 summary: [3.10] Revert "[3.10] gh-107077: Raise SSLCertVerificationError even if the error is set via SSL_ERROR_SYSCALL (GH-107586) (#107589)" (#107602) files: D Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst M Modules/_ssl.c diff --git a/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst deleted file mode 100644 index ecaf437a48e0a..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst +++ /dev/null @@ -1,6 +0,0 @@ -Seems that in some conditions, OpenSSL will return ``SSL_ERROR_SYSCALL`` -instead of ``SSL_ERROR_SSL`` when a certification verification has failed, -but the error parameters will still contain ``ERR_LIB_SSL`` and -``SSL_R_CERTIFICATE_VERIFY_FAILED``. We are now detecting this situation and -raising the appropiate ``ssl.SSLCertVerificationError``. Patch by Pablo -Galindo diff --git a/Modules/_ssl.c b/Modules/_ssl.c index bb0508f9aceec..7a28f2d37f6c5 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -656,10 +656,6 @@ PySSL_SetError(PySSLSocket *sslsock, int ret, const char *filename, int lineno) errstr = "Some I/O error occurred"; } } else { - if (ERR_GET_LIB(e) == ERR_LIB_SSL && - ERR_GET_REASON(e) == SSL_R_CERTIFICATE_VERIFY_FAILED) { - type = state->PySSLCertVerificationErrorObject; - } p = PY_SSL_ERROR_SYSCALL; } break; From webhook-mailer at python.org Thu Aug 3 17:26:19 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Thu, 03 Aug 2023 21:26:19 -0000 Subject: [Python-checkins] gh-106368: Argument clinic tests: improve error message when `expect_success()` fails (#107606) Message-ID: https://github.com/python/cpython/commit/9e6590b0978876de14587f528a09632b8879c369 commit: 9e6590b0978876de14587f528a09632b8879c369 branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-03T21:26:14Z summary: gh-106368: Argument clinic tests: improve error message when `expect_success()` fails (#107606) files: M Lib/test/test_clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 2f94566361872..3aa41631d3637 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1680,7 +1680,8 @@ def run_clinic(self, *args): def expect_success(self, *args): out, err, code = self.run_clinic(*args) - self.assertEqual(code, 0, f"Unexpected failure: {args=}") + if code != 0: + self.fail("\n".join([f"Unexpected failure: {args=}", out, err])) self.assertEqual(err, "") return out From webhook-mailer at python.org Thu Aug 3 18:18:33 2023 From: webhook-mailer at python.org (Yhg1s) Date: Thu, 03 Aug 2023 22:18:33 -0000 Subject: [Python-checkins] [3.12] gh-107080: Fix Py_TRACE_REFS Crashes Under Isolated Subinterpreters (gh-107567) (#107599) Message-ID: https://github.com/python/cpython/commit/58af2293c52a1ad3754d254690c0e54f787c545b commit: 58af2293c52a1ad3754d254690c0e54f787c545b branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-04T00:18:29+02:00 summary: [3.12] gh-107080: Fix Py_TRACE_REFS Crashes Under Isolated Subinterpreters (gh-107567) (#107599) gh-107080: Fix Py_TRACE_REFS Crashes Under Isolated Subinterpreters (gh-107567) The linked list of objects was a global variable, which broke isolation between interpreters, causing crashes. To solve this, we've moved the linked list to each interpreter. (cherry picked from commit 58ef74186795c56e3ec86e8c8f351a1d7826638a) Co-authored-by: Eric Snow files: A Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst M Include/internal/pycore_object.h M Include/internal/pycore_object_state.h M Include/internal/pycore_runtime_init.h M Objects/object.c M Python/pylifecycle.c diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 0981d1122fec5..5a9403456903e 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -271,8 +271,8 @@ extern void _PyDebug_PrintTotalRefs(void); #ifdef Py_TRACE_REFS extern void _Py_AddToAllObjects(PyObject *op, int force); -extern void _Py_PrintReferences(FILE *); -extern void _Py_PrintReferenceAddresses(FILE *); +extern void _Py_PrintReferences(PyInterpreterState *, FILE *); +extern void _Py_PrintReferenceAddresses(PyInterpreterState *, FILE *); #endif diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index 94005d7788143..65feb5af969f8 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -11,17 +11,22 @@ extern "C" { struct _py_object_runtime_state { #ifdef Py_REF_DEBUG Py_ssize_t interpreter_leaks; -#else - int _not_used; #endif + int _not_used; }; struct _py_object_state { #ifdef Py_REF_DEBUG Py_ssize_t reftotal; -#else - int _not_used; #endif +#ifdef Py_TRACE_REFS + /* Head of circular doubly-linked list of all objects. These are linked + * together via the _ob_prev and _ob_next members of a PyObject, which + * exist only in a Py_TRACE_REFS build. + */ + PyObject refchain; +#endif + int _not_used; }; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 4130188079cff..7aace9f86119c 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -101,6 +101,7 @@ extern PyTypeObject _PyExc_MemoryError; { .threshold = 10, }, \ }, \ }, \ + .object_state = _py_object_state_INIT(INTERP), \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ .dict_state = _dict_state_INIT, \ .func_state = { \ @@ -130,6 +131,16 @@ extern PyTypeObject _PyExc_MemoryError; .context_ver = 1, \ } +#ifdef Py_TRACE_REFS +# define _py_object_state_INIT(INTERP) \ + { \ + .refchain = {&INTERP.object_state.refchain, &INTERP.object_state.refchain}, \ + } +#else +# define _py_object_state_INIT(INTERP) \ + { 0 } +#endif + // global objects diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst new file mode 100644 index 0000000000000..5084c854360e3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst @@ -0,0 +1,4 @@ +Trace refs builds (``--with-trace-refs``) were crashing when used with +isolated subinterpreters. The problematic global state has been isolated to +each interpreter. Other fixing the crashes, this change does not affect +users. diff --git a/Objects/object.c b/Objects/object.c index bd0fa40bd2a85..b4c9416036c44 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -158,11 +158,8 @@ _PyDebug_PrintTotalRefs(void) { Do not call them otherwise, they do not initialize the object! */ #ifdef Py_TRACE_REFS -/* Head of circular doubly-linked list of all objects. These are linked - * together via the _ob_prev and _ob_next members of a PyObject, which - * exist only in a Py_TRACE_REFS build. - */ -static PyObject refchain = {&refchain, &refchain}; + +#define REFCHAIN(interp) &interp->object_state.refchain /* Insert op at the front of the list of all objects. If force is true, * op is added even if _ob_prev and _ob_next are non-NULL already. If @@ -187,10 +184,11 @@ _Py_AddToAllObjects(PyObject *op, int force) } #endif if (force || op->_ob_prev == NULL) { - op->_ob_next = refchain._ob_next; - op->_ob_prev = &refchain; - refchain._ob_next->_ob_prev = op; - refchain._ob_next = op; + PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); + op->_ob_next = refchain->_ob_next; + op->_ob_prev = refchain; + refchain->_ob_next->_ob_prev = op; + refchain->_ob_next = op; } } #endif /* Py_TRACE_REFS */ @@ -2206,7 +2204,8 @@ _Py_ForgetReference(PyObject *op) _PyObject_ASSERT_FAILED_MSG(op, "negative refcnt"); } - if (op == &refchain || + PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); + if (op == refchain || op->_ob_prev->_ob_next != op || op->_ob_next->_ob_prev != op) { _PyObject_ASSERT_FAILED_MSG(op, "invalid object chain"); @@ -2214,12 +2213,12 @@ _Py_ForgetReference(PyObject *op) #ifdef SLOW_UNREF_CHECK PyObject *p; - for (p = refchain._ob_next; p != &refchain; p = p->_ob_next) { + for (p = refchain->_ob_next; p != refchain; p = p->_ob_next) { if (p == op) { break; } } - if (p == &refchain) { + if (p == refchain) { /* Not found */ _PyObject_ASSERT_FAILED_MSG(op, "object not found in the objects list"); @@ -2235,11 +2234,15 @@ _Py_ForgetReference(PyObject *op) * interpreter must be in a healthy state. */ void -_Py_PrintReferences(FILE *fp) +_Py_PrintReferences(PyInterpreterState *interp, FILE *fp) { PyObject *op; + if (interp == NULL) { + interp = _PyInterpreterState_Main(); + } fprintf(fp, "Remaining objects:\n"); - for (op = refchain._ob_next; op != &refchain; op = op->_ob_next) { + PyObject *refchain = REFCHAIN(interp); + for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) { fprintf(fp, "%p [%zd] ", (void *)op, Py_REFCNT(op)); if (PyObject_Print(op, fp, 0) != 0) { PyErr_Clear(); @@ -2251,34 +2254,42 @@ _Py_PrintReferences(FILE *fp) /* Print the addresses of all live objects. Unlike _Py_PrintReferences, this * doesn't make any calls to the Python C API, so is always safe to call. */ +// XXX This function is not safe to use if the interpreter has been +// freed or is in an unhealthy state (e.g. late in finalization). +// The call in Py_FinalizeEx() is okay since the main interpreter +// is statically allocated. void -_Py_PrintReferenceAddresses(FILE *fp) +_Py_PrintReferenceAddresses(PyInterpreterState *interp, FILE *fp) { PyObject *op; + PyObject *refchain = REFCHAIN(interp); fprintf(fp, "Remaining object addresses:\n"); - for (op = refchain._ob_next; op != &refchain; op = op->_ob_next) + for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) fprintf(fp, "%p [%zd] %s\n", (void *)op, Py_REFCNT(op), Py_TYPE(op)->tp_name); } +/* The implementation of sys.getobjects(). */ PyObject * _Py_GetObjects(PyObject *self, PyObject *args) { int i, n; PyObject *t = NULL; PyObject *res, *op; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (!PyArg_ParseTuple(args, "i|O", &n, &t)) return NULL; - op = refchain._ob_next; + PyObject *refchain = REFCHAIN(interp); + op = refchain->_ob_next; res = PyList_New(0); if (res == NULL) return NULL; - for (i = 0; (n == 0 || i < n) && op != &refchain; i++) { + for (i = 0; (n == 0 || i < n) && op != refchain; i++) { while (op == self || op == args || op == res || op == t || (t != NULL && !Py_IS_TYPE(op, (PyTypeObject *) t))) { op = op->_ob_next; - if (op == &refchain) + if (op == refchain) return res; } if (PyList_Append(res, op) < 0) { @@ -2290,6 +2301,8 @@ _Py_GetObjects(PyObject *self, PyObject *args) return res; } +#undef REFCHAIN + #endif diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index a67fa26b37227..5b8c8af179c00 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1920,11 +1920,11 @@ Py_FinalizeEx(void) } if (dump_refs) { - _Py_PrintReferences(stderr); + _Py_PrintReferences(tstate->interp, stderr); } if (dump_refs_fp != NULL) { - _Py_PrintReferences(dump_refs_fp); + _Py_PrintReferences(tstate->interp, dump_refs_fp); } #endif /* Py_TRACE_REFS */ @@ -1960,11 +1960,11 @@ Py_FinalizeEx(void) */ if (dump_refs) { - _Py_PrintReferenceAddresses(stderr); + _Py_PrintReferenceAddresses(tstate->interp, stderr); } if (dump_refs_fp != NULL) { - _Py_PrintReferenceAddresses(dump_refs_fp); + _Py_PrintReferenceAddresses(tstate->interp, dump_refs_fp); fclose(dump_refs_fp); } #endif /* Py_TRACE_REFS */ From webhook-mailer at python.org Thu Aug 3 20:17:21 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Fri, 04 Aug 2023 00:17:21 -0000 Subject: [Python-checkins] gh-104146: Argument clinic: remove unused methods and variables (#107608) Message-ID: https://github.com/python/cpython/commit/ee78d01a61f44c31b8add2bffe687718d2d34d60 commit: ee78d01a61f44c31b8add2bffe687718d2d34d60 branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-04T00:17:17Z summary: gh-104146: Argument clinic: remove unused methods and variables (#107608) files: M Tools/clinic/clinic.py M Tools/clinic/cpp.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 1bcdb6b1c3640..733a83ee58c0f 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -377,8 +377,10 @@ def flush() -> None: return tuple(version) def version_comparitor(version1: str, version2: str) -> Literal[-1, 0, 1]: - iterator = itertools.zip_longest(version_splitter(version1), version_splitter(version2), fillvalue=0) - for i, (a, b) in enumerate(iterator): + iterator = itertools.zip_longest( + version_splitter(version1), version_splitter(version2), fillvalue=0 + ) + for a, b in iterator: if a < b: return -1 if a > b: @@ -4368,19 +4370,11 @@ def depth(self) -> int: """ return len(self.indents) - def indent(self, line: str) -> str: - """ - Indents a line by the currently defined margin. - """ - assert self.margin is not None, "Cannot call .indent() before calling .infer()" - return self.margin + line - def dedent(self, line: str) -> str: """ Dedents a line by the currently defined margin. - (The inverse of 'indent'.) """ - assert self.margin is not None, "Cannot call .indent() before calling .infer()" + assert self.margin is not None, "Cannot call .dedent() before calling .infer()" margin = self.margin indent = self.indents[-1] if not line.startswith(margin): @@ -4641,10 +4635,6 @@ def valid_line(self, line: str) -> bool: return True - @staticmethod - def calculate_indent(line: str) -> int: - return len(line) - len(line.strip()) - def next( self, state: StateKeeper, diff --git a/Tools/clinic/cpp.py b/Tools/clinic/cpp.py index 5b7fa06494ad5..21a1b02e862c1 100644 --- a/Tools/clinic/cpp.py +++ b/Tools/clinic/cpp.py @@ -65,14 +65,6 @@ def fail(self, *a: object) -> NoReturn: print(" ", ' '.join(str(x) for x in a)) sys.exit(-1) - def close(self) -> None: - if self.stack: - self.fail("Ended file while still in a preprocessor conditional block!") - - def write(self, s: str) -> None: - for line in s.split("\n"): - self.writeline(line) - def writeline(self, line: str) -> None: self.line_number += 1 line = line.strip() From webhook-mailer at python.org Fri Aug 4 00:37:27 2023 From: webhook-mailer at python.org (brandtbucher) Date: Fri, 04 Aug 2023 04:37:27 -0000 Subject: [Python-checkins] GH-105481: Mark more files as generated (GH-107598) Message-ID: https://github.com/python/cpython/commit/e52e87c349feeee77445205829bfc8db9fe4b80e commit: e52e87c349feeee77445205829bfc8db9fe4b80e branch: main author: Brandt Bucher committer: brandtbucher date: 2023-08-03T21:37:23-07:00 summary: GH-105481: Mark more files as generated (GH-107598) files: M .gitattributes diff --git a/.gitattributes b/.gitattributes index 5d5558da711b1..1f2f8a4cd8735 100644 --- a/.gitattributes +++ b/.gitattributes @@ -72,9 +72,11 @@ Doc/library/token-list.inc generated Include/internal/pycore_ast.h generated Include/internal/pycore_ast_state.h generated Include/internal/pycore_opcode.h generated +Include/internal/pycore_opcode_metadata.h generated Include/internal/pycore_*_generated.h generated Include/opcode.h generated Include/token.h generated +Lib/_opcode_metadata.py generated Lib/keyword.py generated Lib/test/levenshtein_examples.json generated Lib/test/test_stable_abi_ctypes.py generated From webhook-mailer at python.org Fri Aug 4 01:28:29 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Fri, 04 Aug 2023 05:28:29 -0000 Subject: [Python-checkins] gh-107609: Fix duplicate module check in Argument Clinic (#107610) Message-ID: https://github.com/python/cpython/commit/a443c310ac87f214164cb1e3f8af3f799668c867 commit: a443c310ac87f214164cb1e3f8af3f799668c867 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-04T07:28:25+02:00 summary: gh-107609: Fix duplicate module check in Argument Clinic (#107610) Also remove duplicate module def from _testcapi. files: A Misc/NEWS.d/next/Tools-Demos/2023-08-04-00-04-40.gh-issue-107609.2DqgtL.rst M Lib/test/test_clinic.py M Modules/_testcapi/vectorcall.c M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 3aa41631d3637..59669d61f01bd 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -416,6 +416,16 @@ def __init__(self): """ self.expect_failure(block, err, lineno=8) + def test_module_already_got_one(self): + err = "Already defined module 'm'!" + block = """ + /*[clinic input] + module m + module m + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-04-00-04-40.gh-issue-107609.2DqgtL.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-04-00-04-40.gh-issue-107609.2DqgtL.rst new file mode 100644 index 0000000000000..080a6c15d9b8c --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-04-00-04-40.gh-issue-107609.2DqgtL.rst @@ -0,0 +1,3 @@ +Fix duplicate module check in Argument Clinic. Previously, a duplicate +definition would incorrectly be silently accepted. Patch by Erlend E. +Aasland. diff --git a/Modules/_testcapi/vectorcall.c b/Modules/_testcapi/vectorcall.c index 61c6e0f485f47..2b5110fcba2c9 100644 --- a/Modules/_testcapi/vectorcall.c +++ b/Modules/_testcapi/vectorcall.c @@ -155,10 +155,9 @@ VectorCallClass_vectorcall(PyObject *callable, } /*[clinic input] -module _testcapi class _testcapi.VectorCallClass "PyObject *" "&PyType_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=8423a8e919f2f0df]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=95c63c1a47f9a995]*/ /*[clinic input] _testcapi.VectorCallClass.set_vectorcall diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 733a83ee58c0f..7525c1ca52400 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4472,7 +4472,7 @@ def directive_module(self, name: str) -> None: if cls: fail("Can't nest a module inside a class!") - if name in module.classes: + if name in module.modules: fail("Already defined module " + repr(name) + "!") m = Module(name, module) From webhook-mailer at python.org Fri Aug 4 03:48:36 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Fri, 04 Aug 2023 07:48:36 -0000 Subject: [Python-checkins] gh-106368: Increase Argument Clinic test coverage (#107611) Message-ID: https://github.com/python/cpython/commit/0bd784b355edf0d1911a7830db5d41c84171e367 commit: 0bd784b355edf0d1911a7830db5d41c84171e367 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-04T07:48:32Z summary: gh-106368: Increase Argument Clinic test coverage (#107611) Add tests for directives and destinations files: M Lib/test/test_clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 59669d61f01bd..562c66a37e329 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -117,6 +117,7 @@ def __repr__(self): class ClinicWholeFileTest(TestCase): + maxDiff = None def expect_failure(self, raw, errmsg, *, filename=None, lineno=None): _expect_failure(self, self.clinic.parse, raw, errmsg, @@ -426,6 +427,230 @@ def test_module_already_got_one(self): """ self.expect_failure(block, err, lineno=3) + def test_destination_already_got_one(self): + err = "Destination already exists: 'test'" + block = """ + /*[clinic input] + destination test new buffer + destination test new buffer + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_destination_does_not_exist(self): + err = "Destination does not exist: '/dev/null'" + block = """ + /*[clinic input] + output everything /dev/null + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + + def test_class_already_got_one(self): + err = "Already defined class 'C'!" + block = """ + /*[clinic input] + class C "" "" + class C "" "" + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_cant_nest_module_inside_class(self): + err = "Can't nest a module inside a class!" + block = """ + /*[clinic input] + class C "" "" + module C.m + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_dest_buffer_not_empty_at_eof(self): + expected_warning = ("Destination buffer 'buffer' not empty at " + "end of file, emptying.") + expected_generated = dedent(""" + /*[clinic input] + output everything buffer + fn + a: object + / + [clinic start generated code]*/ + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=1c4668687f5fd002]*/ + + /*[clinic input] + dump buffer + [clinic start generated code]*/ + + PyDoc_VAR(fn__doc__); + + PyDoc_STRVAR(fn__doc__, + "fn($module, a, /)\\n" + "--\\n" + "\\n"); + + #define FN_METHODDEF \\ + {"fn", (PyCFunction)fn, METH_O, fn__doc__}, + + static PyObject * + fn(PyObject *module, PyObject *a) + /*[clinic end generated code: output=be6798b148ab4e53 input=524ce2e021e4eba6]*/ + """) + block = dedent(""" + /*[clinic input] + output everything buffer + fn + a: object + / + [clinic start generated code]*/ + """) + with support.captured_stdout() as stdout: + generated = self.clinic.parse(block) + self.assertIn(expected_warning, stdout.getvalue()) + self.assertEqual(generated, expected_generated) + + def test_directive_set_misuse(self): + err = "unknown variable 'ets'" + block = """ + /*[clinic input] + set ets tse + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + + def test_directive_set_prefix(self): + block = dedent(""" + /*[clinic input] + set line_prefix '// ' + output everything suppress + output docstring_prototype buffer + fn + a: object + / + [clinic start generated code]*/ + /* We need to dump the buffer. + * If not, Argument Clinic will emit a warning */ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + generated = self.clinic.parse(block) + expected_docstring_prototype = "// PyDoc_VAR(fn__doc__);" + self.assertIn(expected_docstring_prototype, generated) + + def test_directive_set_suffix(self): + block = dedent(""" + /*[clinic input] + set line_suffix ' // test' + output everything suppress + output docstring_prototype buffer + fn + a: object + / + [clinic start generated code]*/ + /* We need to dump the buffer. + * If not, Argument Clinic will emit a warning */ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + generated = self.clinic.parse(block) + expected_docstring_prototype = "PyDoc_VAR(fn__doc__); // test" + self.assertIn(expected_docstring_prototype, generated) + + def test_directive_set_prefix_and_suffix(self): + block = dedent(""" + /*[clinic input] + set line_prefix '{block comment start} ' + set line_suffix ' {block comment end}' + output everything suppress + output docstring_prototype buffer + fn + a: object + / + [clinic start generated code]*/ + /* We need to dump the buffer. + * If not, Argument Clinic will emit a warning */ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + """) + generated = self.clinic.parse(block) + expected_docstring_prototype = "/* PyDoc_VAR(fn__doc__); */" + self.assertIn(expected_docstring_prototype, generated) + + def test_directive_printout(self): + block = dedent(""" + /*[clinic input] + output everything buffer + printout test + [clinic start generated code]*/ + """) + expected = dedent(""" + /*[clinic input] + output everything buffer + printout test + [clinic start generated code]*/ + test + /*[clinic end generated code: output=4e1243bd22c66e76 input=898f1a32965d44ca]*/ + """) + generated = self.clinic.parse(block) + self.assertEqual(generated, expected) + + def test_directive_preserve_twice(self): + err = "Can't have preserve twice in one block!" + block = """ + /*[clinic input] + preserve + preserve + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=3) + + def test_directive_preserve_input(self): + err = "'preserve' only works for blocks that don't produce any output!" + block = """ + /*[clinic input] + preserve + fn + a: object + / + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=6) + + def test_directive_preserve_output(self): + err = "'preserve' only works for blocks that don't produce any output!" + block = dedent(""" + /*[clinic input] + output everything buffer + preserve + [clinic start generated code]*/ + // Preserve this + /*[clinic end generated code: output=eaa49677ae4c1f7d input=559b5db18fddae6a]*/ + /*[clinic input] + dump buffer + [clinic start generated code]*/ + /*[clinic end generated code: output=da39a3ee5e6b4b0d input=524ce2e021e4eba6]*/ + """) + generated = self.clinic.parse(block) + self.assertEqual(generated, block) + + def test_directive_output_invalid_command(self): + err = ( + "Invalid command / destination name 'cmd', must be one of:\n" + " preset push pop print everything cpp_if docstring_prototype " + "docstring_definition methoddef_define impl_prototype " + "parser_prototype parser_definition cpp_endif methoddef_ifndef " + "impl_definition" + ) + block = """ + /*[clinic input] + output cmd buffer + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): @@ -1496,6 +1721,16 @@ class Foo "" "" """ self.expect_failure(block, err, lineno=3) + def test_duplicate_coexist(self): + err = "Called @coexist twice" + block = """ + module m + @coexist + @coexist + m.fn + """ + self.expect_failure(block, err, lineno=2) + def test_unused_param(self): block = self.parse(""" module foo @@ -1931,6 +2166,67 @@ def test_cli_fail_make_without_srcdir(self): msg = "error: --srcdir must not be empty with --make" self.assertIn(msg, err) + def test_file_dest(self): + block = dedent(""" + /*[clinic input] + destination test new file {path}.h + output everything test + func + a: object + / + [clinic start generated code]*/ + """) + expected_checksum_line = ( + "/*[clinic end generated code: " + "output=da39a3ee5e6b4b0d input=b602ab8e173ac3bd]*/\n" + ) + expected_output = dedent("""\ + /*[clinic input] + preserve + [clinic start generated code]*/ + + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # include "pycore_gc.h" // PyGC_Head + # include "pycore_runtime.h" // _Py_ID() + #endif + + + PyDoc_VAR(func__doc__); + + PyDoc_STRVAR(func__doc__, + "func($module, a, /)\\n" + "--\\n" + "\\n"); + + #define FUNC_METHODDEF \\ + {"func", (PyCFunction)func, METH_O, func__doc__}, + + static PyObject * + func(PyObject *module, PyObject *a) + /*[clinic end generated code: output=56c09670e89a0d9a input=a9049054013a1b77]*/ + """) + with os_helper.temp_dir() as tmp_dir: + in_fn = os.path.join(tmp_dir, "test.c") + out_fn = os.path.join(tmp_dir, "test.c.h") + with open(in_fn, "w", encoding="utf-8") as f: + f.write(block) + with open(out_fn, "w", encoding="utf-8") as f: + f.write("") # Write an empty output file! + # Clinic should complain about the empty output file. + _, err = self.expect_failure(in_fn) + expected_err = (f"Modified destination file {out_fn!r}, " + "not overwriting!") + self.assertIn(expected_err, err) + # Run clinic again, this time with the -f option. + out = self.expect_success("-f", in_fn) + # Read back the generated output. + with open(in_fn, encoding="utf-8") as f: + data = f.read() + expected_block = f"{block}{expected_checksum_line}" + self.assertEqual(data, expected_block) + with open(out_fn, encoding="utf-8") as f: + data = f.read() + self.assertEqual(data, expected_output) try: import _testclinic as ac_tester From webhook-mailer at python.org Fri Aug 4 05:10:33 2023 From: webhook-mailer at python.org (markshannon) Date: Fri, 04 Aug 2023 09:10:33 -0000 Subject: [Python-checkins] GH-107263: Increase C stack limit for most functions, except `_PyEval_EvalFrameDefault()` (GH-107535) Message-ID: https://github.com/python/cpython/commit/fa45958450aa3489607daf9855ca0474a2a20878 commit: fa45958450aa3489607daf9855ca0474a2a20878 branch: main author: Mark Shannon committer: markshannon date: 2023-08-04T10:10:29+01:00 summary: GH-107263: Increase C stack limit for most functions, except `_PyEval_EvalFrameDefault()` (GH-107535) * Set C recursion limit to 1500, set cost of eval loop to 2 frames, and compiler mutliply to 2. files: A Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst M Doc/whatsnew/3.12.rst M Include/cpython/pystate.h M Lib/test/list_tests.py M Lib/test/mapping_tests.py M Lib/test/support/__init__.py M Lib/test/test_ast.py M Lib/test/test_call.py M Lib/test/test_compile.py M Lib/test/test_dict.py M Lib/test/test_dictviews.py M Lib/test/test_exception_group.py M Lib/test/test_zlib.py M Parser/asdl_c.py M Python/Python-ast.c M Python/ast.c M Python/ast_opt.c M Python/bytecodes.c M Python/ceval.c M Python/generated_cases.c.h M Python/symtable.c diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3c39476ced9f4..6553013552d00 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -801,6 +801,11 @@ sys exception instance, rather than to a ``(typ, exc, tb)`` tuple. (Contributed by Irit Katriel in :gh:`103176`.) +* :func:`sys.setrecursionlimit` and :func:`sys.getrecursionlimit`. + The recursion limit now applies only to Python code. Builtin functions do + not use the recursion limit, but are protected by a different mechanism + that prevents recursion from causing a virtual machine crash. + tempfile -------- diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 30de4ee4b6c58..56e473cc5e42d 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -222,7 +222,8 @@ struct _ts { # ifdef __wasi__ # define C_RECURSION_LIMIT 500 # else -# define C_RECURSION_LIMIT 800 + // This value is duplicated in Lib/test/support/__init__.py +# define C_RECURSION_LIMIT 1500 # endif #endif diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index fe3ee80b8d461..b1ef332522d2c 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ +from test.support import ALWAYS_EQ, NEVER_EQ, C_RECURSION_LIMIT class CommonTest(seq_tests.CommonTest): @@ -61,7 +61,7 @@ def test_repr(self): def test_repr_deep(self): a = self.type2test([]) - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT + 1): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index 613206a0855ae..5492bbf86d1f8 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -2,6 +2,7 @@ import unittest import collections import sys +from test.support import C_RECURSION_LIMIT class BasicTestMappingProtocol(unittest.TestCase): @@ -624,7 +625,7 @@ def __repr__(self): def test_repr_deep(self): d = self._empty_mapping() - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT + 1): d0 = d d = self._empty_mapping() d[1] = d0 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 2a59911e54d6b..64c66d8e25d9c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -64,7 +64,8 @@ "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", - "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", + "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "C_RECURSION_LIMIT", + "skip_on_s390x", ] @@ -2460,3 +2461,10 @@ def adjust_int_max_str_digits(max_digits): #For recursion tests, easily exceeds default recursion limit EXCEEDS_RECURSION_LIMIT = 5000 + +# The default C recursion limit (from Include/cpython/pystate.h). +C_RECURSION_LIMIT = 1500 + +#Windows doesn't have os.uname() but it doesn't support s390x. +skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', + 'skipped on s390x') diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index a03fa4c7187b0..5346b39043f0f 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1084,6 +1084,7 @@ def next(self): return self enum._test_simple_enum(_Precedence, ast._Precedence) + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") @support.cpython_only def test_ast_recursion_limit(self): fail_depth = support.EXCEEDS_RECURSION_LIMIT diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 09a531f8cc627..c3c3b1853b573 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,5 +1,5 @@ import unittest -from test.support import cpython_only, requires_limited_api +from test.support import cpython_only, requires_limited_api, skip_on_s390x try: import _testcapi except ImportError: @@ -918,6 +918,7 @@ def test_multiple_values(self): @cpython_only class TestRecursion(unittest.TestCase): + @skip_on_s390x def test_super_deep(self): def recurse(n): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 85ce0a4b39d85..770184c5ef9a9 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -11,10 +11,9 @@ import warnings from test import support from test.support import (script_helper, requires_debug_ranges, - requires_specialization) + requires_specialization, C_RECURSION_LIMIT) from test.support.os_helper import FakePath - class TestSpecifics(unittest.TestCase): def compile_single(self, source): @@ -112,7 +111,7 @@ def __getitem__(self, key): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_extended_arg(self): - repeat = 2000 + repeat = int(C_RECURSION_LIMIT * 0.9) longexpr = 'x = x or ' + '-x' * repeat g = {} code = textwrap.dedent(''' @@ -558,16 +557,12 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): - # Expected limit is sys.getrecursionlimit() * the scaling factor - # in symtable.c (currently 3) - # We expect to fail *at* that limit, because we use up some of - # the stack depth limit in the test suite code - # So we check the expected limit and 75% of that - # XXX (ncoghlan): duplicating the scaling factor here is a little - # ugly. Perhaps it should be exposed somewhere... - fail_depth = sys.getrecursionlimit() * 3 - crash_depth = sys.getrecursionlimit() * 300 - success_depth = int(fail_depth * 0.75) + # Expected limit is C_RECURSION_LIMIT * 2 + # Duplicating the limit here is a little ugly. + # Perhaps it should be exposed somewhere... + fail_depth = C_RECURSION_LIMIT * 2 + 1 + crash_depth = C_RECURSION_LIMIT * 100 + success_depth = int(C_RECURSION_LIMIT * 1.8) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 79638340059f6..fbc6ce8282de3 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -8,7 +8,7 @@ import unittest import weakref from test import support -from test.support import import_helper +from test.support import import_helper, C_RECURSION_LIMIT class DictTest(unittest.TestCase): @@ -596,7 +596,7 @@ def __repr__(self): def test_repr_deep(self): d = {} - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT + 1): d = {1: d} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index 924f4a6829e19..2bd9d6eef8cfc 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -3,6 +3,7 @@ import pickle import sys import unittest +from test.support import C_RECURSION_LIMIT class DictSetTest(unittest.TestCase): @@ -279,7 +280,7 @@ def test_recursive_repr(self): def test_deeply_nested_repr(self): d = {} - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT//2 + 100): d = {42: d.values()} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 2658e027ff3e2..a02d54da35e94 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,7 +1,7 @@ import collections.abc import types import unittest - +from test.support import C_RECURSION_LIMIT class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): @@ -460,7 +460,7 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(2000): + for i in range(C_RECURSION_LIMIT + 1): e = ExceptionGroup('eg', [e]) return e diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 3dac70eb12852..2113757254c0e 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -7,7 +7,7 @@ import pickle import random import sys -from test.support import bigmemtest, _1G, _4G +from test.support import bigmemtest, _1G, _4G, skip_on_s390x zlib = import_helper.import_module('zlib') @@ -44,10 +44,7 @@ # zlib.decompress(func1(data)) == zlib.decompress(func2(data)) == data # # Make the assumption that s390x always has an accelerator to simplify the skip -# condition. Windows doesn't have os.uname() but it doesn't support s390x. -skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', - 'skipped on s390x') - +# condition. class VersionTestCase(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst new file mode 100644 index 0000000000000..fb0940b456dae --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst @@ -0,0 +1,3 @@ +Increase C recursion limit for functions other than the main interpreter +from 800 to 1500. This should allow functions like ``list.__repr__`` and +``json.dumps`` to handle all the inputs that they could prior to 3.12 diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index f159b573ce472..2a36610527f89 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1393,7 +1393,7 @@ class PartingShots(StaticVisitor): int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 3; + int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return 0; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 412de79397477..8047b1259c5d8 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -13073,7 +13073,7 @@ PyObject* PyAST_mod2obj(mod_ty t) int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 3; + int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return 0; diff --git a/Python/ast.c b/Python/ast.c index 68600ce683b97..74c97f948d15e 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1029,7 +1029,7 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps) /* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 3 +#define COMPILER_STACK_FRAME_SCALE 2 int _PyAST_Validate(mod_ty mod) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index ad1e312b084b7..82e7559e5b629 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1112,7 +1112,7 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat #undef CALL_SEQ /* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 3 +#define COMPILER_STACK_FRAME_SCALE 2 int _PyAST_Optimize(mod_ty mod, PyArena *arena, int optimize, int ff_features) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0bea7b59597a2..90e26d3c86b38 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -741,7 +741,7 @@ dummy_func( tstate->cframe = cframe.previous; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; } diff --git a/Python/ceval.c b/Python/ceval.c index 369b9a69152a5..b85e9677747af 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -624,6 +624,11 @@ extern const struct _PyCode_DEF(8) _Py_InitCleanup; # pragma warning(disable:4102) #endif + +/* _PyEval_EvalFrameDefault() is a *big* function, + * so consume 3 units of C stack */ +#define PY_EVAL_C_STACK_UNITS 2 + PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { @@ -676,6 +681,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int frame->previous = &entry_frame; cframe.current_frame = frame; + tstate->c_recursion_remaining -= (PY_EVAL_C_STACK_UNITS - 1); if (_Py_EnterRecursiveCallTstate(tstate, "")) { tstate->c_recursion_remaining--; tstate->py_recursion_remaining--; @@ -907,7 +913,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Restore previous cframe and exit */ tstate->cframe = cframe.previous; assert(tstate->cframe->current_frame == frame->previous); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return NULL; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e43b3dedf14f5..9fa549a120324 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -903,7 +903,7 @@ tstate->cframe = cframe.previous; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; } diff --git a/Python/symtable.c b/Python/symtable.c index 04be3192d6c7c..e9adbd5d29b1f 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -282,17 +282,10 @@ symtable_new(void) return NULL; } -/* When compiling the use of C stack is probably going to be a lot - lighter than when executing Python code but still can overflow - and causing a Python crash if not checked (e.g. eval("()"*300000)). - Using the current recursion limit for the compiler seems too - restrictive (it caused at least one test to fail) so a factor is - used to allow deeper recursion when compiling an expression. - - Using a scaling factor means this should automatically adjust when +/* Using a scaling factor means this should automatically adjust when the recursion limit is adjusted for small or large C stack allocations. */ -#define COMPILER_STACK_FRAME_SCALE 3 +#define COMPILER_STACK_FRAME_SCALE 2 struct symtable * _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) From webhook-mailer at python.org Fri Aug 4 05:34:27 2023 From: webhook-mailer at python.org (markshannon) Date: Fri, 04 Aug 2023 09:34:27 -0000 Subject: [Python-checkins] Add some GC stats to Py_STATS (GH-107581) Message-ID: https://github.com/python/cpython/commit/2ba7c7f7b151ff56cf12bf3cab286981bb646c90 commit: 2ba7c7f7b151ff56cf12bf3cab286981bb646c90 branch: main author: Mark Shannon committer: markshannon date: 2023-08-04T10:34:23+01:00 summary: Add some GC stats to Py_STATS (GH-107581) files: M Include/internal/pycore_code.h M Include/pystats.h M Modules/gcmodule.c M Python/specialize.c M Tools/scripts/summarize_stats.py diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index ee1b85187cbab..00099376635e9 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -274,6 +274,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co); #define EVAL_CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.eval_calls[name]++; } while (0) #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \ do { if (_py_stats && PyFunction_Check(callable)) _py_stats->call_stats.eval_calls[name]++; } while (0) +#define GC_STAT_ADD(gen, name, n) do { if (_py_stats) _py_stats->gc_stats[(gen)].name += (n); } while (0) // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -287,6 +288,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define OBJECT_STAT_INC_COND(name, cond) ((void)0) #define EVAL_CALL_STAT_INC(name) ((void)0) #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) ((void)0) +#define GC_STAT_ADD(gen, name, n) ((void)0) #endif // !Py_STATS // Utility functions for reading/writing 32/64-bit values in the inline caches. diff --git a/Include/pystats.h b/Include/pystats.h index 54c9b8d8b3538..e24aef5fe8072 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -74,12 +74,21 @@ typedef struct _object_stats { uint64_t optimization_traces_created; uint64_t optimization_traces_executed; uint64_t optimization_uops_executed; + /* Temporary value used during GC */ + uint64_t object_visits; } ObjectStats; +typedef struct _gc_stats { + uint64_t collections; + uint64_t object_visits; + uint64_t objects_collected; +} GCStats; + typedef struct _stats { OpcodeStats opcode_stats[256]; CallStats call_stats; ObjectStats object_stats; + GCStats *gc_stats; } PyStats; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 246c0a9e160aa..35a35091bf451 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -460,6 +460,7 @@ update_refs(PyGC_Head *containers) static int visit_decref(PyObject *op, void *parent) { + OBJECT_STAT_INC(object_visits); _PyObject_ASSERT(_PyObject_CAST(parent), !_PyObject_IsFreed(op)); if (_PyObject_IS_GC(op)) { @@ -498,6 +499,7 @@ subtract_refs(PyGC_Head *containers) static int visit_reachable(PyObject *op, PyGC_Head *reachable) { + OBJECT_STAT_INC(object_visits); if (!_PyObject_IS_GC(op)) { return 0; } @@ -725,6 +727,7 @@ clear_unreachable_mask(PyGC_Head *unreachable) static int visit_move(PyObject *op, PyGC_Head *tolist) { + OBJECT_STAT_INC(object_visits); if (_PyObject_IS_GC(op)) { PyGC_Head *gc = AS_GC(op); if (gc_is_collecting(gc)) { @@ -1195,6 +1198,12 @@ gc_collect_main(PyThreadState *tstate, int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, int nofail) { + GC_STAT_ADD(generation, collections, 1); +#ifdef Py_STATS + if (_py_stats) { + _py_stats->object_stats.object_visits = 0; + } +#endif int i; Py_ssize_t m = 0; /* # objects collected */ Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ @@ -1351,6 +1360,15 @@ gc_collect_main(PyThreadState *tstate, int generation, stats->collected += m; stats->uncollectable += n; + GC_STAT_ADD(generation, objects_collected, m); +#ifdef Py_STATS + if (_py_stats) { + GC_STAT_ADD(generation, object_visits, + _py_stats->object_stats.object_visits); + _py_stats->object_stats.object_visits = 0; + } +#endif + if (PyDTrace_GC_DONE_ENABLED()) { PyDTrace_GC_DONE(n + m); } diff --git a/Python/specialize.c b/Python/specialize.c index 1669ce17fc804..de329ef1195cb 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -18,7 +18,8 @@ */ #ifdef Py_STATS -PyStats _py_stats_struct = { 0 }; +GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 }; +PyStats _py_stats_struct = { .gc_stats = &_py_gc_stats[0] }; PyStats *_py_stats = NULL; #define ADD_STAT_TO_DICT(res, field) \ @@ -202,17 +203,32 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->optimization_uops_executed); } +static void +print_gc_stats(FILE *out, GCStats *stats) +{ + for (int i = 0; i < NUM_GENERATIONS; i++) { + fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections); + fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits); + fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected); + } +} + static void print_stats(FILE *out, PyStats *stats) { print_spec_stats(out, stats->opcode_stats); print_call_stats(out, &stats->call_stats); print_object_stats(out, &stats->object_stats); + print_gc_stats(out, stats->gc_stats); } void _Py_StatsClear(void) { + for (int i = 0; i < NUM_GENERATIONS; i++) { + _py_gc_stats[i] = (GCStats) { 0 }; + } _py_stats_struct = (PyStats) { 0 }; + _py_stats_struct.gc_stats = _py_gc_stats; } void diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 9c881897c2de1..f798b2f772d08 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -494,6 +494,22 @@ def calculate_object_stats(stats): rows.append((label, value, ratio)) return rows +def calculate_gc_stats(stats): + gc_stats = [] + for key, value in stats.items(): + if not key.startswith("GC"): + continue + n, _, rest = key[3:].partition("]") + name = rest.strip() + gen_n = int(n) + while len(gc_stats) <= gen_n: + gc_stats.append({}) + gc_stats[gen_n][name] = value + return [ + (i, gen["collections"], gen["objects collected"], gen["object visits"]) + for (i, gen) in enumerate(gc_stats) + ] + def emit_object_stats(stats): with Section("Object stats", summary="allocations, frees and dict materializatons"): rows = calculate_object_stats(stats) @@ -505,6 +521,22 @@ def emit_comparative_object_stats(base_stats, head_stats): head_rows = calculate_object_stats(head_stats) emit_table(("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), join_rows(base_rows, head_rows)) +def emit_gc_stats(stats): + with Section("GC stats", summary="GC collections and effectiveness"): + rows = calculate_gc_stats(stats) + emit_table(("Generation:", "Collections:", "Objects collected:", "Object visits:"), rows) + +def emit_comparative_gc_stats(base_stats, head_stats): + with Section("GC stats", summary="GC collections and effectiveness"): + base_rows = calculate_gc_stats(base_stats) + head_rows = calculate_gc_stats(head_stats) + emit_table( + ("Generation:", + "Base collections:", "Head collections:", + "Base objects collected:", "Head objects collected:", + "Base object visits:", "Head object visits:"), + join_rows(base_rows, head_rows)) + def get_total(opcode_stats): total = 0 for opcode_stat in opcode_stats: @@ -574,6 +606,7 @@ def output_single_stats(stats): emit_specialization_overview(opcode_stats, total) emit_call_stats(stats) emit_object_stats(stats) + emit_gc_stats(stats) with Section("Meta stats", summary="Meta statistics"): emit_table(("", "Count:"), [('Number of data files', stats['__nfiles__'])]) @@ -596,6 +629,7 @@ def output_comparative_stats(base_stats, head_stats): ) emit_comparative_call_stats(base_stats, head_stats) emit_comparative_object_stats(base_stats, head_stats) + emit_comparative_gc_stats(base_stats, head_stats) def output_stats(inputs, json_output=None): if len(inputs) == 1: From webhook-mailer at python.org Fri Aug 4 06:25:55 2023 From: webhook-mailer at python.org (Yhg1s) Date: Fri, 04 Aug 2023 10:25:55 -0000 Subject: [Python-checkins] [3.12] GH-107263: Increase C stack limit for most functions, except `_PyEval_EvalFrameDefault()` (GH-107535) (#107618) Message-ID: https://github.com/python/cpython/commit/98902d6c0522df022d2f88499faf9774970e2838 commit: 98902d6c0522df022d2f88499faf9774970e2838 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-04T12:25:51+02:00 summary: [3.12] GH-107263: Increase C stack limit for most functions, except `_PyEval_EvalFrameDefault()` (GH-107535) (#107618) GH-107263: Increase C stack limit for most functions, except `_PyEval_EvalFrameDefault()` (GH-107535) * Set C recursion limit to 1500, set cost of eval loop to 2 frames, and compiler mutliply to 2. (cherry picked from commit fa45958450aa3489607daf9855ca0474a2a20878) Co-authored-by: Mark Shannon files: A Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst M Doc/whatsnew/3.12.rst M Include/cpython/pystate.h M Lib/test/list_tests.py M Lib/test/mapping_tests.py M Lib/test/support/__init__.py M Lib/test/test_ast.py M Lib/test/test_call.py M Lib/test/test_compile.py M Lib/test/test_dict.py M Lib/test/test_dictviews.py M Lib/test/test_exception_group.py M Lib/test/test_zlib.py M Parser/asdl_c.py M Python/Python-ast.c M Python/ast.c M Python/ast_opt.c M Python/bytecodes.c M Python/ceval.c M Python/generated_cases.c.h M Python/symtable.c diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 39e702bb012fb..79e618588143a 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -802,6 +802,11 @@ sys exception instance, rather than to a ``(typ, exc, tb)`` tuple. (Contributed by Irit Katriel in :gh:`103176`.) +* :func:`sys.setrecursionlimit` and :func:`sys.getrecursionlimit`. + The recursion limit now applies only to Python code. Builtin functions do + not use the recursion limit, but are protected by a different mechanism + that prevents recursion from causing a virtual machine crash. + tempfile -------- diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index f33c72d4cf4d2..628f2e0996e46 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -255,7 +255,8 @@ struct _ts { # ifdef __wasi__ # define C_RECURSION_LIMIT 500 # else -# define C_RECURSION_LIMIT 800 + // This value is duplicated in Lib/test/support/__init__.py +# define C_RECURSION_LIMIT 1500 # endif #endif diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index fe3ee80b8d461..b1ef332522d2c 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ +from test.support import ALWAYS_EQ, NEVER_EQ, C_RECURSION_LIMIT class CommonTest(seq_tests.CommonTest): @@ -61,7 +61,7 @@ def test_repr(self): def test_repr_deep(self): a = self.type2test([]) - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT + 1): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index 613206a0855ae..5492bbf86d1f8 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -2,6 +2,7 @@ import unittest import collections import sys +from test.support import C_RECURSION_LIMIT class BasicTestMappingProtocol(unittest.TestCase): @@ -624,7 +625,7 @@ def __repr__(self): def test_repr_deep(self): d = self._empty_mapping() - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT + 1): d0 = d d = self._empty_mapping() d[1] = d0 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 3b332f49951f0..c3c3cf0a71596 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -64,7 +64,8 @@ "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", - "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", + "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "C_RECURSION_LIMIT", + "skip_on_s390x", ] @@ -2460,3 +2461,10 @@ def adjust_int_max_str_digits(max_digits): #For recursion tests, easily exceeds default recursion limit EXCEEDS_RECURSION_LIMIT = 5000 + +# The default C recursion limit (from Include/cpython/pystate.h). +C_RECURSION_LIMIT = 1500 + +#Windows doesn't have os.uname() but it doesn't support s390x. +skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', + 'skipped on s390x') diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index a03fa4c7187b0..5346b39043f0f 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1084,6 +1084,7 @@ def next(self): return self enum._test_simple_enum(_Precedence, ast._Precedence) + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") @support.cpython_only def test_ast_recursion_limit(self): fail_depth = support.EXCEEDS_RECURSION_LIMIT diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 12759c53bb662..ec8dc29d36c16 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,5 +1,5 @@ import unittest -from test.support import cpython_only, requires_limited_api +from test.support import cpython_only, requires_limited_api, skip_on_s390x try: import _testcapi except ImportError: @@ -931,6 +931,7 @@ def test_multiple_values(self): @cpython_only class TestRecursion(unittest.TestCase): + @skip_on_s390x def test_super_deep(self): def recurse(n): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 784c0550cc09b..142d3f5c125b6 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -11,10 +11,9 @@ import warnings from test import support from test.support import (script_helper, requires_debug_ranges, - requires_specialization) + requires_specialization, C_RECURSION_LIMIT) from test.support.os_helper import FakePath - class TestSpecifics(unittest.TestCase): def compile_single(self, source): @@ -112,7 +111,7 @@ def __getitem__(self, key): @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_extended_arg(self): - repeat = 2000 + repeat = int(C_RECURSION_LIMIT * 0.9) longexpr = 'x = x or ' + '-x' * repeat g = {} code = textwrap.dedent(''' @@ -558,16 +557,12 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_compiler_recursion_limit(self): - # Expected limit is sys.getrecursionlimit() * the scaling factor - # in symtable.c (currently 3) - # We expect to fail *at* that limit, because we use up some of - # the stack depth limit in the test suite code - # So we check the expected limit and 75% of that - # XXX (ncoghlan): duplicating the scaling factor here is a little - # ugly. Perhaps it should be exposed somewhere... - fail_depth = sys.getrecursionlimit() * 3 - crash_depth = sys.getrecursionlimit() * 300 - success_depth = int(fail_depth * 0.75) + # Expected limit is C_RECURSION_LIMIT * 2 + # Duplicating the limit here is a little ugly. + # Perhaps it should be exposed somewhere... + fail_depth = C_RECURSION_LIMIT * 2 + 1 + crash_depth = C_RECURSION_LIMIT * 100 + success_depth = int(C_RECURSION_LIMIT * 1.8) def check_limit(prefix, repeated, mode="single"): expect_ok = prefix + repeated * success_depth diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 79638340059f6..fbc6ce8282de3 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -8,7 +8,7 @@ import unittest import weakref from test import support -from test.support import import_helper +from test.support import import_helper, C_RECURSION_LIMIT class DictTest(unittest.TestCase): @@ -596,7 +596,7 @@ def __repr__(self): def test_repr_deep(self): d = {} - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT + 1): d = {1: d} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index 924f4a6829e19..2bd9d6eef8cfc 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -3,6 +3,7 @@ import pickle import sys import unittest +from test.support import C_RECURSION_LIMIT class DictSetTest(unittest.TestCase): @@ -279,7 +280,7 @@ def test_recursive_repr(self): def test_deeply_nested_repr(self): d = {} - for i in range(sys.getrecursionlimit() + 100): + for i in range(C_RECURSION_LIMIT//2 + 100): d = {42: d.values()} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index fa159a76ec1af..22be3a7795cb9 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,7 +1,7 @@ import collections.abc import types import unittest - +from test.support import C_RECURSION_LIMIT class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): @@ -433,7 +433,7 @@ def test_basics_split_by_predicate__match(self): class DeepRecursionInSplitAndSubgroup(unittest.TestCase): def make_deep_eg(self): e = TypeError(1) - for i in range(2000): + for i in range(C_RECURSION_LIMIT + 1): e = ExceptionGroup('eg', [e]) return e diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 3dac70eb12852..2113757254c0e 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -7,7 +7,7 @@ import pickle import random import sys -from test.support import bigmemtest, _1G, _4G +from test.support import bigmemtest, _1G, _4G, skip_on_s390x zlib = import_helper.import_module('zlib') @@ -44,10 +44,7 @@ # zlib.decompress(func1(data)) == zlib.decompress(func2(data)) == data # # Make the assumption that s390x always has an accelerator to simplify the skip -# condition. Windows doesn't have os.uname() but it doesn't support s390x. -skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', - 'skipped on s390x') - +# condition. class VersionTestCase(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst new file mode 100644 index 0000000000000..fb0940b456dae --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst @@ -0,0 +1,3 @@ +Increase C recursion limit for functions other than the main interpreter +from 800 to 1500. This should allow functions like ``list.__repr__`` and +``json.dumps`` to handle all the inputs that they could prior to 3.12 diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index d4763ea260a5a..aa093b30a74d7 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1393,7 +1393,7 @@ class PartingShots(StaticVisitor): int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 3; + int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return 0; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 5db9ade3af454..f8bb6124fa487 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -13074,7 +13074,7 @@ PyObject* PyAST_mod2obj(mod_ty t) int starting_recursion_depth; /* Be careful here to prevent overflow. */ - int COMPILER_STACK_FRAME_SCALE = 3; + int COMPILER_STACK_FRAME_SCALE = 2; PyThreadState *tstate = _PyThreadState_GET(); if (!tstate) { return 0; diff --git a/Python/ast.c b/Python/ast.c index 68600ce683b97..74c97f948d15e 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1029,7 +1029,7 @@ validate_type_params(struct validator *state, asdl_type_param_seq *tps) /* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 3 +#define COMPILER_STACK_FRAME_SCALE 2 int _PyAST_Validate(mod_ty mod) diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 274bd134e1435..f8c4a9513236b 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -1103,7 +1103,7 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTOptimizeState *stat #undef CALL_SEQ /* See comments in symtable.c. */ -#define COMPILER_STACK_FRAME_SCALE 3 +#define COMPILER_STACK_FRAME_SCALE 2 int _PyAST_Optimize(mod_ty mod, PyArena *arena, _PyASTOptimizeState *state) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7ee48c70b0005..dc2ae221f0bdb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -635,7 +635,7 @@ dummy_func( tstate->cframe = cframe.previous; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; } diff --git a/Python/ceval.c b/Python/ceval.c index 4947420616213..88d6362bf48a1 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -637,6 +637,11 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { # pragma warning(disable:4102) #endif + +/* _PyEval_EvalFrameDefault() is a *big* function, + * so consume 3 units of C stack */ +#define PY_EVAL_C_STACK_UNITS 2 + PyObject* _Py_HOT_FUNCTION _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { @@ -691,6 +696,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int frame->previous = &entry_frame; cframe.current_frame = frame; + tstate->c_recursion_remaining -= (PY_EVAL_C_STACK_UNITS - 1); if (_Py_EnterRecursiveCallTstate(tstate, "")) { tstate->c_recursion_remaining--; tstate->py_recursion_remaining--; @@ -990,7 +996,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int /* Restore previous cframe and exit */ tstate->cframe = cframe.previous; assert(tstate->cframe->current_frame == frame->previous); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return NULL; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index eabb9fe5fac4d..b0a363ce9aa11 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -922,7 +922,7 @@ tstate->cframe = cframe.previous; assert(tstate->cframe->current_frame == frame->previous); assert(!_PyErr_Occurred(tstate)); - _Py_LeaveRecursiveCallTstate(tstate); + tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; #line 928 "Python/generated_cases.c.h" } diff --git a/Python/symtable.c b/Python/symtable.c index e2c00d17480dd..115882da09b75 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -282,17 +282,10 @@ symtable_new(void) return NULL; } -/* When compiling the use of C stack is probably going to be a lot - lighter than when executing Python code but still can overflow - and causing a Python crash if not checked (e.g. eval("()"*300000)). - Using the current recursion limit for the compiler seems too - restrictive (it caused at least one test to fail) so a factor is - used to allow deeper recursion when compiling an expression. - - Using a scaling factor means this should automatically adjust when +/* Using a scaling factor means this should automatically adjust when the recursion limit is adjusted for small or large C stack allocations. */ -#define COMPILER_STACK_FRAME_SCALE 3 +#define COMPILER_STACK_FRAME_SCALE 2 struct symtable * _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future) From webhook-mailer at python.org Fri Aug 4 08:13:14 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Fri, 04 Aug 2023 12:13:14 -0000 Subject: [Python-checkins] gh-107614: Normalise Argument Clinic error messages (#107615) Message-ID: https://github.com/python/cpython/commit/ac7605ed197e8b2336d44c8ac8aeae6faa90a768 commit: ac7605ed197e8b2336d44c8ac8aeae6faa90a768 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-04T12:13:10Z summary: gh-107614: Normalise Argument Clinic error messages (#107615) - always wrap the offending line, token, or name in quotes - in most cases, put the entire error message on one line Added tests for uncovered branches that were touched by this PR. Co-authored-by: Alex Waygood files: M Lib/test/test_clinic.py M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 562c66a37e329..dd17d02519bfa 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -161,11 +161,8 @@ def test_checksum_mismatch(self): [clinic start generated code]*/ /*[clinic end generated code: output=0123456789abcdef input=fedcba9876543210]*/ """ - err = ( - 'Checksum mismatch!\n' - 'Expected: 0123456789abcdef\n' - 'Computed: da39a3ee5e6b4b0d\n' - ) + err = ("Checksum mismatch! " + "Expected '0123456789abcdef', computed 'da39a3ee5e6b4b0d'") self.expect_failure(raw, err, filename="test.c", lineno=3) def test_garbage_after_stop_line(self): @@ -403,7 +400,7 @@ def test_clone_mismatch(self): self.expect_failure(block, err, lineno=9) def test_badly_formed_return_annotation(self): - err = "Badly formed annotation for m.f: 'Custom'" + err = "Badly formed annotation for 'm.f': 'Custom'" block = """ /*[python input] class Custom_return_converter(CReturnConverter): @@ -509,6 +506,15 @@ def test_dest_buffer_not_empty_at_eof(self): self.assertIn(expected_warning, stdout.getvalue()) self.assertEqual(generated, expected_generated) + def test_dest_clear(self): + err = "Can't clear destination 'file': it's not of type 'buffer'" + block = """ + /*[clinic input] + destination file clear + [clinic start generated code]*/ + """ + self.expect_failure(block, err, lineno=2) + def test_directive_set_misuse(self): err = "unknown variable 'ets'" block = """ @@ -598,7 +604,7 @@ def test_directive_printout(self): self.assertEqual(generated, expected) def test_directive_preserve_twice(self): - err = "Can't have preserve twice in one block!" + err = "Can't have 'preserve' twice in one block!" block = """ /*[clinic input] preserve @@ -637,13 +643,24 @@ def test_directive_preserve_output(self): self.assertEqual(generated, block) def test_directive_output_invalid_command(self): - err = ( - "Invalid command / destination name 'cmd', must be one of:\n" - " preset push pop print everything cpp_if docstring_prototype " - "docstring_definition methoddef_define impl_prototype " - "parser_prototype parser_definition cpp_endif methoddef_ifndef " - "impl_definition" - ) + err = dedent(""" + Invalid command or destination name 'cmd'. Must be one of: + - 'preset' + - 'push' + - 'pop' + - 'print' + - 'everything' + - 'cpp_if' + - 'docstring_prototype' + - 'docstring_definition' + - 'methoddef_define' + - 'impl_prototype' + - 'parser_prototype' + - 'parser_definition' + - 'cpp_endif' + - 'methoddef_ifndef' + - 'impl_definition' + """).strip() block = """ /*[clinic input] output cmd buffer @@ -919,7 +936,7 @@ def test_param_default_expr_named_constant(self): self.assertEqual("MAXSIZE", p.converter.c_default) err = ( - "When you specify a named constant ('sys.maxsize') as your default value,\n" + "When you specify a named constant ('sys.maxsize') as your default value, " "you MUST specify a valid c_default." ) block = """ @@ -931,7 +948,7 @@ def test_param_default_expr_named_constant(self): def test_param_default_expr_binop(self): err = ( - "When you specify an expression ('a + b') as your default value,\n" + "When you specify an expression ('a + b') as your default value, " "you MUST specify a valid c_default." ) block = """ @@ -954,7 +971,7 @@ def test_param_no_docstring(self): def test_param_default_parameters_out_of_order(self): err = ( - "Can't have a parameter without a default ('something_else')\n" + "Can't have a parameter without a default ('something_else') " "after a parameter with a default!" ) block = """ @@ -1088,7 +1105,7 @@ def test_return_converter_invalid_syntax(self): module os os.stat -> invalid syntax """ - err = "Badly formed annotation for os.stat: 'invalid syntax'" + err = "Badly formed annotation for 'os.stat': 'invalid syntax'" self.expect_failure(block, err) def test_legacy_converter_disallowed_in_return_annotation(self): @@ -1252,8 +1269,8 @@ def test_nested_groups(self): def test_disallowed_grouping__two_top_groups_on_left(self): err = ( - 'Function two_top_groups_on_left has an unsupported group ' - 'configuration. (Unexpected state 2.b)' + "Function 'two_top_groups_on_left' has an unsupported group " + "configuration. (Unexpected state 2.b)" ) block = """ module foo @@ -1281,7 +1298,7 @@ def test_disallowed_grouping__two_top_groups_on_right(self): ] """ err = ( - "Function two_top_groups_on_right has an unsupported group " + "Function 'two_top_groups_on_right' has an unsupported group " "configuration. (Unexpected state 6.b)" ) self.expect_failure(block, err) @@ -1317,7 +1334,7 @@ def test_disallowed_grouping__group_after_parameter_on_left(self): param: int """ err = ( - "Function group_after_parameter_on_left has an unsupported group " + "Function 'group_after_parameter_on_left' has an unsupported group " "configuration. (Unexpected state 2.b)" ) self.expect_failure(block, err) @@ -1334,7 +1351,7 @@ def test_disallowed_grouping__empty_group_on_left(self): param: int """ err = ( - "Function empty_group has an empty group.\n" + "Function 'empty_group' has an empty group. " "All groups must contain at least one parameter." ) self.expect_failure(block, err) @@ -1351,7 +1368,7 @@ def test_disallowed_grouping__empty_group_on_right(self): ] """ err = ( - "Function empty_group has an empty group.\n" + "Function 'empty_group' has an empty group. " "All groups must contain at least one parameter." ) self.expect_failure(block, err) @@ -1365,7 +1382,7 @@ def test_disallowed_grouping__no_matching_bracket(self): group2: int ] """ - err = "Function empty_group has a ] without a matching [." + err = "Function 'empty_group' has a ']' without a matching '['" self.expect_failure(block, err) def test_no_parameters(self): @@ -1400,7 +1417,7 @@ def test_illegal_module_line(self): foo.bar => int / """ - err = "Illegal function name: foo.bar => int" + err = "Illegal function name: 'foo.bar => int'" self.expect_failure(block, err) def test_illegal_c_basename(self): @@ -1409,7 +1426,7 @@ def test_illegal_c_basename(self): foo.bar as 935 / """ - err = "Illegal C basename: 935" + err = "Illegal C basename: '935'" self.expect_failure(block, err) def test_single_star(self): @@ -1419,7 +1436,7 @@ def test_single_star(self): * * """ - err = "Function bar uses '*' more than once." + err = "Function 'bar' uses '*' more than once." self.expect_failure(block, err) def test_parameters_required_after_star(self): @@ -1429,7 +1446,7 @@ def test_parameters_required_after_star(self): "module foo\nfoo.bar\n this: int\n *", "module foo\nfoo.bar\n this: int\n *\nDocstring.", ) - err = "Function bar specifies '*' without any parameters afterwards." + err = "Function 'bar' specifies '*' without any parameters afterwards." for block in dataset: with self.subTest(block=block): self.expect_failure(block, err) @@ -1442,7 +1459,7 @@ def test_single_slash(self): / """ err = ( - "Function bar has an unsupported group configuration. " + "Function 'bar' has an unsupported group configuration. " "(Unexpected state 0.d)" ) self.expect_failure(block, err) @@ -1456,7 +1473,7 @@ def test_double_slash(self): b: int / """ - err = "Function bar uses '/' more than once." + err = "Function 'bar' uses '/' more than once." self.expect_failure(block, err) def test_mix_star_and_slash(self): @@ -1470,7 +1487,7 @@ def test_mix_star_and_slash(self): / """ err = ( - "Function bar mixes keyword-only and positional-only parameters, " + "Function 'bar' mixes keyword-only and positional-only parameters, " "which is unsupported." ) self.expect_failure(block, err) @@ -1483,7 +1500,7 @@ def test_parameters_not_permitted_after_slash_for_now(self): x: int """ err = ( - "Function bar has an unsupported group configuration. " + "Function 'bar' has an unsupported group configuration. " "(Unexpected state 0.d)" ) self.expect_failure(block, err) @@ -1582,7 +1599,8 @@ def test_indent_stack_no_tabs(self): *vararg1: object \t*vararg2: object """ - err = "Tab characters are illegal in the Clinic DSL." + err = ("Tab characters are illegal in the Clinic DSL: " + r"'\t*vararg2: object'") self.expect_failure(block, err) def test_indent_stack_illegal_outdent(self): @@ -1841,8 +1859,17 @@ def test_vararg_cannot_take_default_value(self): """ self.expect_failure(block, err, lineno=1) + def test_default_is_not_of_correct_type(self): + err = ("int_converter: default value 2.5 for field 'a' " + "is not of type 'int'") + block = """ + fn + a: int = 2.5 + """ + self.expect_failure(block, err, lineno=1) + def test_invalid_legacy_converter(self): - err = "fhi is not a valid legacy converter" + err = "'fhi' is not a valid legacy converter" block = """ fn a: 'fhi' @@ -1850,7 +1877,7 @@ def test_invalid_legacy_converter(self): self.expect_failure(block, err, lineno=1) def test_parent_class_or_module_does_not_exist(self): - err = "Parent class or module z does not exist" + err = "Parent class or module 'z' does not exist" block = """ module m z.func @@ -1877,7 +1904,7 @@ def test_param_requires_custom_c_name(self): self.expect_failure(block, err, lineno=2) def test_state_func_docstring_assert_no_group(self): - err = "Function func has a ] without a matching [." + err = "Function 'func' has a ']' without a matching '['" block = """ module m m.func @@ -1887,7 +1914,7 @@ def test_state_func_docstring_assert_no_group(self): self.expect_failure(block, err, lineno=2) def test_state_func_docstring_no_summary(self): - err = "Docstring for m.func does not have a summary line!" + err = "Docstring for 'm.func' does not have a summary line!" block = """ module m m.func @@ -1983,13 +2010,11 @@ def test_cli_force(self): const char *hand_edited = "output block is overwritten"; /*[clinic end generated code: output=bogus input=bogus]*/ """) - fail_msg = dedent(""" - Checksum mismatch! - Expected: bogus - Computed: 2ed19 - Suggested fix: remove all generated code including the end marker, - or use the '-f' option. - """) + fail_msg = ( + "Checksum mismatch! Expected 'bogus', computed '2ed19'. " + "Suggested fix: remove all generated code including the end marker, " + "or use the '-f' option.\n" + ) with os_helper.temp_dir() as tmp_dir: fn = os.path.join(tmp_dir, "test.c") with open(fn, "w", encoding="utf-8") as f: @@ -2214,7 +2239,7 @@ def test_file_dest(self): f.write("") # Write an empty output file! # Clinic should complain about the empty output file. _, err = self.expect_failure(in_fn) - expected_err = (f"Modified destination file {out_fn!r}, " + expected_err = (f"Modified destination file {out_fn!r}; " "not overwriting!") self.assertIn(expected_err, err) # Run clinic again, this time with the -f option. diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 7525c1ca52400..ee5e0ef70f20f 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -290,9 +290,11 @@ def linear_format(s: str, **kwargs: str) -> str: continue if trailing: - fail("Text found after {" + name + "} block marker! It must be on a line by itself.") + fail(f"Text found after {{{name}}} block marker! " + "It must be on a line by itself.") if indent.strip(): - fail("Non-whitespace characters found before {" + name + "} block marker! It must be on a line by itself.") + fail(f"Non-whitespace characters found before {{{name}}} block marker! " + "It must be on a line by itself.") value = kwargs[name] if not value: @@ -1534,7 +1536,8 @@ def render_function( c.render(p, data) if has_option_groups and (not positional): - fail("You cannot use optional groups ('[' and ']')\nunless all parameters are positional-only ('/').") + fail("You cannot use optional groups ('[' and ']') " + "unless all parameters are positional-only ('/').") # HACK # when we're METH_O, but have a custom return converter, @@ -1871,7 +1874,7 @@ def is_stop_line(line: str) -> bool: for field in shlex.split(arguments): name, equals, value = field.partition('=') if not equals: - fail("Mangled Argument Clinic marker line:", repr(line)) + fail(f"Mangled Argument Clinic marker line: {line!r}") d[name.strip()] = value.strip() if self.verify: @@ -1882,11 +1885,10 @@ def is_stop_line(line: str) -> bool: computed = compute_checksum(output, len(checksum)) if checksum != computed: - fail("Checksum mismatch!\nExpected: {}\nComputed: {}\n" + fail("Checksum mismatch! " + f"Expected {checksum!r}, computed {computed!r}. " "Suggested fix: remove all generated code including " - "the end marker,\n" - "or use the '-f' option." - .format(checksum, computed)) + "the end marker, or use the '-f' option.") else: # put back output output_lines = output.splitlines(keepends=True) @@ -2017,9 +2019,10 @@ def __post_init__(self, args: tuple[str, ...]) -> None: ) extra_arguments = 1 if self.type == "file" else 0 if len(args) < extra_arguments: - fail(f"Not enough arguments for destination {self.name} new {self.type}") + fail(f"Not enough arguments for destination " + f"{self.name!r} new {self.type!r}") if len(args) > extra_arguments: - fail(f"Too many arguments for destination {self.name} new {self.type}") + fail(f"Too many arguments for destination {self.name!r} new {self.type!r}") if self.type =='file': d = {} filename = self.clinic.filename @@ -2041,7 +2044,7 @@ def __repr__(self) -> str: def clear(self) -> None: if self.type != 'buffer': - fail("Can't clear destination" + self.name + " , it's not of type buffer") + fail(f"Can't clear destination {self.name!r}: it's not of type 'buffer'") self.buffers.clear() def dump(self) -> str: @@ -2220,13 +2223,13 @@ def add_destination( *args: str ) -> None: if name in self.destinations: - fail("Destination already exists: " + repr(name)) + fail(f"Destination already exists: {name!r}") self.destinations[name] = Destination(name, type, self, args) def get_destination(self, name: str) -> Destination: d = self.destinations.get(name) if not d: - fail("Destination does not exist: " + repr(name)) + fail(f"Destination does not exist: {name!r}") return d def get_destination_buffer( @@ -2273,15 +2276,16 @@ def parse(self, input: str) -> str: os.makedirs(dirname) except FileExistsError: if not os.path.isdir(dirname): - fail("Can't write to destination {}, " - "can't make directory {}!".format( - destination.filename, dirname)) + fail(f"Can't write to destination " + f"{destination.filename!r}; " + f"can't make directory {dirname!r}!") if self.verify: with open(destination.filename) as f: parser_2 = BlockParser(f.read(), language=self.language) blocks = list(parser_2) if (len(blocks) != 1) or (blocks[0].input != 'preserve\n'): - fail("Modified destination file " + repr(destination.filename) + ", not overwriting!") + fail(f"Modified destination file " + f"{destination.filename!r}; not overwriting!") except FileNotFoundError: pass @@ -2322,7 +2326,8 @@ def _module_and_class( return module, cls child = parent.classes.get(field) if not child: - fail('Parent class or module ' + '.'.join(so_far) + " does not exist.") + fullname = ".".join(so_far) + fail(f"Parent class or module {fullname!r} does not exist.") cls = parent = child return module, cls @@ -2339,12 +2344,12 @@ def parse_file( extension = os.path.splitext(filename)[1][1:] if not extension: - fail("Can't extract file type for file " + repr(filename)) + fail(f"Can't extract file type for file {filename!r}") try: language = extensions[extension](filename) except KeyError: - fail("Can't identify file type for file " + repr(filename)) + fail(f"Can't identify file type for file {filename!r}") with open(filename, encoding="utf-8") as f: raw = f.read() @@ -2823,8 +2828,9 @@ def __init__(self, else: names = [cls.__name__ for cls in self.default_type] types_str = ', '.join(names) - fail("{}: default value {!r} for field {} is not of type {}".format( - self.__class__.__name__, default, name, types_str)) + cls_name = self.__class__.__name__ + fail(f"{cls_name}: default value {default!r} for field " + f"{name!r} is not of type {types_str!r}") self.default = default if c_default: @@ -3146,7 +3152,7 @@ def converter_init(self, *, accept: TypeSet = {object}) -> None: if accept == {int}: self.format_unit = 'i' elif accept != {object}: - fail("bool_converter: illegal 'accept' argument " + repr(accept)) + fail(f"bool_converter: illegal 'accept' argument {accept!r}") if self.default is not unspecified: self.default = bool(self.default) self.c_default = str(int(self.default)) @@ -3196,7 +3202,7 @@ class char_converter(CConverter): def converter_init(self) -> None: if isinstance(self.default, self.default_type): if len(self.default) != 1: - fail("char_converter: illegal default value " + repr(self.default)) + fail(f"char_converter: illegal default value {self.default!r}") self.c_default = repr(bytes(self.default))[1:] if self.c_default == '"\'"': @@ -3335,7 +3341,7 @@ def converter_init( if accept == {str}: self.format_unit = 'C' elif accept != {int}: - fail("int_converter: illegal 'accept' argument " + repr(accept)) + fail(f"int_converter: illegal 'accept' argument {accept!r}") if type is not None: self.type = type @@ -3472,7 +3478,7 @@ def converter_init(self, *, accept: TypeSet = {int}) -> None: elif accept == {int, NoneType}: self.converter = '_Py_convert_optional_to_ssize_t' else: - fail("Py_ssize_t_converter: illegal 'accept' argument " + repr(accept)) + fail(f"Py_ssize_t_converter: illegal 'accept' argument {accept!r}") def parse_arg(self, argname: str, displayname: str) -> str | None: if self.format_unit == 'n': @@ -3502,7 +3508,7 @@ def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: elif accept == {int, NoneType}: self.converter = '_PyEval_SliceIndex' else: - fail("slice_index_converter: illegal 'accept' argument " + repr(accept)) + fail(f"slice_index_converter: illegal 'accept' argument {accept!r}") class size_t_converter(CConverter): type = 'size_t' @@ -3850,7 +3856,7 @@ def converter_init( elif accept == {str, NoneType}: self.converter = '_PyUnicode_WideCharString_Opt_Converter' else: - fail("Py_UNICODE_converter: illegal 'accept' argument " + repr(accept)) + fail(f"Py_UNICODE_converter: illegal 'accept' argument {accept!r}") self.c_default = "NULL" def cleanup(self) -> str: @@ -4378,7 +4384,7 @@ def dedent(self, line: str) -> str: margin = self.margin indent = self.indents[-1] if not line.startswith(margin): - fail('Cannot dedent, line does not start with the previous margin:') + fail('Cannot dedent; line does not start with the previous margin.') return line[indent:] @@ -4464,7 +4470,9 @@ def reset(self) -> None: def directive_version(self, required: str) -> None: global version if version_comparitor(version, required) < 0: - fail("Insufficient Clinic version!\n Version: " + version + "\n Required: " + required) + fail("Insufficient Clinic version!\n" + f" Version: {version}\n" + f" Required: {required}") def directive_module(self, name: str) -> None: fields = name.split('.')[:-1] @@ -4473,7 +4481,7 @@ def directive_module(self, name: str) -> None: fail("Can't nest a module inside a class!") if name in module.modules: - fail("Already defined module " + repr(name) + "!") + fail(f"Already defined module {name!r}!") m = Module(name, module) module.modules[name] = m @@ -4491,7 +4499,7 @@ def directive_class( parent = cls or module if name in parent.classes: - fail("Already defined class " + repr(name) + "!") + fail(f"Already defined class {name!r}!") c = Class(name, module, cls, typedef, type_object) parent.classes[name] = c @@ -4499,7 +4507,7 @@ def directive_class( def directive_set(self, name: str, value: str) -> None: if name not in ("line_prefix", "line_suffix"): - fail("unknown variable", repr(name)) + fail(f"unknown variable {name!r}") value = value.format_map({ 'block comment start': '/*', @@ -4520,7 +4528,7 @@ def directive_destination( case "clear": self.clinic.get_destination(name).clear() case _: - fail("unknown destination command", repr(command)) + fail(f"unknown destination command {command!r}") def directive_output( @@ -4533,7 +4541,7 @@ def directive_output( if command_or_name == "preset": preset = self.clinic.presets.get(destination) if not preset: - fail("Unknown preset " + repr(destination) + "!") + fail(f"Unknown preset {destination!r}!") fd.update(preset) return @@ -4562,7 +4570,11 @@ def directive_output( return if command_or_name not in fd: - fail("Invalid command / destination name " + repr(command_or_name) + ", must be one of:\n preset push pop print everything " + " ".join(fd)) + allowed = ["preset", "push", "pop", "print", "everything"] + allowed.extend(fd) + fail(f"Invalid command or destination name {command_or_name!r}. " + "Must be one of:\n -", + "\n - ".join([repr(word) for word in allowed])) fd[command_or_name] = d def directive_dump(self, name: str) -> None: @@ -4574,7 +4586,7 @@ def directive_printout(self, *args: str) -> None: def directive_preserve(self) -> None: if self.preserve_output: - fail("Can't have preserve twice in one block!") + fail("Can't have 'preserve' twice in one block!") self.preserve_output = True def at_classmethod(self) -> None: @@ -4601,7 +4613,8 @@ def parse(self, block: Block) -> None: lines = block.input.split('\n') for line_number, line in enumerate(lines, self.clinic.block_parser.block_start_line_number): if '\t' in line: - fail('Tab characters are illegal in the Clinic DSL.\n\t' + repr(line), line_number=block_start) + fail(f'Tab characters are illegal in the Clinic DSL: {line!r}', + line_number=block_start) try: self.state(line) except ClinicError as exc: @@ -4712,7 +4725,8 @@ def state_modulename_name(self, line: str) -> None: module, cls = self.clinic._module_and_class(fields) if not (existing_function.kind is self.kind and existing_function.coexist == self.coexist): - fail("'kind' of function and cloned function don't match! (@classmethod/@staticmethod/@coexist)") + fail("'kind' of function and cloned function don't match! " + "(@classmethod/@staticmethod/@coexist)") function = existing_function.copy( name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, docstring='' @@ -4731,9 +4745,9 @@ def state_modulename_name(self, line: str) -> None: c_basename = c_basename.strip() or None if not is_legal_py_identifier(full_name): - fail("Illegal function name:", full_name) + fail(f"Illegal function name: {full_name!r}") if c_basename and not is_legal_c_identifier(c_basename): - fail("Illegal C basename:", c_basename) + fail(f"Illegal C basename: {c_basename!r}") return_converter = None if returns: @@ -4741,7 +4755,7 @@ def state_modulename_name(self, line: str) -> None: try: module_node = ast.parse(ast_input) except SyntaxError: - fail(f"Badly formed annotation for {full_name}: {returns!r}") + fail(f"Badly formed annotation for {full_name!r}: {returns!r}") function_node = module_node.body[0] assert isinstance(function_node, ast.FunctionDef) try: @@ -4752,7 +4766,7 @@ def state_modulename_name(self, line: str) -> None: fail(f"No available return converter called {name!r}") return_converter = return_converters[name](**kwargs) except ValueError: - fail(f"Badly formed annotation for {full_name}: {returns!r}") + fail(f"Badly formed annotation for {full_name!r}: {returns!r}") fields = [x.strip() for x in full_name.split('.')] function_name = fields.pop() @@ -4777,7 +4791,7 @@ def state_modulename_name(self, line: str) -> None: return_converter = CReturnConverter() if not module: - fail("Undefined module used in declaration of " + repr(full_name.strip()) + ".") + fail("Undefined module used in declaration of {full_name.strip()!r}.") self.function = Function(name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, return_converter=return_converter, kind=self.kind, coexist=self.coexist) self.block.signatures.append(self.function) @@ -4963,18 +4977,22 @@ def parse_parameter(self, line: str) -> None: except SyntaxError: pass if not module: - fail("Function " + self.function.name + " has an invalid parameter declaration:\n\t" + line) + fail(f"Function {self.function.name!r} has an invalid parameter declaration:\n\t", + repr(line)) function = module.body[0] assert isinstance(function, ast.FunctionDef) function_args = function.args if len(function_args.args) > 1: - fail("Function " + self.function.name + " has an invalid parameter declaration (comma?):\n\t" + line) + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (comma?): {line!r}") if function_args.defaults or function_args.kw_defaults: - fail("Function " + self.function.name + " has an invalid parameter declaration (default value?):\n\t" + line) + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (default value?): {line!r}") if function_args.kwarg: - fail("Function " + self.function.name + " has an invalid parameter declaration (**kwargs?):\n\t" + line) + fail(f"Function {self.function.name!r} has an " + f"invalid parameter declaration (**kwargs?): {line!r}") if function_args.vararg: is_vararg = True @@ -4988,7 +5006,7 @@ def parse_parameter(self, line: str) -> None: if not default: if self.parameter_state is ParamState.OPTIONAL: - fail(f"Can't have a parameter without a default ({parameter_name!r})\n" + fail(f"Can't have a parameter without a default ({parameter_name!r}) " "after a parameter with a default!") value: Sentinels | Null if is_vararg: @@ -5048,10 +5066,10 @@ def bad_node(self, node: ast.AST) -> None: except NameError: pass # probably a named constant except Exception as e: - fail("Malformed expression given as default value\n" - "{!r} caused {!r}".format(default, e)) + fail("Malformed expression given as default value " + f"{default!r} caused {e!r}") if bad: - fail("Unsupported expression as default value: " + repr(default)) + fail(f"Unsupported expression as default value: {default!r}") assignment = module.body[0] assert isinstance(assignment, ast.Assign) @@ -5069,7 +5087,10 @@ def bad_node(self, node: ast.AST) -> None: )): c_default = kwargs.get("c_default") if not (isinstance(c_default, str) and c_default): - fail("When you specify an expression (" + repr(default) + ") as your default value,\nyou MUST specify a valid c_default." + ast.dump(expr)) + fail(f"When you specify an expression ({default!r}) " + f"as your default value, " + f"you MUST specify a valid c_default.", + ast.dump(expr)) py_default = default value = unknown elif isinstance(expr, ast.Attribute): @@ -5079,13 +5100,16 @@ def bad_node(self, node: ast.AST) -> None: a.append(n.attr) n = n.value if not isinstance(n, ast.Name): - fail("Unsupported default value " + repr(default) + " (looked like a Python constant)") + fail(f"Unsupported default value {default!r} " + "(looked like a Python constant)") a.append(n.id) py_default = ".".join(reversed(a)) c_default = kwargs.get("c_default") if not (isinstance(c_default, str) and c_default): - fail("When you specify a named constant (" + repr(py_default) + ") as your default value,\nyou MUST specify a valid c_default.") + fail(f"When you specify a named constant ({py_default!r}) " + "as your default value, " + "you MUST specify a valid c_default.") try: value = eval(py_default) @@ -5102,13 +5126,15 @@ def bad_node(self, node: ast.AST) -> None: c_default = py_default except SyntaxError as e: - fail("Syntax error: " + repr(e.text)) + fail(f"Syntax error: {e.text!r}") except (ValueError, AttributeError): value = unknown c_default = kwargs.get("c_default") py_default = default if not (isinstance(c_default, str) and c_default): - fail("When you specify a named constant (" + repr(py_default) + ") as your default value,\nyou MUST specify a valid c_default.") + fail("When you specify a named constant " + f"({py_default!r}) as your default value, " + "you MUST specify a valid c_default.") kwargs.setdefault('c_default', c_default) kwargs.setdefault('py_default', py_default) @@ -5116,7 +5142,7 @@ def bad_node(self, node: ast.AST) -> None: dict = legacy_converters if legacy else converters legacy_str = "legacy " if legacy else "" if name not in dict: - fail(f'{name} is not a valid {legacy_str}converter') + fail(f'{name!r} is not a valid {legacy_str}converter') # if you use a c_name for the parameter, we just give that name to the converter # but the parameter object gets the python name converter = dict[name](c_name or parameter_name, parameter_name, self.function, value, **kwargs) @@ -5141,7 +5167,8 @@ def bad_node(self, node: ast.AST) -> None: self.parameter_state = ParamState.START self.function.parameters.clear() else: - fail("A 'self' parameter, if specified, must be the very first thing in the parameter block.") + fail("A 'self' parameter, if specified, must be the " + "very first thing in the parameter block.") if isinstance(converter, defining_class_converter): _lp = len(self.function.parameters) @@ -5153,16 +5180,18 @@ def bad_node(self, node: ast.AST) -> None: if self.group: fail("A 'defining_class' parameter cannot be in an optional group.") else: - fail("A 'defining_class' parameter, if specified, must either be the first thing in the parameter block, or come just after 'self'.") + fail("A 'defining_class' parameter, if specified, must either " + "be the first thing in the parameter block, or come just " + "after 'self'.") p = Parameter(parameter_name, kind, function=self.function, converter=converter, default=value, group=self.group) names = [k.name for k in self.function.parameters.values()] if parameter_name in names[1:]: - fail("You can't have two parameters named " + repr(parameter_name) + "!") + fail(f"You can't have two parameters named {parameter_name!r}!") elif names and parameter_name == names[0] and c_name is None: - fail(f"Parameter '{parameter_name}' requires a custom C name") + fail(f"Parameter {parameter_name!r} requires a custom C name") key = f"{parameter_name}_as_{c_name}" if c_name else parameter_name self.function.parameters[key] = p @@ -5192,7 +5221,7 @@ def parse_converter( def parse_star(self, function: Function) -> None: """Parse keyword-only parameter marker '*'.""" if self.keyword_only: - fail(f"Function {function.name} uses '*' more than once.") + fail(f"Function {function.name!r} uses '*' more than once.") self.keyword_only = True def parse_opening_square_bracket(self, function: Function) -> None: @@ -5203,7 +5232,8 @@ def parse_opening_square_bracket(self, function: Function) -> None: case ParamState.REQUIRED | ParamState.GROUP_AFTER: self.parameter_state = ParamState.GROUP_AFTER case st: - fail(f"Function {function.name} has an unsupported group configuration. " + fail(f"Function {function.name!r} " + f"has an unsupported group configuration. " f"(Unexpected state {st}.b)") self.group += 1 function.docstring_only = True @@ -5211,9 +5241,9 @@ def parse_opening_square_bracket(self, function: Function) -> None: def parse_closing_square_bracket(self, function: Function) -> None: """Parse closing parameter group symbol ']'.""" if not self.group: - fail(f"Function {function.name} has a ] without a matching [.") + fail(f"Function {function.name!r} has a ']' without a matching '['.") if not any(p.group == self.group for p in function.parameters.values()): - fail(f"Function {function.name} has an empty group.\n" + fail(f"Function {function.name!r} has an empty group. " "All groups must contain at least one parameter.") self.group -= 1 match self.parameter_state: @@ -5222,13 +5252,14 @@ def parse_closing_square_bracket(self, function: Function) -> None: case ParamState.GROUP_AFTER | ParamState.RIGHT_SQUARE_AFTER: self.parameter_state = ParamState.RIGHT_SQUARE_AFTER case st: - fail(f"Function {function.name} has an unsupported group configuration. " + fail(f"Function {function.name!r} " + f"has an unsupported group configuration. " f"(Unexpected state {st}.c)") def parse_slash(self, function: Function) -> None: """Parse positional-only parameter marker '/'.""" if self.positional_only: - fail(f"Function {function.name} uses '/' more than once.") + fail(f"Function {function.name!r} uses '/' more than once.") self.positional_only = True # REQUIRED and OPTIONAL are allowed here, that allows positional-only # without option groups to work (and have default values!) @@ -5239,10 +5270,10 @@ def parse_slash(self, function: Function) -> None: ParamState.GROUP_BEFORE, } if (self.parameter_state not in allowed) or self.group: - fail(f"Function {function.name} has an unsupported group configuration. " + fail(f"Function {function.name!r} has an unsupported group configuration. " f"(Unexpected state {self.parameter_state}.d)") if self.keyword_only: - fail(f"Function {function.name} mixes keyword-only and " + fail(f"Function {function.name!r} mixes keyword-only and " "positional-only parameters, which is unsupported.") # fixup preceding parameters for p in function.parameters.values(): @@ -5251,7 +5282,7 @@ def parse_slash(self, function: Function) -> None: if (p.kind is not inspect.Parameter.POSITIONAL_OR_KEYWORD and not isinstance(p.converter, self_converter) ): - fail(f"Function {function.name} mixes keyword-only and " + fail(f"Function {function.name!r} mixes keyword-only and " "positional-only parameters, which is unsupported.") p.kind = inspect.Parameter.POSITIONAL_ONLY @@ -5301,7 +5332,7 @@ def state_function_docstring(self, line: str) -> None: assert self.function is not None if self.group: - fail("Function " + self.function.name + " has a ] without a matching [.") + fail(f"Function {self.function.name!r} has a ']' without a matching '['.") if not self.valid_line(line): return @@ -5528,9 +5559,9 @@ def add_parameter(text: str) -> None: if len(lines) >= 2: if lines[1]: - fail("Docstring for " + f.full_name + " does not have a summary line!\n" + - "Every non-blank function docstring must start with\n" + - "a single line summary followed by an empty line.") + fail(f"Docstring for {f.full_name!r} does not have a summary line!\n" + "Every non-blank function docstring must start with " + "a single line summary followed by an empty line.") elif len(lines) == 1: # the docstring is only one line right now--the summary line. # add an empty line after the summary line so we have space @@ -5573,7 +5604,8 @@ def do_post_block_processing_cleanup(self) -> None: last_parameter = next(reversed(list(values))) no_parameter_after_star = last_parameter.kind != inspect.Parameter.KEYWORD_ONLY if no_parameter_after_star: - fail("Function " + self.function.name + " specifies '*' without any parameters afterwards.") + fail(f"Function {self.function.name!r} specifies '*' " + "without any parameters afterwards.") self.function.docstring = self.format_docstring() From webhook-mailer at python.org Fri Aug 4 09:28:17 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Fri, 04 Aug 2023 13:28:17 -0000 Subject: [Python-checkins] gh-104146: Argument clinic: Remove dead code flagged by mypy's `truthy-bool` check (#107627) Message-ID: https://github.com/python/cpython/commit/7eba274fb3fb959261c6b8fdf0cee10c82004f0e commit: 7eba274fb3fb959261c6b8fdf0cee10c82004f0e branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-04T13:28:13Z summary: gh-104146: Argument clinic: Remove dead code flagged by mypy's `truthy-bool` check (#107627) files: M Tools/clinic/clinic.py M Tools/clinic/mypy.ini diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index ee5e0ef70f20f..6c5a2c7c857c5 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -886,9 +886,7 @@ def output_templates( converters = [p.converter for p in parameters] has_option_groups = parameters and (parameters[0].group or parameters[-1].group) - default_return_converter = (not f.return_converter or - f.return_converter.type == 'PyObject *') - + default_return_converter = f.return_converter.type == 'PyObject *' new_or_init = f.kind.new_or_init vararg: int | str = NO_VARARG @@ -4428,7 +4426,7 @@ class DSLParser: keyword_only: bool positional_only: bool group: int - parameter_state: int + parameter_state: ParamState seen_positional_with_default: bool indent: IndentStack kind: FunctionKind @@ -4790,8 +4788,6 @@ def state_modulename_name(self, line: str) -> None: if not return_converter: return_converter = CReturnConverter() - if not module: - fail("Undefined module used in declaration of {full_name.strip()!r}.") self.function = Function(name=function_name, full_name=full_name, module=module, cls=cls, c_basename=c_basename, return_converter=return_converter, kind=self.kind, coexist=self.coexist) self.block.signatures.append(self.function) diff --git a/Tools/clinic/mypy.ini b/Tools/clinic/mypy.ini index 4cfc05bec0160..b1fdad673c61a 100644 --- a/Tools/clinic/mypy.ini +++ b/Tools/clinic/mypy.ini @@ -8,5 +8,5 @@ python_version = 3.10 # and be strict! strict = True strict_concatenate = True -enable_error_code = ignore-without-code,redundant-expr +enable_error_code = ignore-without-code,redundant-expr,truthy-bool warn_unreachable = True From webhook-mailer at python.org Fri Aug 4 09:41:07 2023 From: webhook-mailer at python.org (Yhg1s) Date: Fri, 04 Aug 2023 13:41:07 -0000 Subject: [Python-checkins] [3.12] Update the expected bytecode magic in test_importlib.test_util (#107626) Message-ID: https://github.com/python/cpython/commit/310e1460c08d71447305e381e4fcd25af6519aeb commit: 310e1460c08d71447305e381e4fcd25af6519aeb branch: 3.12 author: T. Wouters committer: Yhg1s date: 2023-08-04T13:41:03Z summary: [3.12] Update the expected bytecode magic in test_importlib.test_util (#107626) Update the expected bytecode magic in test_importlib.test_util to the final 3.12 magic number. From now on it's not allowed to change for any 3.12 release. files: M Lib/test/test_importlib/test_util.py diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index e967adc9451c8..217c1ad78be25 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -634,7 +634,7 @@ def test_magic_number(self): # stakeholders such as OS package maintainers must be notified # in advance. Such exceptional releases will then require an # adjustment to this test case. - EXPECTED_MAGIC_NUMBER = 3495 + EXPECTED_MAGIC_NUMBER = 3531 actual = int.from_bytes(importlib.util.MAGIC_NUMBER[:2], 'little') msg = ( From webhook-mailer at python.org Fri Aug 4 10:16:14 2023 From: webhook-mailer at python.org (hugovk) Date: Fri, 04 Aug 2023 14:16:14 -0000 Subject: [Python-checkins] Docs: upgrade to python-docs-theme 2023.7 (#107617) Message-ID: https://github.com/python/cpython/commit/19f32b24b2e1680ff9929bb64d681397b259c6fb commit: 19f32b24b2e1680ff9929bb64d681397b259c6fb branch: main author: Hugo van Kemenade committer: hugovk date: 2023-08-04T14:16:10Z summary: Docs: upgrade to python-docs-theme 2023.7 (#107617) files: M Doc/requirements.txt diff --git a/Doc/requirements.txt b/Doc/requirements.txt index bde509febf5bd..e9b6e56b12289 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -7,7 +7,7 @@ # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. # PR #104777: Sphinx 6.2 no longer uses imghdr, removed in Python 3.13. -sphinx==6.2.0 +sphinx==6.2.1 blurb @@ -15,6 +15,6 @@ sphinxext-opengraph==0.7.5 # The theme used by the documentation is stored separately, so we need # to install that as well. -python-docs-theme>=2022.1 +python-docs-theme>=2023.7 -c constraints.txt From webhook-mailer at python.org Fri Aug 4 10:34:08 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Fri, 04 Aug 2023 14:34:08 -0000 Subject: [Python-checkins] gh-107600: Docs: Update ctypes.ArgumentError error message (#107601) Message-ID: https://github.com/python/cpython/commit/09a8cc79846ce0870e51fa8e7c449e153832fe4b commit: 09a8cc79846ce0870e51fa8e7c449e153832fe4b branch: main author: Tomas R committer: erlend-aasland date: 2023-08-04T16:34:04+02:00 summary: gh-107600: Docs: Update ctypes.ArgumentError error message (#107601) files: M Doc/library/ctypes.rst diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index c253a45e1a8b5..ea7873a885172 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -361,7 +361,7 @@ from within *IDLE* or *PythonWin*:: >>> printf(b"%f bottles of beer\n", 42.5) Traceback (most recent call last): File "", line 1, in - ArgumentError: argument 2: TypeError: Don't know how to convert parameter 2 + ctypes.ArgumentError: argument 2: TypeError: Don't know how to convert parameter 2 >>> As has been mentioned before, all Python types except integers, strings, and @@ -444,7 +444,7 @@ prototype for a C function), and tries to convert the arguments to valid types:: >>> printf(b"%d %d %d", 1, 2, 3) Traceback (most recent call last): File "", line 1, in - ArgumentError: argument 2: TypeError: wrong type + ctypes.ArgumentError: argument 2: TypeError: 'int' object cannot be interpreted as ctypes.c_char_p >>> printf(b"%s %d %f\n", b"X", 2, 3) X 2 3.000000 13 From webhook-mailer at python.org Fri Aug 4 10:56:54 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Fri, 04 Aug 2023 14:56:54 -0000 Subject: [Python-checkins] gh-104146: Argument clinic: remove dead code highlighted by the `vulture` tool (#107632) Message-ID: https://github.com/python/cpython/commit/407d7fda94268d70b92e2e685ca9891ec3641e78 commit: 407d7fda94268d70b92e2e685ca9891ec3641e78 branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-04T14:56:50Z summary: gh-104146: Argument clinic: remove dead code highlighted by the `vulture` tool (#107632) files: M Tools/clinic/clinic.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 6c5a2c7c857c5..6eb2c550e696f 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1684,27 +1684,30 @@ class Block: found on the start line of the block between the square brackets. - signatures is either list or None. If it's a list, - it may only contain clinic.Module, clinic.Class, and + signatures is a list. + It may only contain clinic.Module, clinic.Class, and clinic.Function objects. At the moment it should contain at most one of each. output is either str or None. If str, it's the output from this block, with embedded '\n' characters. - indent is either str or None. It's the leading whitespace + indent is a str. It's the leading whitespace that was found on every line of input. (If body_prefix is not empty, this is the indent *after* removing the body_prefix.) - preindent is either str or None. It's the whitespace that + "indent" is different from the concept of "preindent" + (which is not stored as state on Block objects). + "preindent" is the whitespace that was found in front of every line of input *before* the "body_prefix" (see the Language object). If body_prefix is empty, preindent must always be empty too. - To illustrate indent and preindent: Assume that '_' - represents whitespace. If the block processed was in a - Python file, and looked like this: + To illustrate the difference between "indent" and "preindent": + + Assume that '_' represents whitespace. + If the block processed was in a Python file, and looked like this: ____#/*[python] ____#__for a in range(20): ____#____print(a) @@ -1717,7 +1720,6 @@ class Block: signatures: list[Module | Class | Function] = dc.field(default_factory=list) output: Any = None # TODO: Very dynamic; probably untypeable in its current form? indent: str = '' - preindent: str = '' def __repr__(self) -> str: dsl_name = self.dsl_name or "text" @@ -2049,12 +2051,8 @@ def dump(self) -> str: return self.buffers.dump() -# maps strings to Language objects. -# "languages" maps the name of the language ("C", "Python"). -# "extensions" maps the file extension ("c", "py"). +# "extensions" maps the file extension ("c", "py") to Language classes. LangDict = dict[str, Callable[[str], Language]] - -languages = { 'C': CLanguage, 'Python': PythonLanguage } extensions: LangDict = { name: CLanguage for name in "c cc cpp cxx h hh hpp hxx".split() } extensions['py'] = PythonLanguage @@ -4427,7 +4425,6 @@ class DSLParser: positional_only: bool group: int parameter_state: ParamState - seen_positional_with_default: bool indent: IndentStack kind: FunctionKind coexist: bool @@ -4458,7 +4455,6 @@ def reset(self) -> None: self.positional_only = False self.group = 0 self.parameter_state: ParamState = ParamState.START - self.seen_positional_with_default = False self.indent = IndentStack() self.kind = CALLABLE self.coexist = False From webhook-mailer at python.org Fri Aug 4 12:36:00 2023 From: webhook-mailer at python.org (gvanrossum) Date: Fri, 04 Aug 2023 16:36:00 -0000 Subject: [Python-checkins] gh-106812: Refactor cases_generator to allow uops with array stack effects (#107564) Message-ID: https://github.com/python/cpython/commit/400835ea1626c8c6dcd967c7eabe0dad4a923182 commit: 400835ea1626c8c6dcd967c7eabe0dad4a923182 branch: main author: Guido van Rossum committer: gvanrossum date: 2023-08-04T09:35:56-07:00 summary: gh-106812: Refactor cases_generator to allow uops with array stack effects (#107564) Introducing a new file, stacking.py, that takes over several responsibilities related to symbolic evaluation of push/pop operations, with more generality. files: A Tools/cases_generator/stacking.py M Include/internal/pycore_opcode_metadata.h M Lib/test/test_generated_cases.py M Python/executor_cases.c.h M Python/generated_cases.c.h M Tools/cases_generator/analysis.py M Tools/cases_generator/flags.py M Tools/cases_generator/formatting.py M Tools/cases_generator/generate_cases.py M Tools/cases_generator/instructions.py M Tools/cases_generator/parsing.py M Tools/cases_generator/plexer.py diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 64a084618e13b..1cab6c984f3ac 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -679,9 +679,9 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case LOAD_GLOBAL: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_GLOBAL_MODULE: - return ((oparg & 1) ? 1 : 0) + 1; + return (oparg & 1 ? 1 : 0) + 1; case LOAD_GLOBAL_BUILTIN: - return ((oparg & 1) ? 1 : 0) + 1; + return (oparg & 1 ? 1 : 0) + 1; case DELETE_FAST: return 0; case MAKE_CELL: @@ -739,7 +739,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case LOAD_METHOD: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_INSTANCE_VALUE: - return ((oparg & 1) ? 1 : 0) + 1; + return (oparg & 1 ? 1 : 0) + 1; case LOAD_ATTR_MODULE: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_WITH_HINT: @@ -944,7 +944,18 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { } #endif -enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC0, INSTR_FMT_IXC00, INSTR_FMT_IXC000 }; +enum InstructionFormat { + INSTR_FMT_IB, + INSTR_FMT_IBC, + INSTR_FMT_IBC00, + INSTR_FMT_IBC000, + INSTR_FMT_IBC00000000, + INSTR_FMT_IX, + INSTR_FMT_IXC, + INSTR_FMT_IXC0, + INSTR_FMT_IXC00, + INSTR_FMT_IXC000, +}; #define IS_VALID_OPCODE(OP) \ (((OP) >= 0) && ((OP) < OPCODE_METADATA_SIZE) && \ diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 9c23c2e82058c..54378fced5469 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -6,9 +6,9 @@ test_tools.skip_if_missing('cases_generator') with test_tools.imports_under_tool('cases_generator'): + import generate_cases import analysis import formatting - import generate_cases from parsing import StackEffect @@ -46,28 +46,11 @@ def test_effect_sizes(self): (2, "(oparg<<1)"), ) - self.assertEqual( - formatting.string_effect_size( - formatting.list_effect_size(input_effects), - ), "1 + oparg + oparg*2", - ) - self.assertEqual( - formatting.string_effect_size( - formatting.list_effect_size(output_effects), - ), - "2 + oparg*4", - ) - self.assertEqual( - formatting.string_effect_size( - formatting.list_effect_size(other_effects), - ), - "2 + (oparg<<1)", - ) - class TestGeneratedCases(unittest.TestCase): def setUp(self) -> None: super().setUp() + self.maxDiff = None self.temp_dir = tempfile.gettempdir() self.temp_input_filename = os.path.join(self.temp_dir, "input.txt") @@ -140,7 +123,8 @@ def test_inst_one_pop(self): """ output = """ TARGET(OP) { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; spam(); STACK_SHRINK(1); DISPATCH(); @@ -173,8 +157,9 @@ def test_inst_one_push_one_pop(self): """ output = """ TARGET(OP) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; spam(); stack_pointer[-1] = res; DISPATCH(); @@ -190,9 +175,11 @@ def test_binary_op(self): """ output = """ TARGET(OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; spam(); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -209,9 +196,11 @@ def test_overlap(self): """ output = """ TARGET(OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *result; + right = stack_pointer[-1]; + left = stack_pointer[-2]; spam(); stack_pointer[-1] = result; DISPATCH(); @@ -235,8 +224,9 @@ def test_predictions_and_eval_breaker(self): } TARGET(OP3) { - PyObject *arg = stack_pointer[-1]; + PyObject *arg; PyObject *res; + arg = stack_pointer[-1]; DEOPT_IF(xxx, OP1); stack_pointer[-1] = res; CHECK_EVAL_BREAKER(); @@ -281,9 +271,11 @@ def test_error_if_pop(self): """ output = """ TARGET(OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; if (cond) goto pop_2_label; STACK_SHRINK(1); stack_pointer[-1] = res; @@ -299,7 +291,8 @@ def test_cache_effect(self): """ output = """ TARGET(OP) { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; uint16_t counter = read_u16(&next_instr[0].cache); uint32_t extra = read_u32(&next_instr[1].cache); STACK_SHRINK(1); @@ -338,8 +331,10 @@ def test_macro_instruction(self): """ output = """ TARGET(OP1) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); op1(left, right); next_instr += 1; @@ -347,38 +342,38 @@ def test_macro_instruction(self): } TARGET(OP) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; - PyObject *_tmp_3 = stack_pointer[-3]; + static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); + PyObject *right; + PyObject *left; + PyObject *arg2; + PyObject *res; + // OP1 + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; uint16_t counter = read_u16(&next_instr[0].cache); op1(left, right); - _tmp_2 = left; - _tmp_1 = right; } + // OP2 + arg2 = stack_pointer[-3]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *arg2 = _tmp_3; - PyObject *res; uint32_t extra = read_u32(&next_instr[3].cache); res = op2(arg2, left, right); - _tmp_3 = res; } - next_instr += 5; - static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); STACK_SHRINK(2); - stack_pointer[-1] = _tmp_3; + stack_pointer[-1] = res; + next_instr += 5; DISPATCH(); } TARGET(OP3) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; - PyObject *arg2 = stack_pointer[-3]; + PyObject *right; + PyObject *left; + PyObject *arg2; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + arg2 = stack_pointer[-3]; res = op3(arg2, left, right); STACK_SHRINK(2); stack_pointer[-1] = res; @@ -396,9 +391,12 @@ def test_array_input(self): """ output = """ TARGET(OP) { - PyObject *above = stack_pointer[-1]; - PyObject **values = (stack_pointer - (1 + oparg*2)); - PyObject *below = stack_pointer[-(2 + oparg*2)]; + PyObject *above; + PyObject **values; + PyObject *below; + above = stack_pointer[-1]; + values = stack_pointer - 1 - oparg*2; + below = stack_pointer[-2 - oparg*2]; spam(); STACK_SHRINK(oparg*2); STACK_SHRINK(2); @@ -416,12 +414,13 @@ def test_array_output(self): output = """ TARGET(OP) { PyObject *below; - PyObject **values = stack_pointer - (2) + 1; + PyObject **values; PyObject *above; + values = stack_pointer - 1; spam(values, oparg); STACK_GROW(oparg*3); + stack_pointer[-2 - oparg*3] = below; stack_pointer[-1] = above; - stack_pointer[-(2 + oparg*3)] = below; DISPATCH(); } """ @@ -435,8 +434,9 @@ def test_array_input_output(self): """ output = """ TARGET(OP) { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *above; + values = stack_pointer - oparg; spam(values, oparg); STACK_GROW(1); stack_pointer[-1] = above; @@ -453,8 +453,10 @@ def test_array_error_if(self): """ output = """ TARGET(OP) { - PyObject **values = (stack_pointer - oparg); - PyObject *extra = stack_pointer[-(1 + oparg)]; + PyObject **values; + PyObject *extra; + values = stack_pointer - oparg; + extra = stack_pointer[-1 - oparg]; if (oparg == 0) { STACK_SHRINK(oparg); goto pop_1_somewhere; } STACK_SHRINK(oparg); STACK_SHRINK(1); @@ -471,18 +473,21 @@ def test_cond_effect(self): """ output = """ TARGET(OP) { - PyObject *cc = stack_pointer[-1]; - PyObject *input = ((oparg & 1) == 1) ? stack_pointer[-(1 + (((oparg & 1) == 1) ? 1 : 0))] : NULL; - PyObject *aa = stack_pointer[-(2 + (((oparg & 1) == 1) ? 1 : 0))]; + PyObject *cc; + PyObject *input = NULL; + PyObject *aa; PyObject *xx; PyObject *output = NULL; PyObject *zz; + cc = stack_pointer[-1]; + if ((oparg & 1) == 1) { input = stack_pointer[-1 - ((oparg & 1) == 1 ? 1 : 0)]; } + aa = stack_pointer[-2 - ((oparg & 1) == 1 ? 1 : 0)]; output = spam(oparg, input); STACK_SHRINK((((oparg & 1) == 1) ? 1 : 0)); STACK_GROW(((oparg & 2) ? 1 : 0)); + stack_pointer[-2 - (oparg & 2 ? 1 : 0)] = xx; + if (oparg & 2) { stack_pointer[-1 - (oparg & 2 ? 1 : 0)] = output; } stack_pointer[-1] = zz; - if (oparg & 2) { stack_pointer[-(1 + ((oparg & 2) ? 1 : 0))] = output; } - stack_pointer[-(2 + ((oparg & 2) ? 1 : 0))] = xx; DISPATCH(); } """ @@ -500,29 +505,28 @@ def test_macro_cond_effect(self): """ output = """ TARGET(M) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; - PyObject *_tmp_3 = stack_pointer[-3]; + PyObject *right; + PyObject *middle; + PyObject *left; + PyObject *deep; + PyObject *extra = NULL; + PyObject *res; + // A + right = stack_pointer[-1]; + middle = stack_pointer[-2]; + left = stack_pointer[-3]; { - PyObject *right = _tmp_1; - PyObject *middle = _tmp_2; - PyObject *left = _tmp_3; # Body of A } + // B { - PyObject *deep; - PyObject *extra = NULL; - PyObject *res; # Body of B - _tmp_3 = deep; - if (oparg) { _tmp_2 = extra; } - _tmp_1 = res; } STACK_SHRINK(1); STACK_GROW((oparg ? 1 : 0)); - stack_pointer[-1] = _tmp_1; - if (oparg) { stack_pointer[-2] = _tmp_2; } - stack_pointer[-3] = _tmp_3; + stack_pointer[-2 - (oparg ? 1 : 0)] = deep; + if (oparg) { stack_pointer[-1 - (oparg ? 1 : 0)] = extra; } + stack_pointer[-1] = res; DISPATCH(); } """ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index ed7cf4c15ec4a..9363b4955087d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -47,14 +47,16 @@ } case STORE_FAST: { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; SETLOCAL(oparg, value); STACK_SHRINK(1); break; } case POP_TOP: { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; Py_DECREF(value); STACK_SHRINK(1); break; @@ -69,8 +71,10 @@ } case END_SEND: { - PyObject *value = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; Py_DECREF(receiver); STACK_SHRINK(1); stack_pointer[-1] = value; @@ -78,8 +82,9 @@ } case UNARY_NEGATIVE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; res = PyNumber_Negative(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -88,8 +93,9 @@ } case UNARY_NOT: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; assert(PyBool_Check(value)); res = Py_IsFalse(value) ? Py_True : Py_False; stack_pointer[-1] = res; @@ -98,8 +104,9 @@ case TO_BOOL: { static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -119,15 +126,17 @@ } case TO_BOOL_BOOL: { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; DEOPT_IF(!PyBool_Check(value), TO_BOOL); STAT_INC(TO_BOOL, hit); break; } case TO_BOOL_INT: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { @@ -143,8 +152,9 @@ } case TO_BOOL_LIST: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; @@ -154,8 +164,9 @@ } case TO_BOOL_NONE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: DEOPT_IF(!Py_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); @@ -165,8 +176,9 @@ } case TO_BOOL_STR: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { @@ -183,8 +195,9 @@ } case TO_BOOL_ALWAYS_TRUE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; uint32_t version = (uint32_t)operand; // This one is a bit weird, because we expect *some* failures: assert(version); @@ -197,8 +210,9 @@ } case UNARY_INVERT: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; res = PyNumber_Invert(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -207,17 +221,21 @@ } case _GUARD_BOTH_INT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); break; } case _BINARY_OP_MULTIPLY_INT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -229,9 +247,11 @@ } case _BINARY_OP_ADD_INT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -243,9 +263,11 @@ } case _BINARY_OP_SUBTRACT_INT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); @@ -257,17 +279,21 @@ } case _GUARD_BOTH_FLOAT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); break; } case _BINARY_OP_MULTIPLY_FLOAT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval * @@ -279,9 +305,11 @@ } case _BINARY_OP_ADD_FLOAT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval + @@ -293,9 +321,11 @@ } case _BINARY_OP_SUBTRACT_FLOAT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval - @@ -307,17 +337,21 @@ } case _GUARD_BOTH_UNICODE: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); break; } case _BINARY_OP_ADD_UNICODE: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; STAT_INC(BINARY_OP, hit); res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); @@ -330,9 +364,11 @@ case BINARY_SUBSCR: { static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + PyObject *sub; + PyObject *container; PyObject *res; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -353,10 +389,13 @@ } case BINARY_SLICE: { - PyObject *stop = stack_pointer[-1]; - PyObject *start = stack_pointer[-2]; - PyObject *container = stack_pointer[-3]; + PyObject *stop; + PyObject *start; + PyObject *container; PyObject *res; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -375,10 +414,14 @@ } case STORE_SLICE: { - PyObject *stop = stack_pointer[-1]; - PyObject *start = stack_pointer[-2]; - PyObject *container = stack_pointer[-3]; - PyObject *v = stack_pointer[-4]; + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -396,9 +439,11 @@ } case BINARY_SUBSCR_LIST_INT: { - PyObject *sub = stack_pointer[-1]; - PyObject *list = stack_pointer[-2]; + PyObject *sub; + PyObject *list; PyObject *res; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -418,9 +463,11 @@ } case BINARY_SUBSCR_TUPLE_INT: { - PyObject *sub = stack_pointer[-1]; - PyObject *tuple = stack_pointer[-2]; + PyObject *sub; + PyObject *tuple; PyObject *res; + sub = stack_pointer[-1]; + tuple = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -440,9 +487,11 @@ } case BINARY_SUBSCR_DICT: { - PyObject *sub = stack_pointer[-1]; - PyObject *dict = stack_pointer[-2]; + PyObject *sub; + PyObject *dict; PyObject *res; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -463,16 +512,20 @@ } case LIST_APPEND: { - PyObject *v = stack_pointer[-1]; - PyObject *list = stack_pointer[-(2 + (oparg-1))]; + PyObject *v; + PyObject *list; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; STACK_SHRINK(1); break; } case SET_ADD: { - PyObject *v = stack_pointer[-1]; - PyObject *set = stack_pointer[-(2 + (oparg-1))]; + PyObject *v; + PyObject *set; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; int err = PySet_Add(set, v); Py_DECREF(v); if (err) goto pop_1_error; @@ -482,9 +535,12 @@ case STORE_SUBSCR: { static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; - PyObject *v = stack_pointer[-3]; + PyObject *sub; + PyObject *container; + PyObject *v; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + v = stack_pointer[-3]; #if ENABLE_SPECIALIZATION _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -506,9 +562,12 @@ } case STORE_SUBSCR_LIST_INT: { - PyObject *sub = stack_pointer[-1]; - PyObject *list = stack_pointer[-2]; - PyObject *value = stack_pointer[-3]; + PyObject *sub; + PyObject *list; + PyObject *value; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + value = stack_pointer[-3]; DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -530,9 +589,12 @@ } case STORE_SUBSCR_DICT: { - PyObject *sub = stack_pointer[-1]; - PyObject *dict = stack_pointer[-2]; - PyObject *value = stack_pointer[-3]; + PyObject *sub; + PyObject *dict; + PyObject *value; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + value = stack_pointer[-3]; DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); @@ -543,8 +605,10 @@ } case DELETE_SUBSCR: { - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; /* del container[sub] */ int err = PyObject_DelItem(container, sub); Py_DECREF(container); @@ -555,8 +619,9 @@ } case CALL_INTRINSIC_1: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); Py_DECREF(value); @@ -566,9 +631,11 @@ } case CALL_INTRINSIC_2: { - PyObject *value1 = stack_pointer[-1]; - PyObject *value2 = stack_pointer[-2]; + PyObject *value1; + PyObject *value2; PyObject *res; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); Py_DECREF(value2); @@ -580,8 +647,9 @@ } case GET_AITER: { - PyObject *obj = stack_pointer[-1]; + PyObject *obj; PyObject *iter; + obj = stack_pointer[-1]; unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -617,8 +685,9 @@ } case GET_ANEXT: { - PyObject *aiter = stack_pointer[-1]; + PyObject *aiter; PyObject *awaitable; + aiter = stack_pointer[-1]; unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -667,8 +736,9 @@ } case GET_AWAITABLE: { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { @@ -697,7 +767,8 @@ } case POP_EXCEPT: { - PyObject *exc_value = stack_pointer[-1]; + PyObject *exc_value; + exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); STACK_SHRINK(1); @@ -726,7 +797,8 @@ } case STORE_NAME: { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; @@ -768,7 +840,8 @@ case UNPACK_SEQUENCE: { static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq = stack_pointer[-1]; + PyObject *seq; + seq = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -789,8 +862,10 @@ } case UNPACK_SEQUENCE_TWO_TUPLE: { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); @@ -804,8 +879,10 @@ } case UNPACK_SEQUENCE_TUPLE: { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -820,8 +897,10 @@ } case UNPACK_SEQUENCE_LIST: { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -836,7 +915,8 @@ } case UNPACK_EX: { - PyObject *seq = stack_pointer[-1]; + PyObject *seq; + seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); @@ -848,8 +928,10 @@ case STORE_ATTR: { static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner = stack_pointer[-1]; - PyObject *v = stack_pointer[-2]; + PyObject *owner; + PyObject *v; + owner = stack_pointer[-1]; + v = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -871,7 +953,8 @@ } case DELETE_ATTR: { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); @@ -881,7 +964,8 @@ } case STORE_GLOBAL: { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); @@ -920,8 +1004,9 @@ } case _LOAD_FROM_DICT_OR_GLOBALS: { - PyObject *mod_or_class_dict = stack_pointer[-1]; + PyObject *mod_or_class_dict; PyObject *v; + mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { Py_DECREF(mod_or_class_dict); @@ -1004,8 +1089,8 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } stack_pointer[-1] = v; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } break; } @@ -1040,8 +1125,8 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } break; } @@ -1058,8 +1143,8 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } break; } @@ -1085,8 +1170,9 @@ } case LOAD_FROM_DICT_OR_DEREF: { - PyObject *class_dict = stack_pointer[-1]; + PyObject *class_dict; PyObject *value; + class_dict = stack_pointer[-1]; PyObject *name; assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); @@ -1124,7 +1210,8 @@ } case STORE_DEREF: { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); @@ -1148,8 +1235,9 @@ } case BUILD_STRING: { - PyObject **pieces = (stack_pointer - oparg); + PyObject **pieces; PyObject *str; + pieces = stack_pointer - oparg; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); @@ -1162,8 +1250,9 @@ } case BUILD_TUPLE: { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *tup; + values = stack_pointer - oparg; tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); @@ -1173,8 +1262,9 @@ } case BUILD_LIST: { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *list; + values = stack_pointer - oparg; list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); @@ -1184,8 +1274,10 @@ } case LIST_EXTEND: { - PyObject *iterable = stack_pointer[-1]; - PyObject *list = stack_pointer[-(2 + (oparg-1))]; + PyObject *iterable; + PyObject *list; + iterable = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1206,8 +1298,10 @@ } case SET_UPDATE: { - PyObject *iterable = stack_pointer[-1]; - PyObject *set = stack_pointer[-(2 + (oparg-1))]; + PyObject *iterable; + PyObject *set; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; int err = _PySet_Update(set, iterable); Py_DECREF(iterable); if (err < 0) goto pop_1_error; @@ -1216,8 +1310,9 @@ } case BUILD_SET: { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *set; + values = stack_pointer - oparg; set = PySet_New(NULL); if (set == NULL) goto error; @@ -1239,8 +1334,9 @@ } case BUILD_MAP: { - PyObject **values = (stack_pointer - oparg*2); + PyObject **values; PyObject *map; + values = stack_pointer - oparg*2; map = _PyDict_FromItems( values, 2, values+1, 2, @@ -1300,9 +1396,11 @@ } case BUILD_CONST_KEY_MAP: { - PyObject *keys = stack_pointer[-1]; - PyObject **values = (stack_pointer - (1 + oparg)); + PyObject *keys; + PyObject **values; PyObject *map; + keys = stack_pointer[-1]; + values = stack_pointer - 1 - oparg; if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1323,7 +1421,8 @@ } case DICT_UPDATE: { - PyObject *update = stack_pointer[-1]; + PyObject *update; + update = stack_pointer[-1]; PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -1340,7 +1439,8 @@ } case DICT_MERGE: { - PyObject *update = stack_pointer[-1]; + PyObject *update; + update = stack_pointer[-1]; PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { @@ -1354,8 +1454,10 @@ } case MAP_ADD: { - PyObject *value = stack_pointer[-1]; - PyObject *key = stack_pointer[-2]; + PyObject *value; + PyObject *key; + value = stack_pointer[-1]; + key = stack_pointer[-2]; PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ @@ -1366,11 +1468,14 @@ } case LOAD_SUPER_ATTR_ATTR: { - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *res2 = NULL; PyObject *res; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -1383,17 +1488,20 @@ if (res == NULL) goto pop_3_error; STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } break; } case LOAD_SUPER_ATTR_METHOD: { - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *res2; PyObject *res; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -1417,16 +1525,17 @@ res2 = NULL; } STACK_SHRINK(1); - stack_pointer[-1] = res; stack_pointer[-2] = res2; + stack_pointer[-1] = res; break; } case LOAD_ATTR: { static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1473,13 +1582,14 @@ if (res == NULL) goto pop_1_error; } STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } break; } case _GUARD_TYPE_VERSION: { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)operand; PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); @@ -1488,7 +1598,8 @@ } case _CHECK_MANAGED_OBJECT_HAS_VALUES: { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); @@ -1497,9 +1608,10 @@ } case _LOAD_ATTR_INSTANCE_VALUE: { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; uint16_t index = (uint16_t)operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); res = _PyDictOrValues_GetValues(dorv)->values[index]; @@ -1509,16 +1621,18 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } break; } case COMPARE_OP: { static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1546,9 +1660,11 @@ } case COMPARE_OP_FLOAT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -1566,9 +1682,11 @@ } case COMPARE_OP_INT: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -1590,9 +1708,11 @@ } case COMPARE_OP_STR: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -1611,9 +1731,11 @@ } case IS_OP: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; int res = Py_Is(left, right) ^ oparg; Py_DECREF(left); Py_DECREF(right); @@ -1624,9 +1746,11 @@ } case CONTAINS_OP: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; int res = PySequence_Contains(right, left); Py_DECREF(left); Py_DECREF(right); @@ -1638,10 +1762,12 @@ } case CHECK_EG_MATCH: { - PyObject *match_type = stack_pointer[-1]; - PyObject *exc_value = stack_pointer[-2]; + PyObject *match_type; + PyObject *exc_value; PyObject *rest; PyObject *match; + match_type = stack_pointer[-1]; + exc_value = stack_pointer[-2]; if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { Py_DECREF(exc_value); Py_DECREF(match_type); @@ -1662,15 +1788,17 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - stack_pointer[-1] = match; stack_pointer[-2] = rest; + stack_pointer[-1] = match; break; } case CHECK_EXC_MATCH: { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { Py_DECREF(right); @@ -1685,8 +1813,9 @@ } case IS_NONE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *b; + value = stack_pointer[-1]; if (Py_IsNone(value)) { b = Py_True; } @@ -1699,8 +1828,9 @@ } case GET_LEN: { - PyObject *obj = stack_pointer[-1]; + PyObject *obj; PyObject *len_o; + obj = stack_pointer[-1]; // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; @@ -1712,10 +1842,13 @@ } case MATCH_CLASS: { - PyObject *names = stack_pointer[-1]; - PyObject *type = stack_pointer[-2]; - PyObject *subject = stack_pointer[-3]; + PyObject *names; + PyObject *type; + PyObject *subject; PyObject *attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); @@ -1736,8 +1869,9 @@ } case MATCH_MAPPING: { - PyObject *subject = stack_pointer[-1]; + PyObject *subject; PyObject *res; + subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; STACK_GROW(1); @@ -1746,8 +1880,9 @@ } case MATCH_SEQUENCE: { - PyObject *subject = stack_pointer[-1]; + PyObject *subject; PyObject *res; + subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; STACK_GROW(1); @@ -1756,9 +1891,11 @@ } case MATCH_KEYS: { - PyObject *keys = stack_pointer[-1]; - PyObject *subject = stack_pointer[-2]; + PyObject *keys; + PyObject *subject; PyObject *values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error; @@ -1768,8 +1905,9 @@ } case GET_ITER: { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); Py_DECREF(iterable); @@ -1779,8 +1917,9 @@ } case GET_YIELD_FROM_ITER: { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -1810,14 +1949,16 @@ } case _ITER_CHECK_LIST: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; + iter = stack_pointer[-1]; DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); break; } case _IS_ITER_EXHAUSTED_LIST: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *exhausted; + iter = stack_pointer[-1]; _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; @@ -1838,8 +1979,9 @@ } case _ITER_NEXT_LIST: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *next; + iter = stack_pointer[-1]; _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; @@ -1852,14 +1994,16 @@ } case _ITER_CHECK_TUPLE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; + iter = stack_pointer[-1]; DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); break; } case _IS_ITER_EXHAUSTED_TUPLE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *exhausted; + iter = stack_pointer[-1]; _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; @@ -1880,8 +2024,9 @@ } case _ITER_NEXT_TUPLE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *next; + iter = stack_pointer[-1]; _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; @@ -1894,15 +2039,17 @@ } case _ITER_CHECK_RANGE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; + iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); break; } case _IS_ITER_EXHAUSTED_RANGE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *exhausted; + iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); exhausted = r->len <= 0 ? Py_True : Py_False; @@ -1912,8 +2059,9 @@ } case _ITER_NEXT_RANGE: { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *next; + iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); @@ -1928,10 +2076,13 @@ } case WITH_EXCEPT_START: { - PyObject *val = stack_pointer[-1]; - PyObject *lasti = stack_pointer[-3]; - PyObject *exit_func = stack_pointer[-4]; + PyObject *val; + PyObject *lasti; + PyObject *exit_func; PyObject *res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_func = stack_pointer[-4]; /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -1963,8 +2114,9 @@ } case PUSH_EXC_INFO: { - PyObject *new_exc = stack_pointer[-1]; + PyObject *new_exc; PyObject *prev_exc; + new_exc = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -1975,16 +2127,19 @@ assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); STACK_GROW(1); - stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; + stack_pointer[-1] = new_exc; break; } case CALL_NO_KW_TYPE_1: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2001,10 +2156,13 @@ } case CALL_NO_KW_STR_1: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2023,10 +2181,13 @@ } case CALL_NO_KW_TUPLE_1: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2045,7 +2206,8 @@ } case EXIT_INIT_CHECK: { - PyObject *should_be_none = stack_pointer[-1]; + PyObject *should_be_none; + should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -2058,10 +2220,13 @@ } case CALL_NO_KW_BUILTIN_O: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; @@ -2097,10 +2262,13 @@ } case CALL_NO_KW_BUILTIN_FAST: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; @@ -2140,10 +2308,13 @@ } case CALL_NO_KW_LEN: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* len(o) */ int is_meth = method != NULL; @@ -2175,10 +2346,13 @@ } case CALL_NO_KW_ISINSTANCE: { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -2212,9 +2386,11 @@ } case CALL_NO_KW_METHOD_DESCRIPTOR_O: { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; @@ -2253,9 +2429,11 @@ } case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -2292,9 +2470,11 @@ } case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; @@ -2330,8 +2510,9 @@ } case MAKE_FUNCTION: { - PyObject *codeobj = stack_pointer[-1]; + PyObject *codeobj; PyObject *func; + codeobj = stack_pointer[-1]; PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -2348,8 +2529,10 @@ } case SET_FUNCTION_ATTRIBUTE: { - PyObject *func = stack_pointer[-1]; - PyObject *attr = stack_pointer[-2]; + PyObject *func; + PyObject *attr; + func = stack_pointer[-1]; + attr = stack_pointer[-2]; assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -2380,10 +2563,13 @@ } case BUILD_SLICE: { - PyObject *step = (oparg == 3) ? stack_pointer[-(((oparg == 3) ? 1 : 0))] : NULL; - PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; - PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; + PyObject *step = NULL; + PyObject *stop; + PyObject *start; PyObject *slice; + if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } + stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; + start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; slice = PySlice_New(start, stop, step); Py_DECREF(start); Py_DECREF(stop); @@ -2396,8 +2582,9 @@ } case CONVERT_VALUE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *result; + value = stack_pointer[-1]; convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; @@ -2409,8 +2596,9 @@ } case FORMAT_SIMPLE: { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { @@ -2426,9 +2614,11 @@ } case FORMAT_WITH_SPEC: { - PyObject *fmt_spec = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; + PyObject *fmt_spec; + PyObject *value; PyObject *res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); @@ -2439,8 +2629,9 @@ } case COPY: { - PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; + PyObject *bottom; PyObject *top; + bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = Py_NewRef(bottom); STACK_GROW(1); @@ -2450,9 +2641,11 @@ case BINARY_OP: { static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *rhs = stack_pointer[-1]; - PyObject *lhs = stack_pointer[-2]; + PyObject *rhs; + PyObject *lhs; PyObject *res; + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2476,16 +2669,19 @@ } case SWAP: { - PyObject *top = stack_pointer[-1]; - PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; + PyObject *top; + PyObject *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top; stack_pointer[-1] = bottom; - stack_pointer[-(2 + (oparg-2))] = top; break; } case _POP_JUMP_IF_FALSE: { - PyObject *flag = stack_pointer[-1]; + PyObject *flag; + flag = stack_pointer[-1]; if (Py_IsFalse(flag)) { pc = oparg; } @@ -2494,7 +2690,8 @@ } case _POP_JUMP_IF_TRUE: { - PyObject *flag = stack_pointer[-1]; + PyObject *flag; + flag = stack_pointer[-1]; if (Py_IsTrue(flag)) { pc = oparg; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 9fa549a120324..7250240ac2396 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -91,8 +91,8 @@ Py_INCREF(value1); Py_INCREF(value2); STACK_GROW(2); - stack_pointer[-1] = value2; stack_pointer[-2] = value1; + stack_pointer[-1] = value2; DISPATCH(); } @@ -106,15 +106,17 @@ } TARGET(STORE_FAST) { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; SETLOCAL(oparg, value); STACK_SHRINK(1); DISPATCH(); } TARGET(STORE_FAST_LOAD_FAST) { - PyObject *value1 = stack_pointer[-1]; + PyObject *value1; PyObject *value2; + value1 = stack_pointer[-1]; uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); @@ -125,8 +127,10 @@ } TARGET(STORE_FAST_STORE_FAST) { - PyObject *value1 = stack_pointer[-1]; - PyObject *value2 = stack_pointer[-2]; + PyObject *value1; + PyObject *value2; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; SETLOCAL(oparg1, value1); @@ -136,7 +140,8 @@ } TARGET(POP_TOP) { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; Py_DECREF(value); STACK_SHRINK(1); DISPATCH(); @@ -151,14 +156,15 @@ } TARGET(END_FOR) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *value; + // POP_TOP + value = stack_pointer[-1]; { - PyObject *value = _tmp_1; Py_DECREF(value); } + // POP_TOP + value = stack_pointer[-2]; { - PyObject *value = _tmp_2; Py_DECREF(value); } STACK_SHRINK(2); @@ -166,8 +172,10 @@ } TARGET(INSTRUMENTED_END_FOR) { - PyObject *value = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; /* Need to create a fake StopIteration error here, * to conform to PEP 380 */ if (PyGen_Check(receiver)) { @@ -184,8 +192,10 @@ } TARGET(END_SEND) { - PyObject *value = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; Py_DECREF(receiver); STACK_SHRINK(1); stack_pointer[-1] = value; @@ -193,8 +203,10 @@ } TARGET(INSTRUMENTED_END_SEND) { - PyObject *value = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *value; + PyObject *receiver; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) { PyErr_SetObject(PyExc_StopIteration, value); if (monitor_stop_iteration(tstate, frame, next_instr-1)) { @@ -209,8 +221,9 @@ } TARGET(UNARY_NEGATIVE) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; res = PyNumber_Negative(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -219,8 +232,9 @@ } TARGET(UNARY_NOT) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; assert(PyBool_Check(value)); res = Py_IsFalse(value) ? Py_True : Py_False; stack_pointer[-1] = res; @@ -230,8 +244,9 @@ TARGET(TO_BOOL) { PREDICTED(TO_BOOL); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyToBoolCache *cache = (_PyToBoolCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -252,7 +267,8 @@ } TARGET(TO_BOOL_BOOL) { - PyObject *value = stack_pointer[-1]; + PyObject *value; + value = stack_pointer[-1]; DEOPT_IF(!PyBool_Check(value), TO_BOOL); STAT_INC(TO_BOOL, hit); next_instr += 3; @@ -260,8 +276,9 @@ } TARGET(TO_BOOL_INT) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { @@ -278,8 +295,9 @@ } TARGET(TO_BOOL_LIST) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; @@ -290,8 +308,9 @@ } TARGET(TO_BOOL_NONE) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: DEOPT_IF(!Py_IsNone(value), TO_BOOL); STAT_INC(TO_BOOL, hit); @@ -302,8 +321,9 @@ } TARGET(TO_BOOL_STR) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { @@ -321,8 +341,9 @@ } TARGET(TO_BOOL_ALWAYS_TRUE) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; uint32_t version = read_u32(&next_instr[1].cache); // This one is a bit weird, because we expect *some* failures: assert(version); @@ -336,8 +357,9 @@ } TARGET(UNARY_INVERT) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; res = PyNumber_Invert(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; @@ -346,215 +368,192 @@ } TARGET(BINARY_OP_MULTIPLY_INT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_MULTIPLY_INT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_INT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_ADD_INT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_INT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_INT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_SUBTRACT_INT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (res == NULL) goto pop_2_error; - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_MULTIPLY_FLOAT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_MULTIPLY_FLOAT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval * ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_FLOAT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_ADD_FLOAT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval + ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_FLOAT) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_FLOAT + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_SUBTRACT_FLOAT { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res); - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_UNICODE) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + PyObject *res; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_ADD_UNICODE { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; - PyObject *res; STAT_INC(BINARY_OP, hit); res = PyUnicode_Concat(left, right); _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; - _tmp_2 = res; } - next_instr += 1; STACK_SHRINK(1); - stack_pointer[-1] = _tmp_2; + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - PyObject *_tmp_1 = stack_pointer[-1]; - PyObject *_tmp_2 = stack_pointer[-2]; + PyObject *right; + PyObject *left; + // _GUARD_BOTH_UNICODE + right = stack_pointer[-1]; + left = stack_pointer[-2]; { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); - _tmp_2 = left; - _tmp_1 = right; } + // _BINARY_OP_INPLACE_ADD_UNICODE { - PyObject *right = _tmp_1; - PyObject *left = _tmp_2; _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; assert(true_next.op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(true_next.op.arg); @@ -586,9 +585,11 @@ TARGET(BINARY_SUBSCR) { PREDICTED(BINARY_SUBSCR); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + PyObject *sub; + PyObject *container; PyObject *res; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -610,10 +611,13 @@ } TARGET(BINARY_SLICE) { - PyObject *stop = stack_pointer[-1]; - PyObject *start = stack_pointer[-2]; - PyObject *container = stack_pointer[-3]; + PyObject *stop; + PyObject *start; + PyObject *container; PyObject *res; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't // DECREF'ed container yet, and we still own slice. @@ -632,10 +636,14 @@ } TARGET(STORE_SLICE) { - PyObject *stop = stack_pointer[-1]; - PyObject *start = stack_pointer[-2]; - PyObject *container = stack_pointer[-3]; - PyObject *v = stack_pointer[-4]; + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -653,9 +661,11 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { - PyObject *sub = stack_pointer[-1]; - PyObject *list = stack_pointer[-2]; + PyObject *sub; + PyObject *list; PyObject *res; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); @@ -676,9 +686,11 @@ } TARGET(BINARY_SUBSCR_TUPLE_INT) { - PyObject *sub = stack_pointer[-1]; - PyObject *tuple = stack_pointer[-2]; + PyObject *sub; + PyObject *tuple; PyObject *res; + sub = stack_pointer[-1]; + tuple = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); @@ -699,9 +711,11 @@ } TARGET(BINARY_SUBSCR_DICT) { - PyObject *sub = stack_pointer[-1]; - PyObject *dict = stack_pointer[-2]; + PyObject *sub; + PyObject *dict; PyObject *res; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); @@ -723,8 +737,10 @@ } TARGET(BINARY_SUBSCR_GETITEM) { - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); @@ -747,19 +763,24 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + STACK_SHRINK(1); } TARGET(LIST_APPEND) { - PyObject *v = stack_pointer[-1]; - PyObject *list = stack_pointer[-(2 + (oparg-1))]; + PyObject *v; + PyObject *list; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; STACK_SHRINK(1); DISPATCH(); } TARGET(SET_ADD) { - PyObject *v = stack_pointer[-1]; - PyObject *set = stack_pointer[-(2 + (oparg-1))]; + PyObject *v; + PyObject *set; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; int err = PySet_Add(set, v); Py_DECREF(v); if (err) goto pop_1_error; @@ -770,9 +791,12 @@ TARGET(STORE_SUBSCR) { PREDICTED(STORE_SUBSCR); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; - PyObject *v = stack_pointer[-3]; + PyObject *sub; + PyObject *container; + PyObject *v; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + v = stack_pointer[-3]; #if ENABLE_SPECIALIZATION _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -795,9 +819,12 @@ } TARGET(STORE_SUBSCR_LIST_INT) { - PyObject *sub = stack_pointer[-1]; - PyObject *list = stack_pointer[-2]; - PyObject *value = stack_pointer[-3]; + PyObject *sub; + PyObject *list; + PyObject *value; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + value = stack_pointer[-3]; DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -820,9 +847,12 @@ } TARGET(STORE_SUBSCR_DICT) { - PyObject *sub = stack_pointer[-1]; - PyObject *dict = stack_pointer[-2]; - PyObject *value = stack_pointer[-3]; + PyObject *sub; + PyObject *dict; + PyObject *value; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + value = stack_pointer[-3]; DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); @@ -834,8 +864,10 @@ } TARGET(DELETE_SUBSCR) { - PyObject *sub = stack_pointer[-1]; - PyObject *container = stack_pointer[-2]; + PyObject *sub; + PyObject *container; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; /* del container[sub] */ int err = PyObject_DelItem(container, sub); Py_DECREF(container); @@ -846,8 +878,9 @@ } TARGET(CALL_INTRINSIC_1) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value); Py_DECREF(value); @@ -857,9 +890,11 @@ } TARGET(CALL_INTRINSIC_2) { - PyObject *value1 = stack_pointer[-1]; - PyObject *value2 = stack_pointer[-2]; + PyObject *value1; + PyObject *value2; PyObject *res; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); Py_DECREF(value2); @@ -871,7 +906,8 @@ } TARGET(RAISE_VARARGS) { - PyObject **args = (stack_pointer - oparg); + PyObject **args; + args = stack_pointer - oparg; PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -893,10 +929,12 @@ break; } if (true) { STACK_SHRINK(oparg); goto error; } + STACK_SHRINK(oparg); } TARGET(INTERPRETER_EXIT) { - PyObject *retval = stack_pointer[-1]; + PyObject *retval; + retval = stack_pointer[-1]; assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); /* Restore previous cframe and return. */ @@ -905,10 +943,12 @@ assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; + STACK_SHRINK(1); } TARGET(RETURN_VALUE) { - PyObject *retval = stack_pointer[-1]; + PyObject *retval; + retval = stack_pointer[-1]; STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -921,10 +961,12 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; + STACK_SHRINK(1); } TARGET(INSTRUMENTED_RETURN_VALUE) { - PyObject *retval = stack_pointer[-1]; + PyObject *retval; + retval = stack_pointer[-1]; int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, next_instr-1, retval); @@ -941,6 +983,7 @@ frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; + STACK_SHRINK(1); } TARGET(RETURN_CONST) { @@ -980,8 +1023,9 @@ } TARGET(GET_AITER) { - PyObject *obj = stack_pointer[-1]; + PyObject *obj; PyObject *iter; + obj = stack_pointer[-1]; unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -1017,8 +1061,9 @@ } TARGET(GET_ANEXT) { - PyObject *aiter = stack_pointer[-1]; + PyObject *aiter; PyObject *awaitable; + aiter = stack_pointer[-1]; unaryfunc getter = NULL; PyObject *next_iter = NULL; PyTypeObject *type = Py_TYPE(aiter); @@ -1067,8 +1112,9 @@ } TARGET(GET_AWAITABLE) { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; iter = _PyCoro_GetAwaitableIter(iterable); if (iter == NULL) { @@ -1099,9 +1145,11 @@ TARGET(SEND) { PREDICTED(SEND); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - PyObject *v = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *v; + PyObject *receiver; PyObject *retval; + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1154,8 +1202,10 @@ } TARGET(SEND_GEN) { - PyObject *v = stack_pointer[-1]; - PyObject *receiver = stack_pointer[-2]; + PyObject *v; + PyObject *receiver; + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; DEOPT_IF(tstate->interp->eval_frame, SEND); PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && @@ -1174,7 +1224,8 @@ } TARGET(INSTRUMENTED_YIELD_VALUE) { - PyObject *retval = stack_pointer[-1]; + PyObject *retval; + retval = stack_pointer[-1]; assert(frame != &entry_frame); assert(oparg >= 0); /* make the generator identify this as HAS_ARG */ PyGenObject *gen = _PyFrame_GetGenerator(frame); @@ -1195,7 +1246,8 @@ } TARGET(YIELD_VALUE) { - PyObject *retval = stack_pointer[-1]; + PyObject *retval; + retval = stack_pointer[-1]; // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -1215,7 +1267,8 @@ } TARGET(POP_EXCEPT) { - PyObject *exc_value = stack_pointer[-1]; + PyObject *exc_value; + exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); STACK_SHRINK(1); @@ -1223,8 +1276,10 @@ } TARGET(RERAISE) { - PyObject *exc = stack_pointer[-1]; - PyObject **values = (stack_pointer - (1 + oparg)); + PyObject *exc; + PyObject **values; + exc = stack_pointer[-1]; + values = stack_pointer - 1 - oparg; assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1243,11 +1298,14 @@ _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, next_instr-1); goto exception_unwind; + STACK_SHRINK(1); } TARGET(END_ASYNC_FOR) { - PyObject *exc = stack_pointer[-1]; - PyObject *awaitable = stack_pointer[-2]; + PyObject *exc; + PyObject *awaitable; + exc = stack_pointer[-1]; + awaitable = stack_pointer[-2]; assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { Py_DECREF(awaitable); @@ -1264,11 +1322,14 @@ } TARGET(CLEANUP_THROW) { - PyObject *exc_value = stack_pointer[-1]; - PyObject *last_sent_val = stack_pointer[-2]; - PyObject *sub_iter = stack_pointer[-3]; + PyObject *exc_value; + PyObject *last_sent_val; + PyObject *sub_iter; PyObject *none; PyObject *value; + exc_value = stack_pointer[-1]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; assert(throwflag); assert(exc_value && PyExceptionInstance_Check(exc_value)); if (PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration)) { @@ -1284,8 +1345,8 @@ goto exception_unwind; } STACK_SHRINK(1); - stack_pointer[-1] = value; stack_pointer[-2] = none; + stack_pointer[-1] = value; DISPATCH(); } @@ -1311,7 +1372,8 @@ } TARGET(STORE_NAME) { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; @@ -1354,7 +1416,8 @@ TARGET(UNPACK_SEQUENCE) { PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq = stack_pointer[-1]; + PyObject *seq; + seq = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1376,8 +1439,10 @@ } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); assert(oparg == 2); @@ -1392,8 +1457,10 @@ } TARGET(UNPACK_SEQUENCE_TUPLE) { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1409,8 +1476,10 @@ } TARGET(UNPACK_SEQUENCE_LIST) { - PyObject *seq = stack_pointer[-1]; - PyObject **values = stack_pointer - (1); + PyObject *seq; + PyObject **values; + seq = stack_pointer[-1]; + values = stack_pointer - 1; DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); STAT_INC(UNPACK_SEQUENCE, hit); @@ -1426,7 +1495,8 @@ } TARGET(UNPACK_EX) { - PyObject *seq = stack_pointer[-1]; + PyObject *seq; + seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); @@ -1439,8 +1509,10 @@ TARGET(STORE_ATTR) { PREDICTED(STORE_ATTR); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner = stack_pointer[-1]; - PyObject *v = stack_pointer[-2]; + PyObject *owner; + PyObject *v; + owner = stack_pointer[-1]; + v = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1463,7 +1535,8 @@ } TARGET(DELETE_ATTR) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_DelAttr(owner, name); Py_DECREF(owner); @@ -1473,7 +1546,8 @@ } TARGET(STORE_GLOBAL) { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); @@ -1498,27 +1572,25 @@ } TARGET(LOAD_LOCALS) { - PyObject *_tmp_1; - { - PyObject *locals; - locals = LOCALS(); - if (locals == NULL) { - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - if (true) goto error; - } - Py_INCREF(locals); - _tmp_1 = locals; + PyObject *locals; + locals = LOCALS(); + if (locals == NULL) { + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + if (true) goto error; } + Py_INCREF(locals); STACK_GROW(1); - stack_pointer[-1] = _tmp_1; + stack_pointer[-1] = locals; DISPATCH(); } TARGET(LOAD_NAME) { - PyObject *_tmp_1; + PyObject *locals; + PyObject *mod_or_class_dict; + PyObject *v; + // _LOAD_LOCALS { - PyObject *locals; locals = LOCALS(); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -1526,11 +1598,10 @@ if (true) goto error; } Py_INCREF(locals); - _tmp_1 = locals; } + // _LOAD_FROM_DICT_OR_GLOBALS + mod_or_class_dict = locals; { - PyObject *mod_or_class_dict = _tmp_1; - PyObject *v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { Py_DECREF(mod_or_class_dict); @@ -1557,47 +1628,43 @@ } } } - _tmp_1 = v; } STACK_GROW(1); - stack_pointer[-1] = _tmp_1; + stack_pointer[-1] = v; DISPATCH(); } TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - PyObject *_tmp_1 = stack_pointer[-1]; - { - PyObject *mod_or_class_dict = _tmp_1; - PyObject *v; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { - Py_DECREF(mod_or_class_dict); + PyObject *mod_or_class_dict; + PyObject *v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + Py_DECREF(mod_or_class_dict); + goto error; + } + Py_DECREF(mod_or_class_dict); + if (v == NULL) { + v = PyDict_GetItemWithError(GLOBALS(), name); + if (v != NULL) { + Py_INCREF(v); + } + else if (_PyErr_Occurred(tstate)) { goto error; } - Py_DECREF(mod_or_class_dict); - if (v == NULL) { - v = PyDict_GetItemWithError(GLOBALS(), name); - if (v != NULL) { - Py_INCREF(v); - } - else if (_PyErr_Occurred(tstate)) { + else { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { goto error; } - else { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - goto error; - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - goto error; - } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + goto error; } } - _tmp_1 = v; } - stack_pointer[-1] = _tmp_1; + stack_pointer[-1] = v; DISPATCH(); } @@ -1654,15 +1721,16 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } stack_pointer[-1] = v; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } next_instr += 4; DISPATCH(); } TARGET(LOAD_GLOBAL_MODULE) { - PyObject *_tmp_1; - PyObject *_tmp_2; + PyObject *null = NULL; + PyObject *res; + // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&next_instr[1].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -1670,9 +1738,8 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } + // _LOAD_GLOBAL_MODULE { - PyObject *null = NULL; - PyObject *res; uint16_t index = read_u16(&next_instr[3].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); @@ -1681,20 +1748,19 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - if (oparg & 1) { _tmp_2 = null; } - _tmp_1 = res; } - next_instr += 4; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = _tmp_1; - if (oparg & 1) { stack_pointer[-2] = _tmp_2; } + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = res; + next_instr += 4; DISPATCH(); } TARGET(LOAD_GLOBAL_BUILTIN) { - PyObject *_tmp_1; - PyObject *_tmp_2; + PyObject *null = NULL; + PyObject *res; + // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&next_instr[1].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -1702,6 +1768,7 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } + // _GUARD_BUILTINS_VERSION { uint16_t version = read_u16(&next_instr[2].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); @@ -1709,9 +1776,8 @@ DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } + // _LOAD_GLOBAL_BUILTINS { - PyObject *null = NULL; - PyObject *res; uint16_t index = read_u16(&next_instr[3].cache); PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); @@ -1720,14 +1786,12 @@ Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; - if (oparg & 1) { _tmp_2 = null; } - _tmp_1 = res; } - next_instr += 4; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = _tmp_1; - if (oparg & 1) { stack_pointer[-2] = _tmp_2; } + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } + stack_pointer[-1] = res; + next_instr += 4; DISPATCH(); } @@ -1765,8 +1829,9 @@ } TARGET(LOAD_FROM_DICT_OR_DEREF) { - PyObject *class_dict = stack_pointer[-1]; + PyObject *class_dict; PyObject *value; + class_dict = stack_pointer[-1]; PyObject *name; assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); @@ -1804,7 +1869,8 @@ } TARGET(STORE_DEREF) { - PyObject *v = stack_pointer[-1]; + PyObject *v; + v = stack_pointer[-1]; PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); @@ -1828,8 +1894,9 @@ } TARGET(BUILD_STRING) { - PyObject **pieces = (stack_pointer - oparg); + PyObject **pieces; PyObject *str; + pieces = stack_pointer - oparg; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int _i = oparg; --_i >= 0;) { Py_DECREF(pieces[_i]); @@ -1842,8 +1909,9 @@ } TARGET(BUILD_TUPLE) { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *tup; + values = stack_pointer - oparg; tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); @@ -1853,8 +1921,9 @@ } TARGET(BUILD_LIST) { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *list; + values = stack_pointer - oparg; list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); @@ -1864,8 +1933,10 @@ } TARGET(LIST_EXTEND) { - PyObject *iterable = stack_pointer[-1]; - PyObject *list = stack_pointer[-(2 + (oparg-1))]; + PyObject *iterable; + PyObject *list; + iterable = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1886,8 +1957,10 @@ } TARGET(SET_UPDATE) { - PyObject *iterable = stack_pointer[-1]; - PyObject *set = stack_pointer[-(2 + (oparg-1))]; + PyObject *iterable; + PyObject *set; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; int err = _PySet_Update(set, iterable); Py_DECREF(iterable); if (err < 0) goto pop_1_error; @@ -1896,8 +1969,9 @@ } TARGET(BUILD_SET) { - PyObject **values = (stack_pointer - oparg); + PyObject **values; PyObject *set; + values = stack_pointer - oparg; set = PySet_New(NULL); if (set == NULL) goto error; @@ -1919,8 +1993,9 @@ } TARGET(BUILD_MAP) { - PyObject **values = (stack_pointer - oparg*2); + PyObject **values; PyObject *map; + values = stack_pointer - oparg*2; map = _PyDict_FromItems( values, 2, values+1, 2, @@ -1980,9 +2055,11 @@ } TARGET(BUILD_CONST_KEY_MAP) { - PyObject *keys = stack_pointer[-1]; - PyObject **values = (stack_pointer - (1 + oparg)); + PyObject *keys; + PyObject **values; PyObject *map; + keys = stack_pointer[-1]; + values = stack_pointer - 1 - oparg; if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { _PyErr_SetString(tstate, PyExc_SystemError, @@ -2003,7 +2080,8 @@ } TARGET(DICT_UPDATE) { - PyObject *update = stack_pointer[-1]; + PyObject *update; + update = stack_pointer[-1]; PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -2020,7 +2098,8 @@ } TARGET(DICT_MERGE) { - PyObject *update = stack_pointer[-1]; + PyObject *update; + update = stack_pointer[-1]; PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { @@ -2034,8 +2113,10 @@ } TARGET(MAP_ADD) { - PyObject *value = stack_pointer[-1]; - PyObject *key = stack_pointer[-2]; + PyObject *value; + PyObject *key; + value = stack_pointer[-1]; + key = stack_pointer[-2]; PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ @@ -2051,16 +2132,21 @@ // don't want to specialize instrumented instructions INCREMENT_ADAPTIVE_COUNTER(cache->counter); GO_TO_INSTRUCTION(LOAD_SUPER_ATTR); + STACK_SHRINK(2); + STACK_GROW(((oparg & 1) ? 1 : 0)); } TARGET(LOAD_SUPER_ATTR) { PREDICTED(LOAD_SUPER_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *res2 = NULL; PyObject *res; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); int load_method = oparg & 1; #if ENABLE_SPECIALIZATION @@ -2111,18 +2197,21 @@ if (res == NULL) goto pop_3_error; STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 1; DISPATCH(); } TARGET(LOAD_SUPER_ATTR_ATTR) { - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *res2 = NULL; PyObject *res; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -2135,18 +2224,21 @@ if (res == NULL) goto pop_3_error; STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 1; DISPATCH(); } TARGET(LOAD_SUPER_ATTR_METHOD) { - PyObject *self = stack_pointer[-1]; - PyObject *class = stack_pointer[-2]; - PyObject *global_super = stack_pointer[-3]; + PyObject *self; + PyObject *class; + PyObject *global_super; PyObject *res2; PyObject *res; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -2170,8 +2262,8 @@ res2 = NULL; } STACK_SHRINK(1); - stack_pointer[-1] = res; stack_pointer[-2] = res2; + stack_pointer[-1] = res; next_instr += 1; DISPATCH(); } @@ -2179,9 +2271,10 @@ TARGET(LOAD_ATTR) { PREDICTED(LOAD_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2228,35 +2321,33 @@ if (res == NULL) goto pop_1_error; } STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - PyObject *_tmp_1; - PyObject *_tmp_2 = stack_pointer[-1]; + PyObject *owner; + PyObject *res2 = NULL; + PyObject *res; + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; { - PyObject *owner = _tmp_2; uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); - _tmp_2 = owner; } + // _CHECK_MANAGED_OBJECT_HAS_VALUES { - PyObject *owner = _tmp_2; assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - _tmp_2 = owner; } + // _LOAD_ATTR_INSTANCE_VALUE { - PyObject *owner = _tmp_2; - PyObject *res2 = NULL; - PyObject *res; uint16_t index = read_u16(&next_instr[3].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); res = _PyDictOrValues_GetValues(dorv)->values[index]; @@ -2265,20 +2356,19 @@ Py_INCREF(res); res2 = NULL; Py_DECREF(owner); - if (oparg & 1) { _tmp_2 = res2; } - _tmp_1 = res; } - next_instr += 9; STACK_GROW(((oparg & 1) ? 1 : 0)); - stack_pointer[-1] = _tmp_1; - if (oparg & 1) { stack_pointer[-2] = _tmp_2; } + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } + stack_pointer[-1] = res; + next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_MODULE) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); @@ -2295,16 +2385,17 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_WITH_HINT) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(owner); @@ -2335,16 +2426,17 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_SLOT) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; PyObject *res2 = NULL; PyObject *res; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(owner); @@ -2358,16 +2450,17 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_CLASS) { - PyObject *cls = stack_pointer[-1]; + PyObject *cls; PyObject *res2 = NULL; PyObject *res; + cls = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); @@ -2383,14 +2476,15 @@ Py_INCREF(res); Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); + if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } stack_pointer[-1] = res; - if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_PROPERTY) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); @@ -2417,10 +2511,12 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + STACK_GROW(((oparg & 1) ? 1 : 0)); } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - PyObject *owner = stack_pointer[-1]; + PyObject *owner; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); @@ -2449,11 +2545,14 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + STACK_GROW(((oparg & 1) ? 1 : 0)); } TARGET(STORE_ATTR_INSTANCE_VALUE) { - PyObject *owner = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; + PyObject *owner; + PyObject *value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(owner); @@ -2479,8 +2578,10 @@ } TARGET(STORE_ATTR_WITH_HINT) { - PyObject *owner = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; + PyObject *owner; + PyObject *value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(owner); @@ -2527,8 +2628,10 @@ } TARGET(STORE_ATTR_SLOT) { - PyObject *owner = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; + PyObject *owner; + PyObject *value; + owner = stack_pointer[-1]; + value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(owner); @@ -2548,9 +2651,11 @@ TARGET(COMPARE_OP) { PREDICTED(COMPARE_OP); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2579,9 +2684,11 @@ } TARGET(COMPARE_OP_FLOAT) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2600,9 +2707,11 @@ } TARGET(COMPARE_OP_INT) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); @@ -2625,9 +2734,11 @@ } TARGET(COMPARE_OP_STR) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); STAT_INC(COMPARE_OP, hit); @@ -2647,9 +2758,11 @@ } TARGET(IS_OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; int res = Py_Is(left, right) ^ oparg; Py_DECREF(left); Py_DECREF(right); @@ -2660,9 +2773,11 @@ } TARGET(CONTAINS_OP) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; int res = PySequence_Contains(right, left); Py_DECREF(left); Py_DECREF(right); @@ -2674,10 +2789,12 @@ } TARGET(CHECK_EG_MATCH) { - PyObject *match_type = stack_pointer[-1]; - PyObject *exc_value = stack_pointer[-2]; + PyObject *match_type; + PyObject *exc_value; PyObject *rest; PyObject *match; + match_type = stack_pointer[-1]; + exc_value = stack_pointer[-2]; if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { Py_DECREF(exc_value); Py_DECREF(match_type); @@ -2698,15 +2815,17 @@ if (!Py_IsNone(match)) { PyErr_SetHandledException(match); } - stack_pointer[-1] = match; stack_pointer[-2] = rest; + stack_pointer[-1] = match; DISPATCH(); } TARGET(CHECK_EXC_MATCH) { - PyObject *right = stack_pointer[-1]; - PyObject *left = stack_pointer[-2]; + PyObject *right; + PyObject *left; PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; assert(PyExceptionInstance_Check(left)); if (_PyEval_CheckExceptTypeValid(tstate, right) < 0) { Py_DECREF(right); @@ -2721,9 +2840,11 @@ } TARGET(IMPORT_NAME) { - PyObject *fromlist = stack_pointer[-1]; - PyObject *level = stack_pointer[-2]; + PyObject *fromlist; + PyObject *level; PyObject *res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_name(tstate, frame, name, fromlist, level); Py_DECREF(level); @@ -2735,8 +2856,9 @@ } TARGET(IMPORT_FROM) { - PyObject *from = stack_pointer[-1]; + PyObject *from; PyObject *res; + from = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; @@ -2797,7 +2919,8 @@ } TARGET(POP_JUMP_IF_FALSE) { - PyObject *cond = stack_pointer[-1]; + PyObject *cond; + cond = stack_pointer[-1]; assert(PyBool_Check(cond)); JUMPBY(oparg * Py_IsFalse(cond)); STACK_SHRINK(1); @@ -2805,7 +2928,8 @@ } TARGET(POP_JUMP_IF_TRUE) { - PyObject *cond = stack_pointer[-1]; + PyObject *cond; + cond = stack_pointer[-1]; assert(PyBool_Check(cond)); JUMPBY(oparg * Py_IsTrue(cond)); STACK_SHRINK(1); @@ -2813,10 +2937,12 @@ } TARGET(POP_JUMP_IF_NONE) { - PyObject *_tmp_1 = stack_pointer[-1]; + PyObject *value; + PyObject *b; + PyObject *cond; + // IS_NONE + value = stack_pointer[-1]; { - PyObject *value = _tmp_1; - PyObject *b; if (Py_IsNone(value)) { b = Py_True; } @@ -2824,10 +2950,10 @@ b = Py_False; Py_DECREF(value); } - _tmp_1 = b; } + // POP_JUMP_IF_TRUE + cond = b; { - PyObject *cond = _tmp_1; assert(PyBool_Check(cond)); JUMPBY(oparg * Py_IsTrue(cond)); } @@ -2836,10 +2962,12 @@ } TARGET(POP_JUMP_IF_NOT_NONE) { - PyObject *_tmp_1 = stack_pointer[-1]; + PyObject *value; + PyObject *b; + PyObject *cond; + // IS_NONE + value = stack_pointer[-1]; { - PyObject *value = _tmp_1; - PyObject *b; if (Py_IsNone(value)) { b = Py_True; } @@ -2847,10 +2975,10 @@ b = Py_False; Py_DECREF(value); } - _tmp_1 = b; } + // POP_JUMP_IF_FALSE + cond = b; { - PyObject *cond = _tmp_1; assert(PyBool_Check(cond)); JUMPBY(oparg * Py_IsFalse(cond)); } @@ -2869,8 +2997,9 @@ } TARGET(GET_LEN) { - PyObject *obj = stack_pointer[-1]; + PyObject *obj; PyObject *len_o; + obj = stack_pointer[-1]; // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); if (len_i < 0) goto error; @@ -2882,10 +3011,13 @@ } TARGET(MATCH_CLASS) { - PyObject *names = stack_pointer[-1]; - PyObject *type = stack_pointer[-2]; - PyObject *subject = stack_pointer[-3]; + PyObject *names; + PyObject *type; + PyObject *subject; PyObject *attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. assert(PyTuple_CheckExact(names)); @@ -2906,8 +3038,9 @@ } TARGET(MATCH_MAPPING) { - PyObject *subject = stack_pointer[-1]; + PyObject *subject; PyObject *res; + subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? Py_True : Py_False; STACK_GROW(1); @@ -2916,8 +3049,9 @@ } TARGET(MATCH_SEQUENCE) { - PyObject *subject = stack_pointer[-1]; + PyObject *subject; PyObject *res; + subject = stack_pointer[-1]; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? Py_True : Py_False; STACK_GROW(1); @@ -2926,9 +3060,11 @@ } TARGET(MATCH_KEYS) { - PyObject *keys = stack_pointer[-1]; - PyObject *subject = stack_pointer[-2]; + PyObject *keys; + PyObject *subject; PyObject *values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = _PyEval_MatchKeys(tstate, subject, keys); if (values_or_none == NULL) goto error; @@ -2938,8 +3074,9 @@ } TARGET(GET_ITER) { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); Py_DECREF(iterable); @@ -2949,8 +3086,9 @@ } TARGET(GET_YIELD_FROM_ITER) { - PyObject *iterable = stack_pointer[-1]; + PyObject *iterable; PyObject *iter; + iterable = stack_pointer[-1]; /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { /* `iterable` is a coroutine */ @@ -2982,8 +3120,9 @@ TARGET(FOR_ITER) { PREDICTED(FOR_ITER); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter = stack_pointer[-1]; + PyObject *iter; PyObject *next; + iter = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -3051,15 +3190,15 @@ } TARGET(FOR_ITER_LIST) { - PyObject *_tmp_1; - PyObject *_tmp_2 = stack_pointer[-1]; + PyObject *iter; + PyObject *next; + // _ITER_CHECK_LIST + iter = stack_pointer[-1]; { - PyObject *iter = _tmp_2; DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); - _tmp_2 = iter; } + // _ITER_JUMP_LIST { - PyObject *iter = _tmp_2; _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); STAT_INC(FOR_ITER, hit); @@ -3076,37 +3215,32 @@ JUMPBY(oparg + 1); DISPATCH(); } - _tmp_2 = iter; } + // _ITER_NEXT_LIST { - PyObject *iter = _tmp_2; - PyObject *next; _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyList_GET_SIZE(seq)); next = Py_NewRef(PyList_GET_ITEM(seq, it->it_index++)); - _tmp_2 = iter; - _tmp_1 = next; } - next_instr += 1; STACK_GROW(1); - stack_pointer[-1] = _tmp_1; - stack_pointer[-2] = _tmp_2; + stack_pointer[-1] = next; + next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_TUPLE) { - PyObject *_tmp_1; - PyObject *_tmp_2 = stack_pointer[-1]; + PyObject *iter; + PyObject *next; + // _ITER_CHECK_TUPLE + iter = stack_pointer[-1]; { - PyObject *iter = _tmp_2; DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); - _tmp_2 = iter; } + // _ITER_JUMP_TUPLE { - PyObject *iter = _tmp_2; _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); STAT_INC(FOR_ITER, hit); @@ -3123,38 +3257,33 @@ JUMPBY(oparg + 1); DISPATCH(); } - _tmp_2 = iter; } + // _ITER_NEXT_TUPLE { - PyObject *iter = _tmp_2; - PyObject *next; _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; assert(seq); assert(it->it_index < PyTuple_GET_SIZE(seq)); next = Py_NewRef(PyTuple_GET_ITEM(seq, it->it_index++)); - _tmp_2 = iter; - _tmp_1 = next; } - next_instr += 1; STACK_GROW(1); - stack_pointer[-1] = _tmp_1; - stack_pointer[-2] = _tmp_2; + stack_pointer[-1] = next; + next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_RANGE) { - PyObject *_tmp_1; - PyObject *_tmp_2 = stack_pointer[-1]; + PyObject *iter; + PyObject *next; + // _ITER_CHECK_RANGE + iter = stack_pointer[-1]; { - PyObject *iter = _tmp_2; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); - _tmp_2 = iter; } + // _ITER_JUMP_RANGE { - PyObject *iter = _tmp_2; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); STAT_INC(FOR_ITER, hit); @@ -3166,11 +3295,9 @@ JUMPBY(oparg + 1); DISPATCH(); } - _tmp_2 = iter; } + // _ITER_NEXT_RANGE { - PyObject *iter = _tmp_2; - PyObject *next; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); assert(r->len > 0); @@ -3179,18 +3306,16 @@ r->len--; next = PyLong_FromLong(value); if (next == NULL) goto error; - _tmp_2 = iter; - _tmp_1 = next; } - next_instr += 1; STACK_GROW(1); - stack_pointer[-1] = _tmp_1; - stack_pointer[-2] = _tmp_2; + stack_pointer[-1] = next; + next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_GEN) { - PyObject *iter = stack_pointer[-1]; + PyObject *iter; + iter = stack_pointer[-1]; DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); @@ -3206,12 +3331,14 @@ assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); DISPATCH_INLINED(gen_frame); + STACK_GROW(1); } TARGET(BEFORE_ASYNC_WITH) { - PyObject *mgr = stack_pointer[-1]; + PyObject *mgr; PyObject *exit; PyObject *res; + mgr = stack_pointer[-1]; PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); if (enter == NULL) { if (!_PyErr_Occurred(tstate)) { @@ -3242,15 +3369,16 @@ if (true) goto pop_1_error; } STACK_GROW(1); - stack_pointer[-1] = res; stack_pointer[-2] = exit; + stack_pointer[-1] = res; DISPATCH(); } TARGET(BEFORE_WITH) { - PyObject *mgr = stack_pointer[-1]; + PyObject *mgr; PyObject *exit; PyObject *res; + mgr = stack_pointer[-1]; /* pop the context manager, push its __exit__ and the * value returned from calling its __enter__ */ @@ -3284,16 +3412,19 @@ if (true) goto pop_1_error; } STACK_GROW(1); - stack_pointer[-1] = res; stack_pointer[-2] = exit; + stack_pointer[-1] = res; DISPATCH(); } TARGET(WITH_EXCEPT_START) { - PyObject *val = stack_pointer[-1]; - PyObject *lasti = stack_pointer[-3]; - PyObject *exit_func = stack_pointer[-4]; + PyObject *val; + PyObject *lasti; + PyObject *exit_func; PyObject *res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_func = stack_pointer[-4]; /* At the top of the stack are 4 values: - val: TOP = exc_info() - unused: SECOND = previous exception @@ -3325,8 +3456,9 @@ } TARGET(PUSH_EXC_INFO) { - PyObject *new_exc = stack_pointer[-1]; + PyObject *new_exc; PyObject *prev_exc; + new_exc = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = exc_info->exc_value; @@ -3337,15 +3469,16 @@ assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); STACK_GROW(1); - stack_pointer[-1] = new_exc; stack_pointer[-2] = prev_exc; + stack_pointer[-1] = new_exc; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - PyObject *self = stack_pointer[-1]; - PyObject *res2 = NULL; + PyObject *self; + PyObject *res2; PyObject *res; + self = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); @@ -3366,16 +3499,17 @@ assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); res = self; STACK_GROW(1); - stack_pointer[-1] = res; stack_pointer[-2] = res2; + stack_pointer[-1] = res; next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - PyObject *self = stack_pointer[-1]; - PyObject *res2 = NULL; + PyObject *self; + PyObject *res2; PyObject *res; + self = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert(oparg & 1); @@ -3388,15 +3522,16 @@ res2 = Py_NewRef(descr); res = self; STACK_GROW(1); - stack_pointer[-1] = res; stack_pointer[-2] = res2; + stack_pointer[-1] = res; next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - PyObject *self = stack_pointer[-1]; + PyObject *self; PyObject *res; + self = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); @@ -3420,8 +3555,9 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - PyObject *self = stack_pointer[-1]; + PyObject *self; PyObject *res; + self = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert((oparg & 1) == 0); @@ -3439,9 +3575,10 @@ } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - PyObject *self = stack_pointer[-1]; - PyObject *res2 = NULL; + PyObject *self; + PyObject *res2; PyObject *res; + self = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert(oparg & 1); @@ -3458,8 +3595,8 @@ res2 = Py_NewRef(descr); res = self; STACK_GROW(1); - stack_pointer[-1] = res; stack_pointer[-2] = res2; + stack_pointer[-1] = res; next_instr += 9; DISPATCH(); } @@ -3489,10 +3626,13 @@ TARGET(CALL) { PREDICTED(CALL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3583,8 +3723,10 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject *callable; + PyObject *method; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3594,13 +3736,18 @@ PEEK(oparg + 2) = Py_NewRef(meth); // method Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); + STACK_SHRINK(oparg); + STACK_SHRINK(1); } TARGET(CALL_PY_EXACT_ARGS) { PREDICTED(CALL_PY_EXACT_ARGS); - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&next_instr[1].cache); ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -3627,12 +3774,17 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + STACK_SHRINK(oparg); + STACK_SHRINK(1); } TARGET(CALL_PY_WITH_DEFAULTS) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&next_instr[1].cache); ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -3669,13 +3821,18 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); frame->return_offset = 0; DISPATCH_INLINED(new_frame); + STACK_SHRINK(oparg); + STACK_SHRINK(1); } TARGET(CALL_NO_KW_TYPE_1) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3693,10 +3850,13 @@ } TARGET(CALL_NO_KW_STR_1) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3716,10 +3876,13 @@ } TARGET(CALL_NO_KW_TUPLE_1) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3739,9 +3902,12 @@ } TARGET(CALL_NO_KW_ALLOC_AND_ENTER_INIT) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *null = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *null; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + null = stack_pointer[-2 - oparg]; /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) @@ -3792,10 +3958,13 @@ * as it will be checked after start_frame */ tstate->py_recursion_remaining--; goto start_frame; + STACK_SHRINK(oparg); + STACK_SHRINK(1); } TARGET(EXIT_INIT_CHECK) { - PyObject *should_be_none = stack_pointer[-1]; + PyObject *should_be_none; + should_be_none = stack_pointer[-1]; assert(STACK_LEVEL() == 2); if (should_be_none != Py_None) { PyErr_Format(PyExc_TypeError, @@ -3808,10 +3977,13 @@ } TARGET(CALL_BUILTIN_CLASS) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3842,10 +4014,13 @@ } TARGET(CALL_NO_KW_BUILTIN_O) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; @@ -3882,10 +4057,13 @@ } TARGET(CALL_NO_KW_BUILTIN_FAST) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; @@ -3926,10 +4104,13 @@ } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; int total_args = oparg; @@ -3970,10 +4151,13 @@ } TARGET(CALL_NO_KW_LEN) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* len(o) */ int is_meth = method != NULL; @@ -4006,10 +4190,13 @@ } TARGET(CALL_NO_KW_ISINSTANCE) { - PyObject **args = (stack_pointer - oparg); - PyObject *callable = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *callable; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + callable = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* isinstance(o, o2) */ int is_meth = method != NULL; @@ -4044,9 +4231,12 @@ } TARGET(CALL_NO_KW_LIST_APPEND) { - PyObject **args = (stack_pointer - oparg); - PyObject *self = stack_pointer[-(1 + oparg)]; - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *self; + PyObject *method; + args = stack_pointer - oparg; + self = stack_pointer[-1 - oparg]; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); assert(method != NULL); @@ -4064,12 +4254,16 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); + STACK_SHRINK(oparg); + STACK_SHRINK(1); } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; @@ -4109,9 +4303,11 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -4149,9 +4345,11 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -4189,9 +4387,11 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { - PyObject **args = (stack_pointer - oparg); - PyObject *method = stack_pointer[-(2 + oparg)]; + PyObject **args; + PyObject *method; PyObject *res; + args = stack_pointer - oparg; + method = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); int is_meth = method != NULL; int total_args = oparg; @@ -4233,10 +4433,13 @@ TARGET(CALL_FUNCTION_EX) { PREDICTED(CALL_FUNCTION_EX); - PyObject *kwargs = (oparg & 1) ? stack_pointer[-(((oparg & 1) ? 1 : 0))] : NULL; - PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; - PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; + PyObject *kwargs = NULL; + PyObject *callargs; + PyObject *func; PyObject *result; + if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1 ? 1 : 0)]; } + callargs = stack_pointer[-1 - (oparg & 1 ? 1 : 0)]; + func = stack_pointer[-2 - (oparg & 1 ? 1 : 0)]; // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4311,8 +4514,9 @@ } TARGET(MAKE_FUNCTION) { - PyObject *codeobj = stack_pointer[-1]; + PyObject *codeobj; PyObject *func; + codeobj = stack_pointer[-1]; PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4329,8 +4533,10 @@ } TARGET(SET_FUNCTION_ATTRIBUTE) { - PyObject *func = stack_pointer[-1]; - PyObject *attr = stack_pointer[-2]; + PyObject *func; + PyObject *attr; + func = stack_pointer[-1]; + attr = stack_pointer[-2]; assert(PyFunction_Check(func)); PyFunctionObject *func_obj = (PyFunctionObject *)func; switch(oparg) { @@ -4384,10 +4590,13 @@ } TARGET(BUILD_SLICE) { - PyObject *step = (oparg == 3) ? stack_pointer[-(((oparg == 3) ? 1 : 0))] : NULL; - PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; - PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; + PyObject *step = NULL; + PyObject *stop; + PyObject *start; PyObject *slice; + if (oparg == 3) { step = stack_pointer[-(oparg == 3 ? 1 : 0)]; } + stop = stack_pointer[-1 - (oparg == 3 ? 1 : 0)]; + start = stack_pointer[-2 - (oparg == 3 ? 1 : 0)]; slice = PySlice_New(start, stop, step); Py_DECREF(start); Py_DECREF(stop); @@ -4400,8 +4609,9 @@ } TARGET(CONVERT_VALUE) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *result; + value = stack_pointer[-1]; convertion_func_ptr conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = CONVERSION_FUNCTIONS[oparg]; @@ -4413,8 +4623,9 @@ } TARGET(FORMAT_SIMPLE) { - PyObject *value = stack_pointer[-1]; + PyObject *value; PyObject *res; + value = stack_pointer[-1]; /* If value is a unicode object, then we know the result * of format(value) is value itself. */ if (!PyUnicode_CheckExact(value)) { @@ -4430,9 +4641,11 @@ } TARGET(FORMAT_WITH_SPEC) { - PyObject *fmt_spec = stack_pointer[-1]; - PyObject *value = stack_pointer[-2]; + PyObject *fmt_spec; + PyObject *value; PyObject *res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; res = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_DECREF(fmt_spec); @@ -4443,8 +4656,9 @@ } TARGET(COPY) { - PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; + PyObject *bottom; PyObject *top; + bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = Py_NewRef(bottom); STACK_GROW(1); @@ -4455,9 +4669,11 @@ TARGET(BINARY_OP) { PREDICTED(BINARY_OP); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *rhs = stack_pointer[-1]; - PyObject *lhs = stack_pointer[-2]; + PyObject *rhs; + PyObject *lhs; PyObject *res; + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4482,11 +4698,13 @@ } TARGET(SWAP) { - PyObject *top = stack_pointer[-1]; - PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; + PyObject *top; + PyObject *bottom; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; assert(oparg >= 2); + stack_pointer[-2 - (oparg-2)] = top; stack_pointer[-1] = bottom; - stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 53ae7360b8ae1..bd8918a87ffe0 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -13,7 +13,6 @@ MacroParts, OverriddenInstructionPlaceHolder, PseudoInstruction, - StackEffectMapping, ) import parsing from parsing import StackEffect @@ -34,11 +33,12 @@ class Analyzer: input_filenames: list[str] errors: int = 0 + warnings: int = 0 def __init__(self, input_filenames: list[str]): self.input_filenames = input_filenames - def error(self, msg: str, node: parsing.Node) -> None: + def message(self, msg: str, node: parsing.Node) -> None: lineno = 0 filename = "" if context := node.context: @@ -49,8 +49,18 @@ def error(self, msg: str, node: parsing.Node) -> None: if token.kind != "COMMENT": break print(f"{filename}:{lineno}: {msg}", file=sys.stderr) + + def error(self, msg: str, node: parsing.Node) -> None: + self.message("error: " + msg, node) self.errors += 1 + def warning(self, msg: str, node: parsing.Node) -> None: + self.message("warning: " + msg, node) + self.warnings += 1 + + def note(self, msg: str, node: parsing.Node) -> None: + self.message("note: " + msg, node) + everything: list[ parsing.InstDef | parsing.Macro @@ -83,8 +93,15 @@ def parse(self) -> None: self.parse_file(filename, instrs_idx) files = " + ".join(self.input_filenames) + n_instrs = 0 + n_ops = 0 + for instr in self.instrs.values(): + if instr.kind == "op": + n_ops += 1 + else: + n_instrs += 1 print( - f"Read {len(self.instrs)} instructions/ops, " + f"Read {n_instrs} instructions, {n_ops} ops, " f"{len(self.macros)} macros, {len(self.pseudos)} pseudos, " f"and {len(self.families)} families from {files}", file=sys.stderr, @@ -270,14 +287,70 @@ def analyze_macros_and_pseudos(self) -> None: self.macro_instrs = {} self.pseudo_instrs = {} for name, macro in self.macros.items(): - self.macro_instrs[name] = self.analyze_macro(macro) + self.macro_instrs[name] = mac = self.analyze_macro(macro) + self.check_macro_consistency(mac) for name, pseudo in self.pseudos.items(): self.pseudo_instrs[name] = self.analyze_pseudo(pseudo) + # TODO: Merge with similar code in stacking.py, write_components() + def check_macro_consistency(self, mac: MacroInstruction) -> None: + def get_var_names(instr: Instruction) -> dict[str, StackEffect]: + vars: dict[str, StackEffect] = {} + for eff in instr.input_effects + instr.output_effects: + if eff.name in vars: + if vars[eff.name] != eff: + self.error( + f"Instruction {instr.name!r} has " + f"inconsistent type/cond/size for variable " + f"{eff.name!r}: {vars[eff.name]} vs {eff}", + instr.inst, + ) + else: + vars[eff.name] = eff + return vars + + all_vars: dict[str, StackEffect] = {} + # print("Checking", mac.name) + prevop: Instruction | None = None + for part in mac.parts: + if not isinstance(part, Component): + continue + vars = get_var_names(part.instr) + # print(" //", part.instr.name, "//", vars) + for name, eff in vars.items(): + if name in all_vars: + if all_vars[name] != eff: + self.error( + f"Macro {mac.name!r} has " + f"inconsistent type/cond/size for variable " + f"{name!r}: " + f"{all_vars[name]} vs {eff} in {part.instr.name!r}", + mac.macro, + ) + else: + all_vars[name] = eff + if prevop is not None: + pushes = list(prevop.output_effects) + pops = list(reversed(part.instr.input_effects)) + copies: list[tuple[StackEffect, StackEffect]] = [] + while pushes and pops and pushes[-1] == pops[0]: + src, dst = pushes.pop(), pops.pop(0) + if src.name == dst.name or dst.name is UNUSED: + continue + copies.append((src, dst)) + reads = set(copy[0].name for copy in copies) + writes = set(copy[1].name for copy in copies) + if reads & writes: + self.error( + f"Macro {mac.name!r} has conflicting copies " + f"(source of one copy is destination of another): " + f"{reads & writes}", + mac.macro, + ) + prevop = part.instr + def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: components = self.check_macro_components(macro) - stack, initial_sp = self.stack_analysis(components) - sp = initial_sp parts: MacroParts = [] flags = InstructionFlags.newEmpty() offset = 0 @@ -287,20 +360,15 @@ def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: parts.append(ceffect) offset += ceffect.size case Instruction() as instr: - part, sp, offset = self.analyze_instruction( - instr, stack, sp, offset - ) + part, offset = self.analyze_instruction(instr, offset) parts.append(part) flags.add(instr.instr_flags) case _: typing.assert_never(component) - final_sp = sp format = "IB" if offset: format += "C" + "0" * (offset - 1) - return MacroInstruction( - macro.name, stack, initial_sp, final_sp, format, flags, macro, parts, offset - ) + return MacroInstruction(macro.name, format, flags, macro, parts, offset) def analyze_pseudo(self, pseudo: parsing.Pseudo) -> PseudoInstruction: targets = [self.instrs[target] for target in pseudo.targets] @@ -312,24 +380,15 @@ def analyze_pseudo(self, pseudo: parsing.Pseudo) -> PseudoInstruction: return PseudoInstruction(pseudo.name, targets, fmts[0], targets[0].instr_flags) def analyze_instruction( - self, instr: Instruction, stack: list[StackEffect], sp: int, offset: int - ) -> tuple[Component, int, int]: - input_mapping: StackEffectMapping = [] - for ieffect in reversed(instr.input_effects): - sp -= 1 - input_mapping.append((stack[sp], ieffect)) - output_mapping: StackEffectMapping = [] - for oeffect in instr.output_effects: - output_mapping.append((stack[sp], oeffect)) - sp += 1 + self, instr: Instruction, offset: int + ) -> tuple[Component, int]: active_effects: list[ActiveCacheEffect] = [] for ceffect in instr.cache_effects: if ceffect.name != UNUSED: active_effects.append(ActiveCacheEffect(ceffect, offset)) offset += ceffect.size return ( - Component(instr, input_mapping, output_mapping, active_effects), - sp, + Component(instr, active_effects), offset, ) @@ -348,65 +407,3 @@ def check_macro_components( case _: typing.assert_never(uop) return components - - def stack_analysis( - self, components: typing.Iterable[InstructionOrCacheEffect] - ) -> tuple[list[StackEffect], int]: - """Analyze a macro. - - Ignore cache effects. - - Return the list of variables (as StackEffects) and the initial stack pointer. - """ - lowest = current = highest = 0 - conditions: dict[int, str] = {} # Indexed by 'current'. - last_instr: Instruction | None = None - for thing in components: - if isinstance(thing, Instruction): - last_instr = thing - for thing in components: - match thing: - case Instruction() as instr: - if any( - eff.size for eff in instr.input_effects + instr.output_effects - ): - # TODO: Eventually this will be needed, at least for macros. - self.error( - f"Instruction {instr.name!r} has variable-sized stack effect, " - "which are not supported in macro instructions", - instr.inst, # TODO: Pass name+location of macro - ) - if any(eff.cond for eff in instr.input_effects): - self.error( - f"Instruction {instr.name!r} has conditional input stack effect, " - "which are not supported in macro instructions", - instr.inst, # TODO: Pass name+location of macro - ) - if ( - any(eff.cond for eff in instr.output_effects) - and instr is not last_instr - ): - self.error( - f"Instruction {instr.name!r} has conditional output stack effect, " - "but is not the last instruction in a macro", - instr.inst, # TODO: Pass name+location of macro - ) - current -= len(instr.input_effects) - lowest = min(lowest, current) - for eff in instr.output_effects: - if eff.cond: - conditions[current] = eff.cond - current += 1 - highest = max(highest, current) - case parsing.CacheEffect(): - pass - case _: - typing.assert_never(thing) - # At this point, 'current' is the net stack effect, - # and 'lowest' and 'highest' are the extremes. - # Note that 'lowest' may be negative. - stack = [ - StackEffect(f"_tmp_{i}", "", conditions.get(highest - i, "")) - for i in reversed(range(1, highest - lowest + 1)) - ] - return stack, -lowest diff --git a/Tools/cases_generator/flags.py b/Tools/cases_generator/flags.py index 78acd93b73ac3..f7ebdeb0d6567 100644 --- a/Tools/cases_generator/flags.py +++ b/Tools/cases_generator/flags.py @@ -49,9 +49,9 @@ def add(self, other: "InstructionFlags") -> None: if value: setattr(self, name, value) - def names(self, value=None): + def names(self, value=None) -> list[str]: if value is None: - return dataclasses.asdict(self).keys() + return list(dataclasses.asdict(self).keys()) return [n for n, v in dataclasses.asdict(self).items() if v == value] def bitmap(self) -> int: diff --git a/Tools/cases_generator/formatting.py b/Tools/cases_generator/formatting.py index 6b5a375af891b..5894751bd9635 100644 --- a/Tools/cases_generator/formatting.py +++ b/Tools/cases_generator/formatting.py @@ -2,7 +2,7 @@ import re import typing -from parsing import StackEffect +from parsing import StackEffect, Family UNUSED = "unused" @@ -19,8 +19,11 @@ class Formatter: nominal_filename: str def __init__( - self, stream: typing.TextIO, indent: int, - emit_line_directives: bool = False, comment: str = "//", + self, + stream: typing.TextIO, + indent: int, + emit_line_directives: bool = False, + comment: str = "//", ) -> None: self.stream = stream self.prefix = " " * indent @@ -93,8 +96,11 @@ def declare(self, dst: StackEffect, src: StackEffect | None): typ = f"{dst.type}" if dst.type else "PyObject *" if src: cast = self.cast(dst, src) - init = f" = {cast}{src.name}" - elif dst.cond: + initexpr = f"{cast}{src.name}" + if src.cond and src.cond != "1": + initexpr = f"{parenthesize_cond(src.cond)} ? {initexpr} : NULL" + init = f" = {initexpr}" + elif dst.cond and dst.cond != "1": init = " = NULL" else: init = "" @@ -102,10 +108,7 @@ def declare(self, dst: StackEffect, src: StackEffect | None): self.emit(f"{typ}{sepa}{dst.name}{init};") def assign(self, dst: StackEffect, src: StackEffect): - if src.name == UNUSED: - return - if src.size: - # Don't write sized arrays -- it's up to the user code. + if src.name == UNUSED or dst.name == UNUSED: return cast = self.cast(dst, src) if re.match(r"^REG\(oparg(\d+)\)$", dst.name): @@ -122,6 +125,23 @@ def assign(self, dst: StackEffect, src: StackEffect): def cast(self, dst: StackEffect, src: StackEffect) -> str: return f"({dst.type or 'PyObject *'})" if src.type != dst.type else "" + def static_assert_family_size( + self, name: str, family: Family | None, cache_offset: int + ) -> None: + """Emit a static_assert for the size of a family, if known. + + This will fail at compile time if the cache size computed from + the instruction definition does not match the size of the struct + used by specialize.c. + """ + if family and name == family.name: + cache_size = family.size + if cache_size: + self.emit( + f"static_assert({cache_size} == {cache_offset}, " + f'"incorrect cache size");' + ) + def prettify_filename(filename: str) -> str: # Make filename more user-friendly and less platform-specific, @@ -178,11 +198,8 @@ def maybe_parenthesize(sym: str) -> str: return f"({sym})" -def string_effect_size(arg: tuple[int, str]) -> str: - numeric, symbolic = arg - if numeric and symbolic: - return f"{numeric} + {symbolic}" - elif symbolic: - return symbolic - else: - return str(numeric) +def parenthesize_cond(cond: str) -> str: + """Parenthesize a condition, but only if it contains ?: itself.""" + if "?" in cond: + cond = f"({cond})" + return cond diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 967e1e2f5b63b..d35a16a80e8d0 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -4,14 +4,14 @@ """ import argparse -import contextlib import os import posixpath import sys import typing +import stacking # Early import to avoid circular import from analysis import Analyzer -from formatting import Formatter, list_effect_size, maybe_parenthesize +from formatting import Formatter, list_effect_size from flags import InstructionFlags, variable_used from instructions import ( AnyInstruction, @@ -118,41 +118,7 @@ def effect_str(effects: list[StackEffect]) -> str: pushed = "" case parsing.Macro(): instr = self.macro_instrs[thing.name] - parts = [comp for comp in instr.parts if isinstance(comp, Component)] - # Note: stack_analysis() already verifies that macro components - # have no variable-sized stack effects. - low = 0 - sp = 0 - high = 0 - pushed_symbolic: list[str] = [] - for comp in parts: - for effect in comp.instr.input_effects: - assert not effect.cond, effect - assert not effect.size, effect - sp -= 1 - low = min(low, sp) - for effect in comp.instr.output_effects: - assert not effect.size, effect - if effect.cond: - if effect.cond in ("0", "1"): - pushed_symbolic.append(effect.cond) - else: - pushed_symbolic.append( - maybe_parenthesize( - f"{maybe_parenthesize(effect.cond)} ? 1 : 0" - ) - ) - sp += 1 - high = max(sp, high) - if high != max(0, sp): - # If you get this, intermediate stack growth occurs, - # and stack size calculations may go awry. - # E.g. [push, pop]. The fix would be for stack size - # calculations to use the micro ops. - self.error("Macro has virtual stack growth", thing) - popped = str(-low) - pushed_symbolic.append(str(sp - low - len(pushed_symbolic))) - pushed = " + ".join(pushed_symbolic) + popped, pushed = stacking.get_stack_effect_info_for_macro(instr) case parsing.Pseudo(): instr = self.pseudo_instrs[thing.name] popped = pushed = None @@ -258,7 +224,8 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No case _: typing.assert_never(thing) all_formats.add(format) - # Turn it into a list of enum definitions. + + # Turn it into a sorted list of enum values. format_enums = [INSTR_FMT_PREFIX + format for format in sorted(all_formats)] with open(metadata_filename, "w") as f: @@ -276,8 +243,10 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.write_stack_effect_functions() - # Write type definitions - self.out.emit(f"enum InstructionFormat {{ {', '.join(format_enums)} }};") + # Write the enum definition for instruction formats. + with self.out.block("enum InstructionFormat", ";"): + for enum in format_enums: + self.out.emit(enum + ",") self.out.emit("") self.out.emit( @@ -374,7 +343,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No # Since an 'op' is not a bytecode, it has no expansion; but 'inst' is if instr.kind == "inst" and instr.is_viable_uop(): # Construct a dummy Component -- input/output mappings are not used - part = Component(instr, [], [], instr.active_caches) + part = Component(instr, instr.active_caches) self.write_macro_expansions(instr.name, [part]) elif instr.kind == "inst" and variable_used( instr.inst, "oparg1" @@ -468,7 +437,15 @@ def write_macro_expansions(self, name: str, parts: MacroParts) -> None: if isinstance(part, Component): # All component instructions must be viable uops if not part.instr.is_viable_uop(): - print(f"NOTE: Part {part.instr.name} of {name} is not a viable uop") + # This note just reminds us about macros that cannot + # be expanded to Tier 2 uops. It is not an error. + # It is sometimes emitted for macros that have a + # manual translation in translate_bytecode_to_trace() + # in Python/optimizer.c. + self.note( + f"Part {part.instr.name} of {name} is not a viable uop", + part.instr.inst, + ) return if not part.active_caches: size, offset = OPARG_SIZES["OPARG_FULL"], 0 @@ -512,7 +489,7 @@ def write_super_expansions(self, name: str) -> None: instr2 = self.instrs[name2] assert not instr1.active_caches, f"{name1} has active caches" assert not instr2.active_caches, f"{name2} has active caches" - expansions = [ + expansions: list[tuple[str, int, int]] = [ (name1, OPARG_SIZES["OPARG_TOP"], 0), (name2, OPARG_SIZES["OPARG_BOTTOM"], 0), ] @@ -563,7 +540,6 @@ def write_instructions( # Write and count instructions of all kinds n_instrs = 0 n_macros = 0 - n_pseudos = 0 for thing in self.everything: match thing: case OverriddenInstructionPlaceHolder(): @@ -574,15 +550,17 @@ def write_instructions( self.write_instr(self.instrs[thing.name]) case parsing.Macro(): n_macros += 1 - self.write_macro(self.macro_instrs[thing.name]) + mac = self.macro_instrs[thing.name] + stacking.write_macro_instr(mac, self.out, self.families.get(mac.name)) + # self.write_macro(self.macro_instrs[thing.name]) case parsing.Pseudo(): - n_pseudos += 1 + pass case _: typing.assert_never(thing) print( - f"Wrote {n_instrs} instructions, {n_macros} macros, " - f"and {n_pseudos} pseudos to {output_filename}", + f"Wrote {n_instrs} instructions and {n_macros} macros " + f"to {output_filename}", file=sys.stderr, ) @@ -590,6 +568,8 @@ def write_executor_instructions( self, executor_filename: str, emit_line_directives: bool ) -> None: """Generate cases for the Tier 2 interpreter.""" + n_instrs = 0 + n_uops = 0 with open(executor_filename, "w") as f: self.out = Formatter(f, 8, emit_line_directives) self.write_provenance_header() @@ -601,6 +581,10 @@ def write_executor_instructions( case parsing.InstDef(): instr = self.instrs[thing.name] if instr.is_viable_uop(): + if instr.kind == "op": + n_uops += 1 + else: + n_instrs += 1 self.out.emit("") with self.out.block(f"case {thing.name}:"): instr.write(self.out, tier=TIER_TWO) @@ -616,7 +600,7 @@ def write_executor_instructions( case _: typing.assert_never(thing) print( - f"Wrote some stuff to {executor_filename}", + f"Wrote {n_instrs} instructions and {n_uops} ops to {executor_filename}", file=sys.stderr, ) @@ -642,69 +626,6 @@ def write_instr(self, instr: Instruction) -> None: self.out.emit("CHECK_EVAL_BREAKER();") self.out.emit(f"DISPATCH();") - def write_macro(self, mac: MacroInstruction) -> None: - """Write code for a macro instruction.""" - last_instr: Instruction | None = None - with self.wrap_macro(mac): - cache_adjust = 0 - for part in mac.parts: - match part: - case parsing.CacheEffect(size=size): - cache_adjust += size - case Component() as comp: - last_instr = comp.instr - comp.write_body(self.out) - cache_adjust += comp.instr.cache_offset - - if cache_adjust: - self.out.emit(f"next_instr += {cache_adjust};") - - if ( - (family := self.families.get(mac.name)) - and mac.name == family.name - and (cache_size := family.size) - ): - self.out.emit( - f"static_assert({cache_size} == " - f'{cache_adjust}, "incorrect cache size");' - ) - - @contextlib.contextmanager - def wrap_macro(self, mac: MacroInstruction): - """Boilerplate for macro instructions.""" - # TODO: Somewhere (where?) make it so that if one instruction - # has an output that is input to another, and the variable names - # and types match and don't conflict with other instructions, - # that variable is declared with the right name and type in the - # outer block, rather than trusting the compiler to optimize it. - self.out.emit("") - with self.out.block(f"TARGET({mac.name})"): - if mac.predicted: - self.out.emit(f"PREDICTED({mac.name});") - - # The input effects should have no conditionals. - # Only the output effects do (for now). - ieffects = [ - StackEffect(eff.name, eff.type) if eff.cond else eff - for eff in mac.stack - ] - - for i, var in reversed(list(enumerate(ieffects))): - src = None - if i < mac.initial_sp: - src = StackEffect(f"stack_pointer[-{mac.initial_sp - i}]", "") - self.out.declare(var, src) - - yield - - self.out.stack_adjust(ieffects[: mac.initial_sp], mac.stack[: mac.final_sp]) - - for i, var in enumerate(reversed(mac.stack[: mac.final_sp]), 1): - dst = StackEffect(f"stack_pointer[-{i}]", "") - self.out.assign(dst, var) - - self.out.emit(f"DISPATCH();") - def main(): """Parse command line, parse input, analyze, write output.""" diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index 6f42699d900b4..aa94dbb07ea1c 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -2,17 +2,16 @@ import re import typing -from flags import InstructionFlags, variable_used_unspecialized +from flags import InstructionFlags, variable_used, variable_used_unspecialized from formatting import ( Formatter, UNUSED, - string_effect_size, list_effect_size, - maybe_parenthesize, ) import lexer as lx import parsing from parsing import StackEffect +import stacking BITS_PER_CODE_UNIT = 16 @@ -61,6 +60,7 @@ class Instruction: # Computed by constructor always_exits: bool + has_deopt: bool cache_offset: int cache_effects: list[parsing.CacheEffect] input_effects: list[StackEffect] @@ -83,6 +83,7 @@ def __init__(self, inst: parsing.InstDef): self.block ) self.always_exits = always_exits(self.block_text) + self.has_deopt = variable_used(self.inst, "DEOPT_IF") self.cache_effects = [ effect for effect in inst.inputs if isinstance(effect, parsing.CacheEffect) ] @@ -93,7 +94,7 @@ def __init__(self, inst: parsing.InstDef): self.output_effects = inst.outputs # For consistency/completeness unmoved_names: set[str] = set() for ieffect, oeffect in zip(self.input_effects, self.output_effects): - if ieffect.name == oeffect.name: + if ieffect == oeffect and ieffect.name == oeffect.name: unmoved_names.add(ieffect.name) else: break @@ -141,84 +142,17 @@ def is_viable_uop(self) -> bool: def write(self, out: Formatter, tier: Tiers = TIER_ONE) -> None: """Write one instruction, sans prologue and epilogue.""" + # Write a static assertion that a family's cache size is correct - if family := self.family: - if self.name == family.name: - if cache_size := family.size: - out.emit( - f"static_assert({cache_size} == " - f'{self.cache_offset}, "incorrect cache size");' - ) + out.static_assert_family_size(self.name, self.family, self.cache_offset) # Write input stack effect variable declarations and initializations - ieffects = list(reversed(self.input_effects)) - for i, ieffect in enumerate(ieffects): - isize = string_effect_size( - list_effect_size([ieff for ieff in ieffects[: i + 1]]) - ) - if ieffect.size: - src = StackEffect( - f"(stack_pointer - {maybe_parenthesize(isize)})", "PyObject **" - ) - elif ieffect.cond: - src = StackEffect( - f"({ieffect.cond}) ? stack_pointer[-{maybe_parenthesize(isize)}] : NULL", - "", - ) - else: - src = StackEffect(f"stack_pointer[-{maybe_parenthesize(isize)}]", "") - out.declare(ieffect, src) - - # Write output stack effect variable declarations - isize = string_effect_size(list_effect_size(self.input_effects)) - input_names = {ieffect.name for ieffect in self.input_effects} - for i, oeffect in enumerate(self.output_effects): - if oeffect.name not in input_names: - if oeffect.size: - osize = string_effect_size( - list_effect_size([oeff for oeff in self.output_effects[:i]]) - ) - offset = "stack_pointer" - if isize != osize: - if isize != "0": - offset += f" - ({isize})" - if osize != "0": - offset += f" + {osize}" - src = StackEffect(offset, "PyObject **") - out.declare(oeffect, src) - else: - out.declare(oeffect, None) - - # out.emit(f"next_instr += OPSIZE({self.inst.name}) - 1;") - - self.write_body(out, 0, self.active_caches, tier=tier) + stacking.write_single_instr(self, out, tier) # Skip the rest if the block always exits if self.always_exits: return - # Write net stack growth/shrinkage - out.stack_adjust( - [ieff for ieff in self.input_effects], - [oeff for oeff in self.output_effects], - ) - - # Write output stack effect assignments - oeffects = list(reversed(self.output_effects)) - for i, oeffect in enumerate(oeffects): - if oeffect.name in self.unmoved_names: - continue - osize = string_effect_size( - list_effect_size([oeff for oeff in oeffects[: i + 1]]) - ) - if oeffect.size: - dst = StackEffect( - f"stack_pointer - {maybe_parenthesize(osize)}", "PyObject **" - ) - else: - dst = StackEffect(f"stack_pointer[-{maybe_parenthesize(osize)}]", "") - out.assign(dst, oeffect) - # Write cache effect if tier == TIER_ONE and self.cache_offset: out.emit(f"next_instr += {self.cache_offset};") @@ -274,7 +208,12 @@ def write_body( # These aren't DECREF'ed so they can stay. ieffs = list(self.input_effects) oeffs = list(self.output_effects) - while ieffs and oeffs and ieffs[0] == oeffs[0]: + while ( + ieffs + and oeffs + and ieffs[0] == oeffs[0] + and ieffs[0].name == oeffs[0].name + ): ieffs.pop(0) oeffs.pop(0) ninputs, symbolic = list_effect_size(ieffs) @@ -307,30 +246,13 @@ def write_body( InstructionOrCacheEffect = Instruction | parsing.CacheEffect -StackEffectMapping = list[tuple[StackEffect, StackEffect]] @dataclasses.dataclass class Component: instr: Instruction - input_mapping: StackEffectMapping - output_mapping: StackEffectMapping active_caches: list[ActiveCacheEffect] - def write_body(self, out: Formatter) -> None: - with out.block(""): - input_names = {ieffect.name for _, ieffect in self.input_mapping} - for var, ieffect in self.input_mapping: - out.declare(ieffect, var) - for _, oeffect in self.output_mapping: - if oeffect.name not in input_names: - out.declare(oeffect, None) - - self.instr.write_body(out, -4, self.active_caches) - - for var, oeffect in self.output_mapping: - out.assign(var, oeffect) - MacroParts = list[Component | parsing.CacheEffect] @@ -340,9 +262,6 @@ class MacroInstruction: """A macro instruction.""" name: str - stack: list[StackEffect] - initial_sp: int - final_sp: int instr_fmt: str instr_flags: InstructionFlags macro: parsing.Macro diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 290285dc03123..5610ac661a894 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -69,12 +69,18 @@ class Block(Node): @dataclass class StackEffect(Node): - name: str + name: str = field(compare=False) # __eq__ only uses type, cond, size type: str = "" # Optional `:type` cond: str = "" # Optional `if (cond)` size: str = "" # Optional `[size]` # Note: size cannot be combined with type or cond + def __repr__(self): + items = [self.name, self.type, self.cond, self.size] + while items and items[-1] == "": + del items[-1] + return f"StackEffect({', '.join(repr(item) for item in items)})" + @dataclass class Expression(Node): @@ -130,6 +136,7 @@ class Family(Node): size: str # Variable giving the cache size in code units members: list[str] + @dataclass class Pseudo(Node): name: str @@ -154,7 +161,13 @@ def inst_def(self) -> InstDef | None: if hdr := self.inst_header(): if block := self.block(): return InstDef( - hdr.override, hdr.register, hdr.kind, hdr.name, hdr.inputs, hdr.outputs, block + hdr.override, + hdr.register, + hdr.kind, + hdr.name, + hdr.inputs, + hdr.outputs, + block, ) raise self.make_syntax_error("Expected block") return None @@ -371,9 +384,7 @@ def pseudo_def(self) -> Pseudo | None: raise self.make_syntax_error("Expected {") if members := self.members(): if self.expect(lx.RBRACE) and self.expect(lx.SEMI): - return Pseudo( - tkn.text, members - ) + return Pseudo(tkn.text, members) return None def members(self) -> list[str] | None: diff --git a/Tools/cases_generator/plexer.py b/Tools/cases_generator/plexer.py index a73254ed5b1da..cb6c537586649 100644 --- a/Tools/cases_generator/plexer.py +++ b/Tools/cases_generator/plexer.py @@ -1,4 +1,5 @@ import lexer as lx + Token = lx.Token @@ -64,7 +65,9 @@ def require(self, kind: str) -> Token: tkn = self.next() if tkn is not None and tkn.kind == kind: return tkn - raise self.make_syntax_error(f"Expected {kind!r} but got {tkn and tkn.text!r}", tkn) + raise self.make_syntax_error( + f"Expected {kind!r} but got {tkn and tkn.text!r}", tkn + ) def extract_line(self, lineno: int) -> str: # Return source line `lineno` (1-based) @@ -73,18 +76,20 @@ def extract_line(self, lineno: int) -> str: return "" return lines[lineno - 1] - def make_syntax_error(self, message: str, tkn: Token|None = None) -> SyntaxError: + def make_syntax_error(self, message: str, tkn: Token | None = None) -> SyntaxError: # Construct a SyntaxError instance from message and token if tkn is None: tkn = self.peek() if tkn is None: tkn = self.tokens[-1] - return lx.make_syntax_error(message, - self.filename, tkn.line, tkn.column, self.extract_line(tkn.line)) + return lx.make_syntax_error( + message, self.filename, tkn.line, tkn.column, self.extract_line(tkn.line) + ) if __name__ == "__main__": import sys + if sys.argv[1:]: filename = sys.argv[1] if filename == "-c" and sys.argv[2:]: diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py new file mode 100644 index 0000000000000..23eca3037f896 --- /dev/null +++ b/Tools/cases_generator/stacking.py @@ -0,0 +1,400 @@ +import dataclasses +import typing + +from formatting import ( + Formatter, + UNUSED, + maybe_parenthesize, + parenthesize_cond, +) +from instructions import ( + ActiveCacheEffect, + Instruction, + MacroInstruction, + Component, + Tiers, + TIER_ONE, +) +from parsing import StackEffect, CacheEffect, Family + + + at dataclasses.dataclass +class StackOffset: + """Represent the stack offset for a PEEK or POKE. + + - At stack_pointer[0], deep and high are both empty. + (Note that that is an invalid stack reference.) + - Below stack top, only deep is non-empty. + - Above stack top, only high is non-empty. + - In complex cases, both deep and high may be non-empty. + + All this would be much simpler if all stack entries were the same + size, but with conditional and array effects, they aren't. + The offsets are each represented by a list of StackEffect objects. + The name in the StackEffects is unused. + """ + + deep: list[StackEffect] = dataclasses.field(default_factory=list) + high: list[StackEffect] = dataclasses.field(default_factory=list) + + def clone(self) -> "StackOffset": + return StackOffset(list(self.deep), list(self.high)) + + def negate(self) -> "StackOffset": + return StackOffset(list(self.high), list(self.deep)) + + def deeper(self, eff: StackEffect) -> None: + if eff in self.high: + self.high.remove(eff) + else: + self.deep.append(eff) + + def higher(self, eff: StackEffect) -> None: + if eff in self.deep: + self.deep.remove(eff) + else: + self.high.append(eff) + + def as_terms(self) -> list[tuple[str, str]]: + num = 0 + terms: list[tuple[str, str]] = [] + for eff in self.deep: + if eff.size: + terms.append(("-", maybe_parenthesize(eff.size))) + elif eff.cond and eff.cond != "1": + terms.append(("-", f"({parenthesize_cond(eff.cond)} ? 1 : 0)")) + elif eff.cond != "0": + num -= 1 + for eff in self.high: + if eff.size: + terms.append(("+", maybe_parenthesize(eff.size))) + elif eff.cond and eff.cond != "1": + terms.append(("+", f"({parenthesize_cond(eff.cond)} ? 1 : 0)")) + elif eff.cond != "0": + num += 1 + if num < 0: + terms.insert(0, ("-", str(-num))) + elif num > 0: + terms.append(("+", str(num))) + return terms + + def as_index(self) -> str: + terms = self.as_terms() + return make_index(terms) + + +def make_index(terms: list[tuple[str, str]]) -> str: + # Produce an index expression from the terms honoring PEP 8, + # surrounding binary ops with spaces but not unary minus + index = "" + for sign, term in terms: + if index: + index += f" {sign} {term}" + elif sign == "+": + index = term + else: + index = sign + term + return index or "0" + + + at dataclasses.dataclass +class StackItem: + offset: StackOffset + effect: StackEffect + + def as_variable(self, lax: bool = False) -> str: + """Return e.g. stack_pointer[-1].""" + terms = self.offset.as_terms() + if self.effect.size: + terms.insert(0, ("+", "stack_pointer")) + index = make_index(terms) + if self.effect.size: + res = index + else: + res = f"stack_pointer[{index}]" + if not lax: + # Check that we're not reading or writing above stack top. + # Skip this for output variable initialization (lax=True). + assert ( + self.effect in self.offset.deep and not self.offset.high + ), f"Push or pop above current stack level: {res}" + return res + + + at dataclasses.dataclass +class CopyEffect: + src: StackEffect + dst: StackEffect + + +class EffectManager: + """Manage stack effects and offsets for an instruction.""" + + instr: Instruction + active_caches: list[ActiveCacheEffect] + peeks: list[StackItem] + pokes: list[StackItem] + copies: list[CopyEffect] # See merge() + # Track offsets from stack pointer + min_offset: StackOffset + final_offset: StackOffset + + def __init__( + self, + instr: Instruction, + active_caches: list[ActiveCacheEffect], + pred: "EffectManager | None" = None, + ): + self.instr = instr + self.active_caches = active_caches + self.peeks = [] + self.pokes = [] + self.copies = [] + self.final_offset = pred.final_offset.clone() if pred else StackOffset() + for eff in reversed(instr.input_effects): + self.final_offset.deeper(eff) + self.peeks.append(StackItem(offset=self.final_offset.clone(), effect=eff)) + self.min_offset = self.final_offset.clone() + for eff in instr.output_effects: + self.pokes.append(StackItem(offset=self.final_offset.clone(), effect=eff)) + self.final_offset.higher(eff) + + if pred: + # Replace push(x) + pop(y) with copy(x, y). + # Check that the sources and destinations are disjoint. + sources: set[str] = set() + destinations: set[str] = set() + while ( + pred.pokes + and self.peeks + and pred.pokes[-1].effect == self.peeks[-1].effect + ): + src = pred.pokes.pop(-1).effect + dst = self.peeks.pop(0).effect + pred.final_offset.deeper(src) + if dst.name != UNUSED: + destinations.add(dst.name) + if dst.name != src.name: + sources.add(src.name) + self.copies.append(CopyEffect(src, dst)) + # TODO: Turn this into an error (pass an Analyzer instance?) + assert sources & destinations == set(), ( + pred.instr.name, + self.instr.name, + sources, + destinations, + ) + + def adjust_deeper(self, eff: StackEffect) -> None: + for peek in self.peeks: + peek.offset.deeper(eff) + for poke in self.pokes: + poke.offset.deeper(eff) + self.min_offset.deeper(eff) + self.final_offset.deeper(eff) + + def adjust_higher(self, eff: StackEffect) -> None: + for peek in self.peeks: + peek.offset.higher(eff) + for poke in self.pokes: + poke.offset.higher(eff) + self.min_offset.higher(eff) + self.final_offset.higher(eff) + + def adjust(self, offset: StackOffset) -> None: + for down in offset.deep: + self.adjust_deeper(down) + for up in offset.high: + self.adjust_higher(up) + + def adjust_inverse(self, offset: StackOffset) -> None: + for down in offset.deep: + self.adjust_higher(down) + for up in offset.high: + self.adjust_deeper(up) + + def collect_vars(self) -> dict[str, StackEffect]: + """Collect all variables, skipping unused ones.""" + vars: dict[str, StackEffect] = {} + + def add(eff: StackEffect) -> None: + if eff.name != UNUSED: + if eff.name in vars: + # TODO: Make this an error + assert vars[eff.name] == eff, ( + self.instr.name, + eff.name, + vars[eff.name], + eff, + ) + else: + vars[eff.name] = eff + + for copy in self.copies: + add(copy.src) + add(copy.dst) + for peek in self.peeks: + add(peek.effect) + for poke in self.pokes: + add(poke.effect) + + return vars + + +def less_than(a: StackOffset, b: StackOffset) -> bool: + # TODO: Handle more cases + if a.high != b.high: + return False + return a.deep[: len(b.deep)] == b.deep + + +def get_managers(parts: list[Component]) -> list[EffectManager]: + managers: list[EffectManager] = [] + pred: EffectManager | None = None + for part in parts: + mgr = EffectManager(part.instr, part.active_caches, pred) + managers.append(mgr) + pred = mgr + return managers + + +def get_stack_effect_info_for_macro(mac: MacroInstruction) -> tuple[str, str]: + """Get the stack effect info for a macro instruction. + + Returns a tuple (popped, pushed) where each is a string giving a + symbolic expression for the number of values popped/pushed. + """ + parts = [part for part in mac.parts if isinstance(part, Component)] + managers = get_managers(parts) + popped = StackOffset() + for mgr in managers: + if less_than(mgr.min_offset, popped): + popped = mgr.min_offset.clone() + # Compute pushed = final - popped + pushed = managers[-1].final_offset.clone() + for effect in popped.deep: + pushed.higher(effect) + for effect in popped.high: + pushed.deeper(effect) + return popped.negate().as_index(), pushed.as_index() + + +def write_single_instr( + instr: Instruction, out: Formatter, tier: Tiers = TIER_ONE +) -> None: + try: + write_components( + [Component(instr, instr.active_caches)], + out, + tier, + ) + except AssertionError as err: + raise AssertionError(f"Error writing instruction {instr.name}") from err + + +def write_macro_instr( + mac: MacroInstruction, out: Formatter, family: Family | None +) -> None: + parts = [part for part in mac.parts if isinstance(part, Component)] + + cache_adjust = 0 + for part in mac.parts: + match part: + case CacheEffect(size=size): + cache_adjust += size + case Component(instr=instr): + cache_adjust += instr.cache_offset + case _: + typing.assert_never(part) + + out.emit("") + with out.block(f"TARGET({mac.name})"): + if mac.predicted: + out.emit(f"PREDICTED({mac.name});") + out.static_assert_family_size(mac.name, family, cache_adjust) + try: + write_components(parts, out, TIER_ONE) + except AssertionError as err: + raise AssertionError(f"Error writing macro {mac.name}") from err + if cache_adjust: + out.emit(f"next_instr += {cache_adjust};") + out.emit("DISPATCH();") + + +def write_components( + parts: list[Component], + out: Formatter, + tier: Tiers, +) -> None: + managers = get_managers(parts) + + all_vars: dict[str, StackEffect] = {} + for mgr in managers: + for name, eff in mgr.collect_vars().items(): + if name in all_vars: + # TODO: Turn this into an error -- variable conflict + assert all_vars[name] == eff, ( + name, + mgr.instr.name, + all_vars[name], + eff, + ) + else: + all_vars[name] = eff + + # Declare all variables + for name, eff in all_vars.items(): + out.declare(eff, None) + + for mgr in managers: + if len(parts) > 1: + out.emit(f"// {mgr.instr.name}") + + for copy in mgr.copies: + if copy.src.name != copy.dst.name: + out.assign(copy.dst, copy.src) + for peek in mgr.peeks: + out.assign( + peek.effect, + StackEffect( + peek.as_variable(), + peek.effect.type, + peek.effect.cond, + peek.effect.size, + ), + ) + # Initialize array outputs + for poke in mgr.pokes: + if poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: + out.assign( + poke.effect, + StackEffect( + poke.as_variable(lax=True), + poke.effect.type, + poke.effect.cond, + poke.effect.size, + ), + ) + + if len(parts) == 1: + mgr.instr.write_body(out, 0, mgr.active_caches, tier) + else: + with out.block(""): + mgr.instr.write_body(out, -4, mgr.active_caches, tier) + + if mgr is managers[-1]: + out.stack_adjust(mgr.final_offset.deep, mgr.final_offset.high) + # Use clone() since adjust_inverse() mutates final_offset + mgr.adjust_inverse(mgr.final_offset.clone()) + + for poke in mgr.pokes: + if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: + out.assign( + StackEffect( + poke.as_variable(), + poke.effect.type, + poke.effect.cond, + poke.effect.size, + ), + poke.effect, + ) From webhook-mailer at python.org Fri Aug 4 13:49:11 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Fri, 04 Aug 2023 17:49:11 -0000 Subject: [Python-checkins] gh-104683: Argument Clinic: Use CConverter.length_name where possible (#107638) Message-ID: https://github.com/python/cpython/commit/321f0f79325adfe7656645060c2008d5779e1a7f commit: 321f0f79325adfe7656645060c2008d5779e1a7f branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-04T17:49:07Z summary: gh-104683: Argument Clinic: Use CConverter.length_name where possible (#107638) Also make it a cached property. Co-authored-by: Alex Waygood files: M Tools/clinic/clinic.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 6eb2c550e696f..917f1bfeb1d25 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2779,7 +2779,7 @@ class CConverter(metaclass=CConverterAutoRegister): # Only used by the 'O!' format unit (and the "object" converter). subclass_of: str | None = None - # Do we want an adjacent '_length' variable for this variable? + # See also the 'length_name' property. # Only used by format units ending with '#'. length = False @@ -2873,12 +2873,12 @@ def _render_self(self, parameter: Parameter, data: CRenderData) -> None: s = ("&" if self.impl_by_reference else "") + name data.impl_arguments.append(s) if self.length: - data.impl_arguments.append(self.length_name()) + data.impl_arguments.append(self.length_name) # impl_parameters data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference)) if self.length: - data.impl_parameters.append("Py_ssize_t " + self.length_name()) + data.impl_parameters.append(f"Py_ssize_t {self.length_name}") def _render_non_self( self, @@ -2937,6 +2937,7 @@ def render(self, parameter: Parameter, data: CRenderData) -> None: self._render_self(parameter, data) self._render_non_self(parameter, data) + @functools.cached_property def length_name(self) -> str: """Computes the name of the associated "length" variable.""" assert self.length is not None @@ -2960,7 +2961,7 @@ def parse_argument(self, args: list[str]) -> None: args.append(s) if self.length: - args.append("&" + self.length_name()) + args.append(f"&{self.length_name}") # # All the functions after here are intended as extension points. @@ -3005,9 +3006,8 @@ def declaration(self, *, in_parser: bool = False) -> str: declaration.append(default) declaration.append(";") if self.length: - declaration.append('\nPy_ssize_t ') - declaration.append(self.length_name()) - declaration.append(';') + declaration.append('\n') + declaration.append(f"Py_ssize_t {self.length_name};") return "".join(declaration) def initialize(self) -> str: @@ -3686,29 +3686,29 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: _PyArg_BadArgument("{{name}}", {displayname}, "str", {argname}); goto exit; }}}} - Py_ssize_t {paramname}_length; - {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{paramname}_length); + Py_ssize_t {length_name}; + {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); if ({paramname} == NULL) {{{{ goto exit; }}}} - if (strlen({paramname}) != (size_t){paramname}_length) {{{{ + if (strlen({paramname}) != (size_t){length_name}) {{{{ PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; }}}} """.format(argname=argname, paramname=self.parser_name, - displayname=displayname) + displayname=displayname, length_name=self.length_name) if self.format_unit == 'z': return """ if ({argname} == Py_None) {{{{ {paramname} = NULL; }}}} else if (PyUnicode_Check({argname})) {{{{ - Py_ssize_t {paramname}_length; - {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{paramname}_length); + Py_ssize_t {length_name}; + {paramname} = PyUnicode_AsUTF8AndSize({argname}, &{length_name}); if ({paramname} == NULL) {{{{ goto exit; }}}} - if (strlen({paramname}) != (size_t){paramname}_length) {{{{ + if (strlen({paramname}) != (size_t){length_name}) {{{{ PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; }}}} @@ -3718,7 +3718,7 @@ def parse_arg(self, argname: str, displayname: str) -> str | None: goto exit; }}}} """.format(argname=argname, paramname=self.parser_name, - displayname=displayname) + displayname=displayname, length_name=self.length_name) return super().parse_arg(argname, displayname) # From webhook-mailer at python.org Fri Aug 4 14:33:39 2023 From: webhook-mailer at python.org (hugovk) Date: Fri, 04 Aug 2023 18:33:39 -0000 Subject: [Python-checkins] Docs: Only include Plausible for html, not for epub etc (#107637) Message-ID: https://github.com/python/cpython/commit/904b5319b3cc72063f4bfcd7beb3a1ef0fc641be commit: 904b5319b3cc72063f4bfcd7beb3a1ef0fc641be branch: main author: Hugo van Kemenade committer: hugovk date: 2023-08-04T21:33:34+03:00 summary: Docs: Only include Plausible for html, not for epub etc (#107637) Only include Plausible for html, not for epub etc files: M Doc/tools/templates/layout.html diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 9832feba14167..80103158ea01e 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,7 +26,9 @@ {% endblock %} {% block extrahead %} - + {% if builder == "html" %} + + {% endif %} {% if builder != "htmlhelp" %} {% if pagename == 'whatsnew/changelog' and not embedded %} From webhook-mailer at python.org Fri Aug 4 15:11:46 2023 From: webhook-mailer at python.org (hugovk) Date: Fri, 04 Aug 2023 19:11:46 -0000 Subject: [Python-checkins] [3.11] Docs: Only include Plausible for html, not for epub etc (GH-107637) (#107643) Message-ID: https://github.com/python/cpython/commit/6a2f981418e781e4bdbb4ed257845a14a5e82874 commit: 6a2f981418e781e4bdbb4ed257845a14a5e82874 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: hugovk date: 2023-08-04T13:11:42-06:00 summary: [3.11] Docs: Only include Plausible for html, not for epub etc (GH-107637) (#107643) Co-authored-by: Hugo van Kemenade files: M Doc/tools/templates/layout.html diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 9832feba14167..80103158ea01e 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,7 +26,9 @@ {% endblock %} {% block extrahead %} - + {% if builder == "html" %} + + {% endif %} {% if builder != "htmlhelp" %} {% if pagename == 'whatsnew/changelog' and not embedded %} From webhook-mailer at python.org Fri Aug 4 15:41:08 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Fri, 04 Aug 2023 19:41:08 -0000 Subject: [Python-checkins] gh-106368: Argument clinic: improve coverage for `self.valid_line()` calls (#107641) Message-ID: https://github.com/python/cpython/commit/2c25bd82f46df72c89ca5bca10eaa06137ab8290 commit: 2c25bd82f46df72c89ca5bca10eaa06137ab8290 branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-04T20:41:04+01:00 summary: gh-106368: Argument clinic: improve coverage for `self.valid_line()` calls (#107641) files: M Lib/test/test_clinic.py M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index dd17d02519bfa..84b6a193ecf91 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1057,6 +1057,38 @@ def test_explicit_parameters_in_docstring(self): Okay, we're done here. """) + def test_docstring_with_comments(self): + function = self.parse_function(dedent(""" + module foo + foo.bar + x: int + # We're about to have + # the documentation for x. + Documentation for x. + # We've just had + # the documentation for x. + y: int + + # We're about to have + # the documentation for foo. + This is the documentation for foo. + # We've just had + # the documentation for foo. + + Okay, we're done here. + """)) + self.checkDocstring(function, """ + bar($module, /, x, y) + -- + + This is the documentation for foo. + + x + Documentation for x. + + Okay, we're done here. + """) + def test_parser_regression_special_character_in_parameter_column_of_docstring_first_line(self): function = self.parse_function(dedent(""" module os diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 917f1bfeb1d25..7fbae1e0d870a 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4685,9 +4685,7 @@ def state_modulename_name(self, line: str) -> None: # this line is permitted to start with whitespace. # we'll call this number of spaces F (for "function"). - if not self.valid_line(line): - return - + assert self.valid_line(line) self.indent.infer(line) # are we cloning? From webhook-mailer at python.org Fri Aug 4 16:20:24 2023 From: webhook-mailer at python.org (Yhg1s) Date: Fri, 04 Aug 2023 20:20:24 -0000 Subject: [Python-checkins] [3.12] Docs: upgrade to python-docs-theme 2023.7 (GH-107617) (#107633) Message-ID: https://github.com/python/cpython/commit/28a9849d7c2f2ae3c283cf61b904418d8b01ec01 commit: 28a9849d7c2f2ae3c283cf61b904418d8b01ec01 branch: 3.12 author: Hugo van Kemenade committer: Yhg1s date: 2023-08-04T22:20:20+02:00 summary: [3.12] Docs: upgrade to python-docs-theme 2023.7 (GH-107617) (#107633) (cherry picked from commit 19f32b24b2e1680ff9929bb64d681397b259c6fb) files: M Doc/requirements.txt diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 4c9d02ea37ab5..fb558b3c5af24 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -14,6 +14,6 @@ sphinxext-opengraph==0.7.5 # The theme used by the documentation is stored separately, so we need # to install that as well. -python-docs-theme>=2022.1 +python-docs-theme>=2023.7 -c constraints.txt From webhook-mailer at python.org Fri Aug 4 16:20:50 2023 From: webhook-mailer at python.org (Yhg1s) Date: Fri, 04 Aug 2023 20:20:50 -0000 Subject: [Python-checkins] [3.12] Docs: Only include Plausible for html, not for epub etc (GH-107637) (#107642) Message-ID: https://github.com/python/cpython/commit/0e7a4f733685e7a7ccf28d850bae1b2222977362 commit: 0e7a4f733685e7a7ccf28d850bae1b2222977362 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-04T22:20:46+02:00 summary: [3.12] Docs: Only include Plausible for html, not for epub etc (GH-107637) (#107642) Docs: Only include Plausible for html, not for epub etc (GH-107637) Only include Plausible for html, not for epub etc (cherry picked from commit 904b5319b3cc72063f4bfcd7beb3a1ef0fc641be) Co-authored-by: Hugo van Kemenade files: M Doc/tools/templates/layout.html diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 9832feba14167..80103158ea01e 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,7 +26,9 @@ {% endblock %} {% block extrahead %} - + {% if builder == "html" %} + + {% endif %} {% if builder != "htmlhelp" %} {% if pagename == 'whatsnew/changelog' and not embedded %} From webhook-mailer at python.org Fri Aug 4 17:14:25 2023 From: webhook-mailer at python.org (hugovk) Date: Fri, 04 Aug 2023 21:14:25 -0000 Subject: [Python-checkins] [3.11] Docs: upgrade to python-docs-theme 2023.7 (GH-107617) (#107634) Message-ID: https://github.com/python/cpython/commit/f978a79130133de1cf26436c460ff169cf5c8dc5 commit: f978a79130133de1cf26436c460ff169cf5c8dc5 branch: 3.11 author: Hugo van Kemenade committer: hugovk date: 2023-08-05T00:14:21+03:00 summary: [3.11] Docs: upgrade to python-docs-theme 2023.7 (GH-107617) (#107634) (cherry picked from commit 19f32b24b2e1680ff9929bb64d681397b259c6fb) files: M Doc/requirements.txt diff --git a/Doc/requirements.txt b/Doc/requirements.txt index ddecdb4a5f7c2..2b8d4df214730 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -15,6 +15,6 @@ sphinxext-opengraph>=0.7.1 # The theme used by the documentation is stored separately, so we need # to install that as well. -python-docs-theme>=2022.1 +python-docs-theme>=2023.7 -c constraints.txt From webhook-mailer at python.org Fri Aug 4 19:12:16 2023 From: webhook-mailer at python.org (barneygale) Date: Fri, 04 Aug 2023 23:12:16 -0000 Subject: [Python-checkins] GH-70303: Emit FutureWarning when pathlib glob pattern ends with `**` (GH-105413) Message-ID: https://github.com/python/cpython/commit/ec0a0d2bd9faa247d5b3208a8138e4399b2da890 commit: ec0a0d2bd9faa247d5b3208a8138e4399b2da890 branch: main author: Barney Gale committer: barneygale date: 2023-08-04T23:12:12Z summary: GH-70303: Emit FutureWarning when pathlib glob pattern ends with `**` (GH-105413) In a future Python release, patterns with this ending will match both files and directories. Users may add a trailing slash to remove the warning. files: A Misc/NEWS.d/next/Library/2023-06-07-00-13-00.gh-issue-70303.frwUKH.rst M Doc/library/pathlib.rst M Lib/pathlib.py M Lib/test/test_pathlib.py diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 01dabe286969b..22360b22fd924 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -976,6 +976,11 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.13 The *follow_symlinks* parameter was added. + .. versionchanged:: 3.13 + Emits :exc:`FutureWarning` if the pattern ends with "``**``". In a + future Python release, patterns with this ending will match both files + and directories. Add a trailing slash to match only directories. + .. method:: Path.group() Return the name of the group owning the file. :exc:`KeyError` is raised diff --git a/Lib/pathlib.py b/Lib/pathlib.py index c83cf3d2ef696..758f70f5ba707 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -1069,6 +1069,11 @@ def _glob(self, pattern, case_sensitive, follow_symlinks): pattern_parts.append('') if pattern_parts[-1] == '**': # GH-70303: '**' only matches directories. Add trailing slash. + warnings.warn( + "Pattern ending '**' will match files and directories in a " + "future Python release. Add a trailing slash to match only " + "directories and remove this warning.", + FutureWarning, 3) pattern_parts.append('') if case_sensitive is None: diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 5789a932c5903..74deec84336f7 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1903,11 +1903,11 @@ def _check(glob, expected): "dirC/dirD", "dirC/dirD/fileD"]) _check(p.rglob("file*"), ["dirC/fileC", "dirC/dirD/fileD"]) _check(p.rglob("**/file*"), ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p.rglob("dir*/**"), ["dirC/dirD"]) + _check(p.rglob("dir*/**/"), ["dirC/dirD"]) _check(p.rglob("*/*"), ["dirC/dirD/fileD"]) _check(p.rglob("*/"), ["dirC/dirD"]) _check(p.rglob(""), ["dirC", "dirC/dirD"]) - _check(p.rglob("**"), ["dirC", "dirC/dirD"]) + _check(p.rglob("**/"), ["dirC", "dirC/dirD"]) # gh-91616, a re module regression _check(p.rglob("*.txt"), ["dirC/novel.txt"]) _check(p.rglob("*.*"), ["dirC/novel.txt"]) @@ -2057,7 +2057,20 @@ def test_glob_above_recursion_limit(self): path.mkdir(parents=True) with set_recursion_limit(recursion_limit): - list(base.glob('**')) + list(base.glob('**/')) + + def test_glob_recursive_no_trailing_slash(self): + P = self.cls + p = P(BASE) + with self.assertWarns(FutureWarning): + p.glob('**') + with self.assertWarns(FutureWarning): + p.glob('*/**') + with self.assertWarns(FutureWarning): + p.rglob('**') + with self.assertWarns(FutureWarning): + p.rglob('*/**') + def test_readlink(self): if not self.can_symlink: diff --git a/Misc/NEWS.d/next/Library/2023-06-07-00-13-00.gh-issue-70303.frwUKH.rst b/Misc/NEWS.d/next/Library/2023-06-07-00-13-00.gh-issue-70303.frwUKH.rst new file mode 100644 index 0000000000000..39a891ac5964a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-06-07-00-13-00.gh-issue-70303.frwUKH.rst @@ -0,0 +1,4 @@ +Emit :exc:`FutureWarning` from :meth:`pathlib.Path.glob` and +:meth:`~pathlib.Path.rglob` if the given pattern ends with "``**``". In a +future Python release, patterns with this ending will match both files and +directories. Add a trailing slash to only match directories. From webhook-mailer at python.org Fri Aug 4 19:24:54 2023 From: webhook-mailer at python.org (brandtbucher) Date: Fri, 04 Aug 2023 23:24:54 -0000 Subject: [Python-checkins] GH-84436: Skip refcounting for known immortals (GH-107605) Message-ID: https://github.com/python/cpython/commit/05a824f294f1409f33e32f1799d5b413dcf24445 commit: 05a824f294f1409f33e32f1799d5b413dcf24445 branch: main author: Brandt Bucher committer: brandtbucher date: 2023-08-04T16:24:50-07:00 summary: GH-84436: Skip refcounting for known immortals (GH-107605) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-03-13-38-14.gh-issue-84436.gl1wHx.rst M Include/internal/pycore_long.h M Modules/_asynciomodule.c M Modules/_io/textio.c M Modules/_json.c M Modules/_pickle.c M Objects/boolobject.c M Objects/bytesobject.c M Objects/funcobject.c M Objects/longobject.c M Objects/rangeobject.c M Objects/sliceobject.c M Objects/tupleobject.c M Objects/typeobject.c M Objects/unicodeobject.c M Python/ceval.c M Python/context.c M Python/fileutils.c M Python/hamt.c diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 3f8d8adc83b20..3dc00ec7e04c6 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -64,19 +64,19 @@ extern void _PyLong_FiniTypes(PyInterpreterState *interp); # error "_PY_NSMALLPOSINTS must be greater than or equal to 257" #endif -// Return a borrowed reference to the zero singleton. +// Return a reference to the immortal zero singleton. // The function cannot return NULL. static inline PyObject* _PyLong_GetZero(void) { return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS]; } -// Return a borrowed reference to the one singleton. +// Return a reference to the immortal one singleton. // The function cannot return NULL. static inline PyObject* _PyLong_GetOne(void) { return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS+1]; } static inline PyObject* _PyLong_FromUnsignedChar(unsigned char i) { - return Py_NewRef((PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS+i]); + return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS+i]; } extern PyObject *_PyLong_Add(PyLongObject *left, PyLongObject *right); diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-03-13-38-14.gh-issue-84436.gl1wHx.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-03-13-38-14.gh-issue-84436.gl1wHx.rst new file mode 100644 index 0000000000000..71044c32feebc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-03-13-38-14.gh-issue-84436.gl1wHx.rst @@ -0,0 +1 @@ +Skip reference count modifications for many known immortal objects. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index f5a589b00c48d..39c803355ba95 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -1398,7 +1398,8 @@ FutureObj_get_state(FutureObj *fut, void *Py_UNUSED(ignored)) default: assert (0); } - return Py_XNewRef(ret); + assert(_Py_IsImmortal(ret)); + return ret; } static PyObject * diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 24d846e663437..6ce90b2ed774c 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -234,7 +234,7 @@ _io_IncrementalNewlineDecoder___init___impl(nldecoder_object *self, { if (errors == NULL) { - errors = Py_NewRef(&_Py_ID(strict)); + errors = &_Py_ID(strict); } else { errors = Py_NewRef(errors); @@ -1138,7 +1138,7 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, if (encoding == NULL && _PyRuntime.preconfig.utf8_mode) { _Py_DECLARE_STR(utf_8, "utf-8"); - self->encoding = Py_NewRef(&_Py_STR(utf_8)); + self->encoding = &_Py_STR(utf_8); } else if (encoding == NULL || (strcmp(encoding, "locale") == 0)) { self->encoding = _Py_GetLocaleEncodingObject(); @@ -2267,7 +2267,7 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit) Py_CLEAR(chunks); } if (line == NULL) { - line = Py_NewRef(&_Py_STR(empty)); + line = &_Py_STR(empty); } return line; diff --git a/Modules/_json.c b/Modules/_json.c index 4fcaa07d9cfd8..c7cfe50b52faf 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1277,13 +1277,13 @@ _encoded_const(PyObject *obj) { /* Return the JSON string representation of None, True, False */ if (obj == Py_None) { - return Py_NewRef(&_Py_ID(null)); + return &_Py_ID(null); } else if (obj == Py_True) { - return Py_NewRef(&_Py_ID(true)); + return &_Py_ID(true); } else if (obj == Py_False) { - return Py_NewRef(&_Py_ID(false)); + return &_Py_ID(false); } else { PyErr_SetString(PyExc_ValueError, "not a const"); diff --git a/Modules/_pickle.c b/Modules/_pickle.c index d4c0be7893523..c2b04cc513a66 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2029,8 +2029,7 @@ whichmodule(PyObject *global, PyObject *dotted_path) } /* If no module is found, use __main__. */ - module_name = &_Py_ID(__main__); - return Py_NewRef(module_name); + return &_Py_ID(__main__); } /* fast_save_enter() and fast_save_leave() are guards against recursive diff --git a/Objects/boolobject.c b/Objects/boolobject.c index bbb187cb7121e..e2e359437f0ed 100644 --- a/Objects/boolobject.c +++ b/Objects/boolobject.c @@ -13,8 +13,7 @@ static PyObject * bool_repr(PyObject *self) { - PyObject *res = self == Py_True ? &_Py_ID(True) : &_Py_ID(False); - return Py_NewRef(res); + return self == Py_True ? &_Py_ID(True) : &_Py_ID(False); } /* Function to return a bool from a C long */ diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 6b9231a9fa769..afe9192720c62 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -41,17 +41,12 @@ Py_LOCAL_INLINE(Py_ssize_t) _PyBytesWriter_GetSize(_PyBytesWriter *writer, #define EMPTY (&_Py_SINGLETON(bytes_empty)) -// Return a borrowed reference to the empty bytes string singleton. +// Return a reference to the immortal empty bytes string singleton. static inline PyObject* bytes_get_empty(void) { - return &EMPTY->ob_base.ob_base; -} - - -// Return a strong reference to the empty bytes string singleton. -static inline PyObject* bytes_new_empty(void) -{ - return Py_NewRef(EMPTY); + PyObject *empty = &EMPTY->ob_base.ob_base; + assert(_Py_IsImmortal(empty)); + return empty; } @@ -84,7 +79,7 @@ _PyBytes_FromSize(Py_ssize_t size, int use_calloc) assert(size >= 0); if (size == 0) { - return bytes_new_empty(); + return bytes_get_empty(); } if ((size_t)size > (size_t)PY_SSIZE_T_MAX - PyBytesObject_SIZE) { @@ -123,10 +118,11 @@ PyBytes_FromStringAndSize(const char *str, Py_ssize_t size) } if (size == 1 && str != NULL) { op = CHARACTER(*str & 255); - return Py_NewRef(op); + assert(_Py_IsImmortal(op)); + return (PyObject *)op; } if (size == 0) { - return bytes_new_empty(); + return bytes_get_empty(); } op = (PyBytesObject *)_PyBytes_FromSize(size, 0); @@ -154,11 +150,12 @@ PyBytes_FromString(const char *str) } if (size == 0) { - return bytes_new_empty(); + return bytes_get_empty(); } else if (size == 1) { op = CHARACTER(*str & 255); - return Py_NewRef(op); + assert(_Py_IsImmortal(op)); + return (PyObject *)op; } /* Inline PyObject_NewVar */ @@ -3065,7 +3062,7 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) goto error; } if (newsize == 0) { - *pv = bytes_new_empty(); + *pv = bytes_get_empty(); Py_DECREF(v); return 0; } diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 7fffa1c8bbff9..8c0bface3ac71 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -831,8 +831,8 @@ func_clear(PyFunctionObject *op) // However, name and qualname could be str subclasses, so they // could have reference cycles. The solution is to replace them // with a genuinely immutable string. - Py_SETREF(op->func_name, Py_NewRef(&_Py_STR(empty))); - Py_SETREF(op->func_qualname, Py_NewRef(&_Py_STR(empty))); + Py_SETREF(op->func_name, &_Py_STR(empty)); + Py_SETREF(op->func_qualname, &_Py_STR(empty)); return 0; } diff --git a/Objects/longobject.c b/Objects/longobject.c index 5d9b413861478..354cba9d6d800 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -174,7 +174,7 @@ _PyLong_FromDigits(int negative, Py_ssize_t digit_count, digit *digits) { assert(digit_count >= 0); if (digit_count == 0) { - return (PyLongObject *)Py_NewRef(_PyLong_GetZero()); + return (PyLongObject *)_PyLong_GetZero(); } PyLongObject *result = _PyLong_New(digit_count); if (result == NULL) { @@ -2857,8 +2857,7 @@ long_divrem(PyLongObject *a, PyLongObject *b, if (*prem == NULL) { return -1; } - PyObject *zero = _PyLong_GetZero(); - *pdiv = (PyLongObject*)Py_NewRef(zero); + *pdiv = (PyLongObject*)_PyLong_GetZero(); return 0; } if (size_b == 1) { diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 1e3d5acc8ae6f..6e06bef95032c 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -106,8 +106,8 @@ range_from_array(PyTypeObject *type, PyObject *const *args, Py_ssize_t num_args) if (!stop) { return NULL; } - start = Py_NewRef(_PyLong_GetZero()); - step = Py_NewRef(_PyLong_GetOne()); + start = _PyLong_GetZero(); + step = _PyLong_GetOne(); break; case 0: PyErr_SetString(PyExc_TypeError, diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index dc3aad11ce10e..8cf654fb6f812 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -415,7 +415,7 @@ _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, /* Convert step to an integer; raise for zero step. */ if (self->step == Py_None) { - step = Py_NewRef(_PyLong_GetOne()); + step = _PyLong_GetOne(); step_is_negative = 0; } else { @@ -443,7 +443,7 @@ _PySlice_GetLongIndices(PySliceObject *self, PyObject *length, goto error; } else { - lower = Py_NewRef(_PyLong_GetZero()); + lower = _PyLong_GetZero(); upper = Py_NewRef(length); } diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index c3ff40fdb14c6..b669a3dd8525e 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -62,7 +62,7 @@ tuple_alloc(Py_ssize_t size) static inline PyObject * tuple_get_empty(void) { - return Py_NewRef(&_Py_SINGLETON(tuple_empty)); + return (PyObject *)&_Py_SINGLETON(tuple_empty); } PyObject * diff --git a/Objects/typeobject.c b/Objects/typeobject.c index abe33f1562059..76809494dd881 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1085,7 +1085,7 @@ type_module(PyTypeObject *type, void *context) PyUnicode_InternInPlace(&mod); } else { - mod = Py_NewRef(&_Py_ID(builtins)); + mod = &_Py_ID(builtins); } } return mod; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index cc979b2ef28d7..c6876d4ca0ef0 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -211,21 +211,13 @@ static int unicode_is_singleton(PyObject *unicode); #endif -// Return a borrowed reference to the empty string singleton. +// Return a reference to the immortal empty string singleton. static inline PyObject* unicode_get_empty(void) { _Py_DECLARE_STR(empty, ""); return &_Py_STR(empty); } - -// Return a strong reference to the empty string singleton. -static inline PyObject* unicode_new_empty(void) -{ - PyObject *empty = unicode_get_empty(); - return Py_NewRef(empty); -} - /* This dictionary holds all interned unicode strings. Note that references to strings in this dictionary are *not* counted in the string's ob_refcnt. When the interned string reaches a refcnt of 0 the string deallocation @@ -310,7 +302,7 @@ clear_interned_dict(PyInterpreterState *interp) #define _Py_RETURN_UNICODE_EMPTY() \ do { \ - return unicode_new_empty(); \ + return unicode_get_empty(); \ } while (0) static inline void @@ -650,7 +642,6 @@ unicode_result(PyObject *unicode) PyObject *empty = unicode_get_empty(); if (unicode != empty) { Py_DECREF(unicode); - Py_INCREF(empty); } return empty; } @@ -662,7 +653,6 @@ unicode_result(PyObject *unicode) Py_UCS1 ch = data[0]; PyObject *latin1_char = LATIN1(ch); if (unicode != latin1_char) { - Py_INCREF(latin1_char); Py_DECREF(unicode); } return latin1_char; @@ -1199,7 +1189,7 @@ PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) { /* Optimization for empty strings */ if (size == 0) { - return unicode_new_empty(); + return unicode_get_empty(); } PyObject *obj; @@ -1669,7 +1659,7 @@ unicode_resize(PyObject **p_unicode, Py_ssize_t length) return 0; if (length == 0) { - PyObject *empty = unicode_new_empty(); + PyObject *empty = unicode_get_empty(); Py_SETREF(*p_unicode, empty); return 0; } @@ -1764,7 +1754,9 @@ unicode_write_cstr(PyObject *unicode, Py_ssize_t index, static PyObject* get_latin1_char(Py_UCS1 ch) { - return Py_NewRef(LATIN1(ch)); + PyObject *o = LATIN1(ch); + assert(_Py_IsImmortal(o)); + return o; } static PyObject* @@ -1891,7 +1883,7 @@ PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) "NULL string with positive size with NULL passed to PyUnicode_FromStringAndSize"); return NULL; } - return unicode_new_empty(); + return unicode_get_empty(); } PyObject * @@ -10261,7 +10253,7 @@ replace(PyObject *self, PyObject *str1, } new_size = slen + n * (len2 - len1); if (new_size == 0) { - u = unicode_new_empty(); + u = unicode_get_empty(); goto done; } if (new_size > (PY_SSIZE_T_MAX / rkind)) { @@ -14505,7 +14497,7 @@ unicode_new_impl(PyTypeObject *type, PyObject *x, const char *encoding, { PyObject *unicode; if (x == NULL) { - unicode = unicode_new_empty(); + unicode = unicode_get_empty(); } else if (encoding == NULL && errors == NULL) { unicode = PyObject_Str(x); @@ -14994,8 +14986,7 @@ unicode_ascii_iter_next(unicodeiterobject *it) Py_UCS1 chr = (Py_UCS1)PyUnicode_READ(PyUnicode_1BYTE_KIND, data, it->it_index); it->it_index++; - PyObject *item = (PyObject*)&_Py_SINGLETON(strings).ascii[chr]; - return Py_NewRef(item); + return (PyObject*)&_Py_SINGLETON(strings).ascii[chr]; } it->it_seq = NULL; Py_DECREF(seq); @@ -15025,7 +15016,7 @@ unicodeiter_reduce(unicodeiterobject *it, PyObject *Py_UNUSED(ignored)) if (it->it_seq != NULL) { return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } else { - PyObject *u = unicode_new_empty(); + PyObject *u = unicode_get_empty(); if (u == NULL) { Py_XDECREF(iter); return NULL; diff --git a/Python/ceval.c b/Python/ceval.c index b85e9677747af..30f722e45e476 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1265,7 +1265,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, if (co->co_flags & CO_VARARGS) { PyObject *u = NULL; if (argcount == n) { - u = Py_NewRef(&_Py_SINGLETON(tuple_empty)); + u = (PyObject *)&_Py_SINGLETON(tuple_empty); } else { assert(args != NULL); diff --git a/Python/context.c b/Python/context.c index 9ac51874fb5be..c94c014219d0e 100644 --- a/Python/context.c +++ b/Python/context.c @@ -1267,7 +1267,7 @@ PyTypeObject _PyContextTokenMissing_Type = { static PyObject * get_token_missing(void) { - return Py_NewRef(&_Py_SINGLETON(context_token_missing)); + return (PyObject *)&_Py_SINGLETON(context_token_missing); } diff --git a/Python/fileutils.c b/Python/fileutils.c index f262c3e095c9b..19b23f6bd18b3 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -105,7 +105,7 @@ _Py_device_encoding(int fd) #else if (_PyRuntime.preconfig.utf8_mode) { _Py_DECLARE_STR(utf_8, "utf-8"); - return Py_NewRef(&_Py_STR(utf_8)); + return &_Py_STR(utf_8); } return _Py_GetLocaleEncodingObject(); #endif diff --git a/Python/hamt.c b/Python/hamt.c index c78b5a7fab94f..24265edc2c3fd 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -514,7 +514,7 @@ hamt_node_bitmap_new(Py_ssize_t size) /* Since bitmap nodes are immutable, we can cache the instance for size=0 and reuse it whenever we need an empty bitmap node. */ - return (PyHamtNode *)Py_NewRef(&_Py_SINGLETON(hamt_bitmap_node_empty)); + return (PyHamtNode *)&_Py_SINGLETON(hamt_bitmap_node_empty); } assert(size >= 0); From webhook-mailer at python.org Sat Aug 5 00:10:50 2023 From: webhook-mailer at python.org (gvanrossum) Date: Sat, 05 Aug 2023 04:10:50 -0000 Subject: [Python-checkins] gh-106608: make uop trace variable length (#107531) Message-ID: https://github.com/python/cpython/commit/4e6fac7fcc31fc6198fcddc612688b0a05ff7ae4 commit: 4e6fac7fcc31fc6198fcddc612688b0a05ff7ae4 branch: main author: Ivin Lee <62840497+rdrf2838 at users.noreply.github.com> committer: gvanrossum date: 2023-08-04T21:10:46-07:00 summary: gh-106608: make uop trace variable length (#107531) Executors are now more like tuples. files: A Misc/NEWS.d/next/Core and Builtins/2023-08-01-09-41-36.gh-issue-106608.OFZogw.rst M Include/cpython/optimizer.h M Include/internal/pycore_uops.h M Python/optimizer.c diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 2260501bfd608..da34ec1882a53 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -12,7 +12,7 @@ typedef struct { } _PyVMData; typedef struct _PyExecutorObject { - PyObject_HEAD + PyObject_VAR_HEAD /* WARNING: execute consumes a reference to self. This is necessary to allow executors to tail call into each other. */ struct _PyInterpreterFrame *(*execute)(struct _PyExecutorObject *self, struct _PyInterpreterFrame *frame, PyObject **stack_pointer); _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h index edb141cc79f75..57a5970353b36 100644 --- a/Include/internal/pycore_uops.h +++ b/Include/internal/pycore_uops.h @@ -18,7 +18,7 @@ typedef struct { typedef struct { _PyExecutorObject base; - _PyUOpInstruction trace[_Py_UOP_MAX_TRACE_LENGTH]; // TODO: variable length + _PyUOpInstruction trace[1]; } _PyUOpExecutorObject; _PyInterpreterFrame *_PyUopExecute( diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-01-09-41-36.gh-issue-106608.OFZogw.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-01-09-41-36.gh-issue-106608.OFZogw.rst new file mode 100644 index 0000000000000..20d43a7c4f754 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-01-09-41-36.gh-issue-106608.OFZogw.rst @@ -0,0 +1 @@ +Make ``_PyUOpExecutorObject`` variable length. diff --git a/Python/optimizer.c b/Python/optimizer.c index 238ab02d09faa..d2ed8dfcd2cff 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -320,13 +320,7 @@ uop_name(int index) { static Py_ssize_t uop_len(_PyUOpExecutorObject *self) { - int count = 0; - for (; count < _Py_UOP_MAX_TRACE_LENGTH; count++) { - if (self->trace[count].opcode == 0) { - break; - } - } - return count; + return Py_SIZE(self); } static PyObject * @@ -368,8 +362,8 @@ PySequenceMethods uop_as_sequence = { static PyTypeObject UOpExecutor_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) .tp_name = "uop_executor", - .tp_basicsize = sizeof(_PyUOpExecutorObject), - .tp_itemsize = 0, + .tp_basicsize = sizeof(_PyUOpExecutorObject) - sizeof(_PyUOpInstruction), + .tp_itemsize = sizeof(_PyUOpInstruction), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_dealloc = (destructor)uop_dealloc, .tp_as_sequence = &uop_as_sequence, @@ -699,15 +693,12 @@ uop_optimize( return trace_length; } OBJECT_STAT_INC(optimization_traces_created); - _PyUOpExecutorObject *executor = PyObject_New(_PyUOpExecutorObject, &UOpExecutor_Type); + _PyUOpExecutorObject *executor = PyObject_NewVar(_PyUOpExecutorObject, &UOpExecutor_Type, trace_length); if (executor == NULL) { return -1; } executor->base.execute = _PyUopExecute; memcpy(executor->trace, trace, trace_length * sizeof(_PyUOpInstruction)); - if (trace_length < _Py_UOP_MAX_TRACE_LENGTH) { - executor->trace[trace_length].opcode = 0; // Sentinel - } *exec_ptr = (_PyExecutorObject *)executor; return 1; } From webhook-mailer at python.org Sat Aug 5 00:50:40 2023 From: webhook-mailer at python.org (gvanrossum) Date: Sat, 05 Aug 2023 04:50:40 -0000 Subject: [Python-checkins] gh-106812: Fix two tiny bugs in analysis.py (#107649) Message-ID: https://github.com/python/cpython/commit/85e5b1f5b806289744ef9a5a13dabfb23044f713 commit: 85e5b1f5b806289744ef9a5a13dabfb23044f713 branch: main author: Guido van Rossum committer: gvanrossum date: 2023-08-05T04:50:36Z summary: gh-106812: Fix two tiny bugs in analysis.py (#107649) This fixes two tiny defects in analysis.py that I didn't catch on time in #107564: - `get_var_names` in `check_macro_consistency` should skip `UNUSED` names. - Fix an occurrence of `is UNUSED` (should be `==`). files: M Tools/cases_generator/analysis.py diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index bd8918a87ffe0..2db1cd01c19ae 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -297,6 +297,8 @@ def check_macro_consistency(self, mac: MacroInstruction) -> None: def get_var_names(instr: Instruction) -> dict[str, StackEffect]: vars: dict[str, StackEffect] = {} for eff in instr.input_effects + instr.output_effects: + if eff.name == UNUSED: + continue if eff.name in vars: if vars[eff.name] != eff: self.error( @@ -335,7 +337,7 @@ def get_var_names(instr: Instruction) -> dict[str, StackEffect]: copies: list[tuple[StackEffect, StackEffect]] = [] while pushes and pops and pushes[-1] == pops[0]: src, dst = pushes.pop(), pops.pop(0) - if src.name == dst.name or dst.name is UNUSED: + if src.name == dst.name or dst.name == UNUSED: continue copies.append((src, dst)) reads = set(copy[0].name for copy in copies) From webhook-mailer at python.org Sat Aug 5 07:28:31 2023 From: webhook-mailer at python.org (hugovk) Date: Sat, 05 Aug 2023 11:28:31 -0000 Subject: [Python-checkins] gh-107432 Fix incorrect indentation in annotations HOWTO (#107445) Message-ID: https://github.com/python/cpython/commit/5e2746d6e2fb0da29225ead7135f078c5f087b57 commit: 5e2746d6e2fb0da29225ead7135f078c5f087b57 branch: main author: Daniele Procida committer: hugovk date: 2023-08-05T14:28:28+03:00 summary: gh-107432 Fix incorrect indentation in annotations HOWTO (#107445) gh-107432 Fix incorrect indentation in annotations document Body text in https://docs.python.org/3/howto/annotations.html was indented throughout, and was being rendered in blockquote elements. files: M Doc/howto/annotations.rst diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 472069032d650..1134686c947d6 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -32,201 +32,201 @@ Annotations Best Practices Accessing The Annotations Dict Of An Object In Python 3.10 And Newer ==================================================================== - Python 3.10 adds a new function to the standard library: - :func:`inspect.get_annotations`. In Python versions 3.10 - and newer, calling this function is the best practice for - accessing the annotations dict of any object that supports - annotations. This function can also "un-stringize" - stringized annotations for you. - - If for some reason :func:`inspect.get_annotations` isn't - viable for your use case, you may access the - ``__annotations__`` data member manually. Best practice - for this changed in Python 3.10 as well: as of Python 3.10, - ``o.__annotations__`` is guaranteed to *always* work - on Python functions, classes, and modules. If you're - certain the object you're examining is one of these three - *specific* objects, you may simply use ``o.__annotations__`` - to get at the object's annotations dict. - - However, other types of callables--for example, - callables created by :func:`functools.partial`--may - not have an ``__annotations__`` attribute defined. When - accessing the ``__annotations__`` of a possibly unknown - object, best practice in Python versions 3.10 and - newer is to call :func:`getattr` with three arguments, - for example ``getattr(o, '__annotations__', None)``. - - Before Python 3.10, accessing ``__annotations__`` on a class that - defines no annotations but that has a parent class with - annotations would return the parent's ``__annotations__``. - In Python 3.10 and newer, the child class's annotations - will be an empty dict instead. +Python 3.10 adds a new function to the standard library: +:func:`inspect.get_annotations`. In Python versions 3.10 +and newer, calling this function is the best practice for +accessing the annotations dict of any object that supports +annotations. This function can also "un-stringize" +stringized annotations for you. + +If for some reason :func:`inspect.get_annotations` isn't +viable for your use case, you may access the +``__annotations__`` data member manually. Best practice +for this changed in Python 3.10 as well: as of Python 3.10, +``o.__annotations__`` is guaranteed to *always* work +on Python functions, classes, and modules. If you're +certain the object you're examining is one of these three +*specific* objects, you may simply use ``o.__annotations__`` +to get at the object's annotations dict. + +However, other types of callables--for example, +callables created by :func:`functools.partial`--may +not have an ``__annotations__`` attribute defined. When +accessing the ``__annotations__`` of a possibly unknown +object, best practice in Python versions 3.10 and +newer is to call :func:`getattr` with three arguments, +for example ``getattr(o, '__annotations__', None)``. + +Before Python 3.10, accessing ``__annotations__`` on a class that +defines no annotations but that has a parent class with +annotations would return the parent's ``__annotations__``. +In Python 3.10 and newer, the child class's annotations +will be an empty dict instead. Accessing The Annotations Dict Of An Object In Python 3.9 And Older =================================================================== - In Python 3.9 and older, accessing the annotations dict - of an object is much more complicated than in newer versions. - The problem is a design flaw in these older versions of Python, - specifically to do with class annotations. +In Python 3.9 and older, accessing the annotations dict +of an object is much more complicated than in newer versions. +The problem is a design flaw in these older versions of Python, +specifically to do with class annotations. - Best practice for accessing the annotations dict of other - objects--functions, other callables, and modules--is the same - as best practice for 3.10, assuming you aren't calling - :func:`inspect.get_annotations`: you should use three-argument - :func:`getattr` to access the object's ``__annotations__`` - attribute. +Best practice for accessing the annotations dict of other +objects--functions, other callables, and modules--is the same +as best practice for 3.10, assuming you aren't calling +:func:`inspect.get_annotations`: you should use three-argument +:func:`getattr` to access the object's ``__annotations__`` +attribute. - Unfortunately, this isn't best practice for classes. The problem - is that, since ``__annotations__`` is optional on classes, and - because classes can inherit attributes from their base classes, - accessing the ``__annotations__`` attribute of a class may - inadvertently return the annotations dict of a *base class.* - As an example:: +Unfortunately, this isn't best practice for classes. The problem +is that, since ``__annotations__`` is optional on classes, and +because classes can inherit attributes from their base classes, +accessing the ``__annotations__`` attribute of a class may +inadvertently return the annotations dict of a *base class.* +As an example:: - class Base: - a: int = 3 - b: str = 'abc' + class Base: + a: int = 3 + b: str = 'abc' - class Derived(Base): - pass + class Derived(Base): + pass - print(Derived.__annotations__) + print(Derived.__annotations__) - This will print the annotations dict from ``Base``, not - ``Derived``. +This will print the annotations dict from ``Base``, not +``Derived``. - Your code will have to have a separate code path if the object - you're examining is a class (``isinstance(o, type)``). - In that case, best practice relies on an implementation detail - of Python 3.9 and before: if a class has annotations defined, - they are stored in the class's ``__dict__`` dictionary. Since - the class may or may not have annotations defined, best practice - is to call the ``get`` method on the class dict. +Your code will have to have a separate code path if the object +you're examining is a class (``isinstance(o, type)``). +In that case, best practice relies on an implementation detail +of Python 3.9 and before: if a class has annotations defined, +they are stored in the class's ``__dict__`` dictionary. Since +the class may or may not have annotations defined, best practice +is to call the ``get`` method on the class dict. - To put it all together, here is some sample code that safely - accesses the ``__annotations__`` attribute on an arbitrary - object in Python 3.9 and before:: +To put it all together, here is some sample code that safely +accesses the ``__annotations__`` attribute on an arbitrary +object in Python 3.9 and before:: - if isinstance(o, type): - ann = o.__dict__.get('__annotations__', None) - else: - ann = getattr(o, '__annotations__', None) + if isinstance(o, type): + ann = o.__dict__.get('__annotations__', None) + else: + ann = getattr(o, '__annotations__', None) - After running this code, ``ann`` should be either a - dictionary or ``None``. You're encouraged to double-check - the type of ``ann`` using :func:`isinstance` before further - examination. +After running this code, ``ann`` should be either a +dictionary or ``None``. You're encouraged to double-check +the type of ``ann`` using :func:`isinstance` before further +examination. - Note that some exotic or malformed type objects may not have - a ``__dict__`` attribute, so for extra safety you may also wish - to use :func:`getattr` to access ``__dict__``. +Note that some exotic or malformed type objects may not have +a ``__dict__`` attribute, so for extra safety you may also wish +to use :func:`getattr` to access ``__dict__``. Manually Un-Stringizing Stringized Annotations ============================================== - In situations where some annotations may be "stringized", - and you wish to evaluate those strings to produce the - Python values they represent, it really is best to - call :func:`inspect.get_annotations` to do this work - for you. - - If you're using Python 3.9 or older, or if for some reason - you can't use :func:`inspect.get_annotations`, you'll need - to duplicate its logic. You're encouraged to examine the - implementation of :func:`inspect.get_annotations` in the - current Python version and follow a similar approach. - - In a nutshell, if you wish to evaluate a stringized annotation - on an arbitrary object ``o``: - - * If ``o`` is a module, use ``o.__dict__`` as the - ``globals`` when calling :func:`eval`. - * If ``o`` is a class, use ``sys.modules[o.__module__].__dict__`` - as the ``globals``, and ``dict(vars(o))`` as the ``locals``, - when calling :func:`eval`. - * If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, - :func:`functools.wraps`, or :func:`functools.partial`, iteratively - unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as - appropriate, until you have found the root unwrapped function. - * If ``o`` is a callable (but not a class), use - ``o.__globals__`` as the globals when calling :func:`eval`. - - However, not all string values used as annotations can - be successfully turned into Python values by :func:`eval`. - String values could theoretically contain any valid string, - and in practice there are valid use cases for type hints that - require annotating with string values that specifically - *can't* be evaluated. For example: - - * :pep:`604` union types using ``|``, before support for this - was added to Python 3.10. - * Definitions that aren't needed at runtime, only imported - when :const:`typing.TYPE_CHECKING` is true. - - If :func:`eval` attempts to evaluate such values, it will - fail and raise an exception. So, when designing a library - API that works with annotations, it's recommended to only - attempt to evaluate string values when explicitly requested - to by the caller. +In situations where some annotations may be "stringized", +and you wish to evaluate those strings to produce the +Python values they represent, it really is best to +call :func:`inspect.get_annotations` to do this work +for you. + +If you're using Python 3.9 or older, or if for some reason +you can't use :func:`inspect.get_annotations`, you'll need +to duplicate its logic. You're encouraged to examine the +implementation of :func:`inspect.get_annotations` in the +current Python version and follow a similar approach. + +In a nutshell, if you wish to evaluate a stringized annotation +on an arbitrary object ``o``: + +* If ``o`` is a module, use ``o.__dict__`` as the + ``globals`` when calling :func:`eval`. +* If ``o`` is a class, use ``sys.modules[o.__module__].__dict__`` + as the ``globals``, and ``dict(vars(o))`` as the ``locals``, + when calling :func:`eval`. +* If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, + :func:`functools.wraps`, or :func:`functools.partial`, iteratively + unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as + appropriate, until you have found the root unwrapped function. +* If ``o`` is a callable (but not a class), use + ``o.__globals__`` as the globals when calling :func:`eval`. + +However, not all string values used as annotations can +be successfully turned into Python values by :func:`eval`. +String values could theoretically contain any valid string, +and in practice there are valid use cases for type hints that +require annotating with string values that specifically +*can't* be evaluated. For example: + +* :pep:`604` union types using ``|``, before support for this + was added to Python 3.10. +* Definitions that aren't needed at runtime, only imported + when :const:`typing.TYPE_CHECKING` is true. + +If :func:`eval` attempts to evaluate such values, it will +fail and raise an exception. So, when designing a library +API that works with annotations, it's recommended to only +attempt to evaluate string values when explicitly requested +to by the caller. Best Practices For ``__annotations__`` In Any Python Version ============================================================ - * You should avoid assigning to the ``__annotations__`` member - of objects directly. Let Python manage setting ``__annotations__``. +* You should avoid assigning to the ``__annotations__`` member + of objects directly. Let Python manage setting ``__annotations__``. - * If you do assign directly to the ``__annotations__`` member - of an object, you should always set it to a ``dict`` object. +* If you do assign directly to the ``__annotations__`` member + of an object, you should always set it to a ``dict`` object. - * If you directly access the ``__annotations__`` member - of an object, you should ensure that it's a - dictionary before attempting to examine its contents. +* If you directly access the ``__annotations__`` member + of an object, you should ensure that it's a + dictionary before attempting to examine its contents. - * You should avoid modifying ``__annotations__`` dicts. +* You should avoid modifying ``__annotations__`` dicts. - * You should avoid deleting the ``__annotations__`` attribute - of an object. +* You should avoid deleting the ``__annotations__`` attribute + of an object. ``__annotations__`` Quirks ========================== - In all versions of Python 3, function - objects lazy-create an annotations dict if no annotations - are defined on that object. You can delete the ``__annotations__`` - attribute using ``del fn.__annotations__``, but if you then - access ``fn.__annotations__`` the object will create a new empty dict - that it will store and return as its annotations. Deleting the - annotations on a function before it has lazily created its annotations - dict will throw an ``AttributeError``; using ``del fn.__annotations__`` - twice in a row is guaranteed to always throw an ``AttributeError``. - - Everything in the above paragraph also applies to class and module - objects in Python 3.10 and newer. - - In all versions of Python 3, you can set ``__annotations__`` - on a function object to ``None``. However, subsequently - accessing the annotations on that object using ``fn.__annotations__`` - will lazy-create an empty dictionary as per the first paragraph of - this section. This is *not* true of modules and classes, in any Python - version; those objects permit setting ``__annotations__`` to any - Python value, and will retain whatever value is set. - - If Python stringizes your annotations for you - (using ``from __future__ import annotations``), and you - specify a string as an annotation, the string will - itself be quoted. In effect the annotation is quoted - *twice.* For example:: - - from __future__ import annotations - def foo(a: "str"): pass - - print(foo.__annotations__) - - This prints ``{'a': "'str'"}``. This shouldn't really be considered - a "quirk"; it's mentioned here simply because it might be surprising. +In all versions of Python 3, function +objects lazy-create an annotations dict if no annotations +are defined on that object. You can delete the ``__annotations__`` +attribute using ``del fn.__annotations__``, but if you then +access ``fn.__annotations__`` the object will create a new empty dict +that it will store and return as its annotations. Deleting the +annotations on a function before it has lazily created its annotations +dict will throw an ``AttributeError``; using ``del fn.__annotations__`` +twice in a row is guaranteed to always throw an ``AttributeError``. + +Everything in the above paragraph also applies to class and module +objects in Python 3.10 and newer. + +In all versions of Python 3, you can set ``__annotations__`` +on a function object to ``None``. However, subsequently +accessing the annotations on that object using ``fn.__annotations__`` +will lazy-create an empty dictionary as per the first paragraph of +this section. This is *not* true of modules and classes, in any Python +version; those objects permit setting ``__annotations__`` to any +Python value, and will retain whatever value is set. + +If Python stringizes your annotations for you +(using ``from __future__ import annotations``), and you +specify a string as an annotation, the string will +itself be quoted. In effect the annotation is quoted +*twice.* For example:: + + from __future__ import annotations + def foo(a: "str"): pass + + print(foo.__annotations__) + +This prints ``{'a': "'str'"}``. This shouldn't really be considered +a "quirk"; it's mentioned here simply because it might be surprising. From webhook-mailer at python.org Sat Aug 5 07:40:48 2023 From: webhook-mailer at python.org (hugovk) Date: Sat, 05 Aug 2023 11:40:48 -0000 Subject: [Python-checkins] [3.11] gh-107432 Fix incorrect indentation in annotations HOWTO (GH-107445) (#107655) Message-ID: https://github.com/python/cpython/commit/7e834c45541dbc000a8fcff7d70228eb948dea74 commit: 7e834c45541dbc000a8fcff7d70228eb948dea74 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: hugovk date: 2023-08-05T14:40:44+03:00 summary: [3.11] gh-107432 Fix incorrect indentation in annotations HOWTO (GH-107445) (#107655) gh-107432 Fix incorrect indentation in annotations HOWTO (GH-107445) gh-107432 Fix incorrect indentation in annotations document Body text in https://docs.python.org/3/howto/annotations.html was indented throughout, and was being rendered in blockquote elements. (cherry picked from commit 5e2746d6e2fb0da29225ead7135f078c5f087b57) Co-authored-by: Daniele Procida files: M Doc/howto/annotations.rst diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 472069032d650..1134686c947d6 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -32,201 +32,201 @@ Annotations Best Practices Accessing The Annotations Dict Of An Object In Python 3.10 And Newer ==================================================================== - Python 3.10 adds a new function to the standard library: - :func:`inspect.get_annotations`. In Python versions 3.10 - and newer, calling this function is the best practice for - accessing the annotations dict of any object that supports - annotations. This function can also "un-stringize" - stringized annotations for you. - - If for some reason :func:`inspect.get_annotations` isn't - viable for your use case, you may access the - ``__annotations__`` data member manually. Best practice - for this changed in Python 3.10 as well: as of Python 3.10, - ``o.__annotations__`` is guaranteed to *always* work - on Python functions, classes, and modules. If you're - certain the object you're examining is one of these three - *specific* objects, you may simply use ``o.__annotations__`` - to get at the object's annotations dict. - - However, other types of callables--for example, - callables created by :func:`functools.partial`--may - not have an ``__annotations__`` attribute defined. When - accessing the ``__annotations__`` of a possibly unknown - object, best practice in Python versions 3.10 and - newer is to call :func:`getattr` with three arguments, - for example ``getattr(o, '__annotations__', None)``. - - Before Python 3.10, accessing ``__annotations__`` on a class that - defines no annotations but that has a parent class with - annotations would return the parent's ``__annotations__``. - In Python 3.10 and newer, the child class's annotations - will be an empty dict instead. +Python 3.10 adds a new function to the standard library: +:func:`inspect.get_annotations`. In Python versions 3.10 +and newer, calling this function is the best practice for +accessing the annotations dict of any object that supports +annotations. This function can also "un-stringize" +stringized annotations for you. + +If for some reason :func:`inspect.get_annotations` isn't +viable for your use case, you may access the +``__annotations__`` data member manually. Best practice +for this changed in Python 3.10 as well: as of Python 3.10, +``o.__annotations__`` is guaranteed to *always* work +on Python functions, classes, and modules. If you're +certain the object you're examining is one of these three +*specific* objects, you may simply use ``o.__annotations__`` +to get at the object's annotations dict. + +However, other types of callables--for example, +callables created by :func:`functools.partial`--may +not have an ``__annotations__`` attribute defined. When +accessing the ``__annotations__`` of a possibly unknown +object, best practice in Python versions 3.10 and +newer is to call :func:`getattr` with three arguments, +for example ``getattr(o, '__annotations__', None)``. + +Before Python 3.10, accessing ``__annotations__`` on a class that +defines no annotations but that has a parent class with +annotations would return the parent's ``__annotations__``. +In Python 3.10 and newer, the child class's annotations +will be an empty dict instead. Accessing The Annotations Dict Of An Object In Python 3.9 And Older =================================================================== - In Python 3.9 and older, accessing the annotations dict - of an object is much more complicated than in newer versions. - The problem is a design flaw in these older versions of Python, - specifically to do with class annotations. +In Python 3.9 and older, accessing the annotations dict +of an object is much more complicated than in newer versions. +The problem is a design flaw in these older versions of Python, +specifically to do with class annotations. - Best practice for accessing the annotations dict of other - objects--functions, other callables, and modules--is the same - as best practice for 3.10, assuming you aren't calling - :func:`inspect.get_annotations`: you should use three-argument - :func:`getattr` to access the object's ``__annotations__`` - attribute. +Best practice for accessing the annotations dict of other +objects--functions, other callables, and modules--is the same +as best practice for 3.10, assuming you aren't calling +:func:`inspect.get_annotations`: you should use three-argument +:func:`getattr` to access the object's ``__annotations__`` +attribute. - Unfortunately, this isn't best practice for classes. The problem - is that, since ``__annotations__`` is optional on classes, and - because classes can inherit attributes from their base classes, - accessing the ``__annotations__`` attribute of a class may - inadvertently return the annotations dict of a *base class.* - As an example:: +Unfortunately, this isn't best practice for classes. The problem +is that, since ``__annotations__`` is optional on classes, and +because classes can inherit attributes from their base classes, +accessing the ``__annotations__`` attribute of a class may +inadvertently return the annotations dict of a *base class.* +As an example:: - class Base: - a: int = 3 - b: str = 'abc' + class Base: + a: int = 3 + b: str = 'abc' - class Derived(Base): - pass + class Derived(Base): + pass - print(Derived.__annotations__) + print(Derived.__annotations__) - This will print the annotations dict from ``Base``, not - ``Derived``. +This will print the annotations dict from ``Base``, not +``Derived``. - Your code will have to have a separate code path if the object - you're examining is a class (``isinstance(o, type)``). - In that case, best practice relies on an implementation detail - of Python 3.9 and before: if a class has annotations defined, - they are stored in the class's ``__dict__`` dictionary. Since - the class may or may not have annotations defined, best practice - is to call the ``get`` method on the class dict. +Your code will have to have a separate code path if the object +you're examining is a class (``isinstance(o, type)``). +In that case, best practice relies on an implementation detail +of Python 3.9 and before: if a class has annotations defined, +they are stored in the class's ``__dict__`` dictionary. Since +the class may or may not have annotations defined, best practice +is to call the ``get`` method on the class dict. - To put it all together, here is some sample code that safely - accesses the ``__annotations__`` attribute on an arbitrary - object in Python 3.9 and before:: +To put it all together, here is some sample code that safely +accesses the ``__annotations__`` attribute on an arbitrary +object in Python 3.9 and before:: - if isinstance(o, type): - ann = o.__dict__.get('__annotations__', None) - else: - ann = getattr(o, '__annotations__', None) + if isinstance(o, type): + ann = o.__dict__.get('__annotations__', None) + else: + ann = getattr(o, '__annotations__', None) - After running this code, ``ann`` should be either a - dictionary or ``None``. You're encouraged to double-check - the type of ``ann`` using :func:`isinstance` before further - examination. +After running this code, ``ann`` should be either a +dictionary or ``None``. You're encouraged to double-check +the type of ``ann`` using :func:`isinstance` before further +examination. - Note that some exotic or malformed type objects may not have - a ``__dict__`` attribute, so for extra safety you may also wish - to use :func:`getattr` to access ``__dict__``. +Note that some exotic or malformed type objects may not have +a ``__dict__`` attribute, so for extra safety you may also wish +to use :func:`getattr` to access ``__dict__``. Manually Un-Stringizing Stringized Annotations ============================================== - In situations where some annotations may be "stringized", - and you wish to evaluate those strings to produce the - Python values they represent, it really is best to - call :func:`inspect.get_annotations` to do this work - for you. - - If you're using Python 3.9 or older, or if for some reason - you can't use :func:`inspect.get_annotations`, you'll need - to duplicate its logic. You're encouraged to examine the - implementation of :func:`inspect.get_annotations` in the - current Python version and follow a similar approach. - - In a nutshell, if you wish to evaluate a stringized annotation - on an arbitrary object ``o``: - - * If ``o`` is a module, use ``o.__dict__`` as the - ``globals`` when calling :func:`eval`. - * If ``o`` is a class, use ``sys.modules[o.__module__].__dict__`` - as the ``globals``, and ``dict(vars(o))`` as the ``locals``, - when calling :func:`eval`. - * If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, - :func:`functools.wraps`, or :func:`functools.partial`, iteratively - unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as - appropriate, until you have found the root unwrapped function. - * If ``o`` is a callable (but not a class), use - ``o.__globals__`` as the globals when calling :func:`eval`. - - However, not all string values used as annotations can - be successfully turned into Python values by :func:`eval`. - String values could theoretically contain any valid string, - and in practice there are valid use cases for type hints that - require annotating with string values that specifically - *can't* be evaluated. For example: - - * :pep:`604` union types using ``|``, before support for this - was added to Python 3.10. - * Definitions that aren't needed at runtime, only imported - when :const:`typing.TYPE_CHECKING` is true. - - If :func:`eval` attempts to evaluate such values, it will - fail and raise an exception. So, when designing a library - API that works with annotations, it's recommended to only - attempt to evaluate string values when explicitly requested - to by the caller. +In situations where some annotations may be "stringized", +and you wish to evaluate those strings to produce the +Python values they represent, it really is best to +call :func:`inspect.get_annotations` to do this work +for you. + +If you're using Python 3.9 or older, or if for some reason +you can't use :func:`inspect.get_annotations`, you'll need +to duplicate its logic. You're encouraged to examine the +implementation of :func:`inspect.get_annotations` in the +current Python version and follow a similar approach. + +In a nutshell, if you wish to evaluate a stringized annotation +on an arbitrary object ``o``: + +* If ``o`` is a module, use ``o.__dict__`` as the + ``globals`` when calling :func:`eval`. +* If ``o`` is a class, use ``sys.modules[o.__module__].__dict__`` + as the ``globals``, and ``dict(vars(o))`` as the ``locals``, + when calling :func:`eval`. +* If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, + :func:`functools.wraps`, or :func:`functools.partial`, iteratively + unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as + appropriate, until you have found the root unwrapped function. +* If ``o`` is a callable (but not a class), use + ``o.__globals__`` as the globals when calling :func:`eval`. + +However, not all string values used as annotations can +be successfully turned into Python values by :func:`eval`. +String values could theoretically contain any valid string, +and in practice there are valid use cases for type hints that +require annotating with string values that specifically +*can't* be evaluated. For example: + +* :pep:`604` union types using ``|``, before support for this + was added to Python 3.10. +* Definitions that aren't needed at runtime, only imported + when :const:`typing.TYPE_CHECKING` is true. + +If :func:`eval` attempts to evaluate such values, it will +fail and raise an exception. So, when designing a library +API that works with annotations, it's recommended to only +attempt to evaluate string values when explicitly requested +to by the caller. Best Practices For ``__annotations__`` In Any Python Version ============================================================ - * You should avoid assigning to the ``__annotations__`` member - of objects directly. Let Python manage setting ``__annotations__``. +* You should avoid assigning to the ``__annotations__`` member + of objects directly. Let Python manage setting ``__annotations__``. - * If you do assign directly to the ``__annotations__`` member - of an object, you should always set it to a ``dict`` object. +* If you do assign directly to the ``__annotations__`` member + of an object, you should always set it to a ``dict`` object. - * If you directly access the ``__annotations__`` member - of an object, you should ensure that it's a - dictionary before attempting to examine its contents. +* If you directly access the ``__annotations__`` member + of an object, you should ensure that it's a + dictionary before attempting to examine its contents. - * You should avoid modifying ``__annotations__`` dicts. +* You should avoid modifying ``__annotations__`` dicts. - * You should avoid deleting the ``__annotations__`` attribute - of an object. +* You should avoid deleting the ``__annotations__`` attribute + of an object. ``__annotations__`` Quirks ========================== - In all versions of Python 3, function - objects lazy-create an annotations dict if no annotations - are defined on that object. You can delete the ``__annotations__`` - attribute using ``del fn.__annotations__``, but if you then - access ``fn.__annotations__`` the object will create a new empty dict - that it will store and return as its annotations. Deleting the - annotations on a function before it has lazily created its annotations - dict will throw an ``AttributeError``; using ``del fn.__annotations__`` - twice in a row is guaranteed to always throw an ``AttributeError``. - - Everything in the above paragraph also applies to class and module - objects in Python 3.10 and newer. - - In all versions of Python 3, you can set ``__annotations__`` - on a function object to ``None``. However, subsequently - accessing the annotations on that object using ``fn.__annotations__`` - will lazy-create an empty dictionary as per the first paragraph of - this section. This is *not* true of modules and classes, in any Python - version; those objects permit setting ``__annotations__`` to any - Python value, and will retain whatever value is set. - - If Python stringizes your annotations for you - (using ``from __future__ import annotations``), and you - specify a string as an annotation, the string will - itself be quoted. In effect the annotation is quoted - *twice.* For example:: - - from __future__ import annotations - def foo(a: "str"): pass - - print(foo.__annotations__) - - This prints ``{'a': "'str'"}``. This shouldn't really be considered - a "quirk"; it's mentioned here simply because it might be surprising. +In all versions of Python 3, function +objects lazy-create an annotations dict if no annotations +are defined on that object. You can delete the ``__annotations__`` +attribute using ``del fn.__annotations__``, but if you then +access ``fn.__annotations__`` the object will create a new empty dict +that it will store and return as its annotations. Deleting the +annotations on a function before it has lazily created its annotations +dict will throw an ``AttributeError``; using ``del fn.__annotations__`` +twice in a row is guaranteed to always throw an ``AttributeError``. + +Everything in the above paragraph also applies to class and module +objects in Python 3.10 and newer. + +In all versions of Python 3, you can set ``__annotations__`` +on a function object to ``None``. However, subsequently +accessing the annotations on that object using ``fn.__annotations__`` +will lazy-create an empty dictionary as per the first paragraph of +this section. This is *not* true of modules and classes, in any Python +version; those objects permit setting ``__annotations__`` to any +Python value, and will retain whatever value is set. + +If Python stringizes your annotations for you +(using ``from __future__ import annotations``), and you +specify a string as an annotation, the string will +itself be quoted. In effect the annotation is quoted +*twice.* For example:: + + from __future__ import annotations + def foo(a: "str"): pass + + print(foo.__annotations__) + +This prints ``{'a': "'str'"}``. This shouldn't really be considered +a "quirk"; it's mentioned here simply because it might be surprising. From webhook-mailer at python.org Sat Aug 5 07:44:58 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sat, 05 Aug 2023 11:44:58 -0000 Subject: [Python-checkins] [3.12] gh-107630: Revert "[3.12] gh-107080: Fix Py_TRACE_REFS Crashes Under Isolated Subinterpreters (gh-107567) (#107599)" (#107648) Message-ID: https://github.com/python/cpython/commit/6e4eec760648a71e1cd8f8f551997b1823b4bb9f commit: 6e4eec760648a71e1cd8f8f551997b1823b4bb9f branch: 3.12 author: Eric Snow committer: Yhg1s date: 2023-08-05T13:44:54+02:00 summary: [3.12] gh-107630: Revert "[3.12] gh-107080: Fix Py_TRACE_REFS Crashes Under Isolated Subinterpreters (gh-107567) (#107599)" (#107648) Revert "[3.12] gh-107080: Fix Py_TRACE_REFS Crashes Under Isolated Subinterpreters (gh-107567) (#107599)" This reverts commit 58af2293c52a1ad3754d254690c0e54f787c545b. files: D Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst M Include/internal/pycore_object.h M Include/internal/pycore_object_state.h M Include/internal/pycore_runtime_init.h M Objects/object.c M Python/pylifecycle.c diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 5a9403456903e..0981d1122fec5 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -271,8 +271,8 @@ extern void _PyDebug_PrintTotalRefs(void); #ifdef Py_TRACE_REFS extern void _Py_AddToAllObjects(PyObject *op, int force); -extern void _Py_PrintReferences(PyInterpreterState *, FILE *); -extern void _Py_PrintReferenceAddresses(PyInterpreterState *, FILE *); +extern void _Py_PrintReferences(FILE *); +extern void _Py_PrintReferenceAddresses(FILE *); #endif diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index 65feb5af969f8..94005d7788143 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -11,22 +11,17 @@ extern "C" { struct _py_object_runtime_state { #ifdef Py_REF_DEBUG Py_ssize_t interpreter_leaks; -#endif +#else int _not_used; +#endif }; struct _py_object_state { #ifdef Py_REF_DEBUG Py_ssize_t reftotal; -#endif -#ifdef Py_TRACE_REFS - /* Head of circular doubly-linked list of all objects. These are linked - * together via the _ob_prev and _ob_next members of a PyObject, which - * exist only in a Py_TRACE_REFS build. - */ - PyObject refchain; -#endif +#else int _not_used; +#endif }; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 7aace9f86119c..4130188079cff 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -101,7 +101,6 @@ extern PyTypeObject _PyExc_MemoryError; { .threshold = 10, }, \ }, \ }, \ - .object_state = _py_object_state_INIT(INTERP), \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ .dict_state = _dict_state_INIT, \ .func_state = { \ @@ -131,16 +130,6 @@ extern PyTypeObject _PyExc_MemoryError; .context_ver = 1, \ } -#ifdef Py_TRACE_REFS -# define _py_object_state_INIT(INTERP) \ - { \ - .refchain = {&INTERP.object_state.refchain, &INTERP.object_state.refchain}, \ - } -#else -# define _py_object_state_INIT(INTERP) \ - { 0 } -#endif - // global objects diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst deleted file mode 100644 index 5084c854360e3..0000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst +++ /dev/null @@ -1,4 +0,0 @@ -Trace refs builds (``--with-trace-refs``) were crashing when used with -isolated subinterpreters. The problematic global state has been isolated to -each interpreter. Other fixing the crashes, this change does not affect -users. diff --git a/Objects/object.c b/Objects/object.c index b4c9416036c44..bd0fa40bd2a85 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -158,8 +158,11 @@ _PyDebug_PrintTotalRefs(void) { Do not call them otherwise, they do not initialize the object! */ #ifdef Py_TRACE_REFS - -#define REFCHAIN(interp) &interp->object_state.refchain +/* Head of circular doubly-linked list of all objects. These are linked + * together via the _ob_prev and _ob_next members of a PyObject, which + * exist only in a Py_TRACE_REFS build. + */ +static PyObject refchain = {&refchain, &refchain}; /* Insert op at the front of the list of all objects. If force is true, * op is added even if _ob_prev and _ob_next are non-NULL already. If @@ -184,11 +187,10 @@ _Py_AddToAllObjects(PyObject *op, int force) } #endif if (force || op->_ob_prev == NULL) { - PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); - op->_ob_next = refchain->_ob_next; - op->_ob_prev = refchain; - refchain->_ob_next->_ob_prev = op; - refchain->_ob_next = op; + op->_ob_next = refchain._ob_next; + op->_ob_prev = &refchain; + refchain._ob_next->_ob_prev = op; + refchain._ob_next = op; } } #endif /* Py_TRACE_REFS */ @@ -2204,8 +2206,7 @@ _Py_ForgetReference(PyObject *op) _PyObject_ASSERT_FAILED_MSG(op, "negative refcnt"); } - PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); - if (op == refchain || + if (op == &refchain || op->_ob_prev->_ob_next != op || op->_ob_next->_ob_prev != op) { _PyObject_ASSERT_FAILED_MSG(op, "invalid object chain"); @@ -2213,12 +2214,12 @@ _Py_ForgetReference(PyObject *op) #ifdef SLOW_UNREF_CHECK PyObject *p; - for (p = refchain->_ob_next; p != refchain; p = p->_ob_next) { + for (p = refchain._ob_next; p != &refchain; p = p->_ob_next) { if (p == op) { break; } } - if (p == refchain) { + if (p == &refchain) { /* Not found */ _PyObject_ASSERT_FAILED_MSG(op, "object not found in the objects list"); @@ -2234,15 +2235,11 @@ _Py_ForgetReference(PyObject *op) * interpreter must be in a healthy state. */ void -_Py_PrintReferences(PyInterpreterState *interp, FILE *fp) +_Py_PrintReferences(FILE *fp) { PyObject *op; - if (interp == NULL) { - interp = _PyInterpreterState_Main(); - } fprintf(fp, "Remaining objects:\n"); - PyObject *refchain = REFCHAIN(interp); - for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) { + for (op = refchain._ob_next; op != &refchain; op = op->_ob_next) { fprintf(fp, "%p [%zd] ", (void *)op, Py_REFCNT(op)); if (PyObject_Print(op, fp, 0) != 0) { PyErr_Clear(); @@ -2254,42 +2251,34 @@ _Py_PrintReferences(PyInterpreterState *interp, FILE *fp) /* Print the addresses of all live objects. Unlike _Py_PrintReferences, this * doesn't make any calls to the Python C API, so is always safe to call. */ -// XXX This function is not safe to use if the interpreter has been -// freed or is in an unhealthy state (e.g. late in finalization). -// The call in Py_FinalizeEx() is okay since the main interpreter -// is statically allocated. void -_Py_PrintReferenceAddresses(PyInterpreterState *interp, FILE *fp) +_Py_PrintReferenceAddresses(FILE *fp) { PyObject *op; - PyObject *refchain = REFCHAIN(interp); fprintf(fp, "Remaining object addresses:\n"); - for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) + for (op = refchain._ob_next; op != &refchain; op = op->_ob_next) fprintf(fp, "%p [%zd] %s\n", (void *)op, Py_REFCNT(op), Py_TYPE(op)->tp_name); } -/* The implementation of sys.getobjects(). */ PyObject * _Py_GetObjects(PyObject *self, PyObject *args) { int i, n; PyObject *t = NULL; PyObject *res, *op; - PyInterpreterState *interp = _PyInterpreterState_GET(); if (!PyArg_ParseTuple(args, "i|O", &n, &t)) return NULL; - PyObject *refchain = REFCHAIN(interp); - op = refchain->_ob_next; + op = refchain._ob_next; res = PyList_New(0); if (res == NULL) return NULL; - for (i = 0; (n == 0 || i < n) && op != refchain; i++) { + for (i = 0; (n == 0 || i < n) && op != &refchain; i++) { while (op == self || op == args || op == res || op == t || (t != NULL && !Py_IS_TYPE(op, (PyTypeObject *) t))) { op = op->_ob_next; - if (op == refchain) + if (op == &refchain) return res; } if (PyList_Append(res, op) < 0) { @@ -2301,8 +2290,6 @@ _Py_GetObjects(PyObject *self, PyObject *args) return res; } -#undef REFCHAIN - #endif diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 5b8c8af179c00..a67fa26b37227 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1920,11 +1920,11 @@ Py_FinalizeEx(void) } if (dump_refs) { - _Py_PrintReferences(tstate->interp, stderr); + _Py_PrintReferences(stderr); } if (dump_refs_fp != NULL) { - _Py_PrintReferences(tstate->interp, dump_refs_fp); + _Py_PrintReferences(dump_refs_fp); } #endif /* Py_TRACE_REFS */ @@ -1960,11 +1960,11 @@ Py_FinalizeEx(void) */ if (dump_refs) { - _Py_PrintReferenceAddresses(tstate->interp, stderr); + _Py_PrintReferenceAddresses(stderr); } if (dump_refs_fp != NULL) { - _Py_PrintReferenceAddresses(tstate->interp, dump_refs_fp); + _Py_PrintReferenceAddresses(dump_refs_fp); fclose(dump_refs_fp); } #endif /* Py_TRACE_REFS */ From webhook-mailer at python.org Sat Aug 5 08:08:08 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sat, 05 Aug 2023 12:08:08 -0000 Subject: [Python-checkins] [3.12] gh-107432 Fix incorrect indentation in annotations HOWTO (GH-107445) (#107654) Message-ID: https://github.com/python/cpython/commit/236cdadb08f1881bda96b48429ce8e882f8bcb9d commit: 236cdadb08f1881bda96b48429ce8e882f8bcb9d branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-05T14:08:04+02:00 summary: [3.12] gh-107432 Fix incorrect indentation in annotations HOWTO (GH-107445) (#107654) gh-107432 Fix incorrect indentation in annotations HOWTO (GH-107445) gh-107432 Fix incorrect indentation in annotations document Body text in https://docs.python.org/3/howto/annotations.html was indented throughout, and was being rendered in blockquote elements. (cherry picked from commit 5e2746d6e2fb0da29225ead7135f078c5f087b57) Co-authored-by: Daniele Procida files: M Doc/howto/annotations.rst diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 472069032d650..1134686c947d6 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -32,201 +32,201 @@ Annotations Best Practices Accessing The Annotations Dict Of An Object In Python 3.10 And Newer ==================================================================== - Python 3.10 adds a new function to the standard library: - :func:`inspect.get_annotations`. In Python versions 3.10 - and newer, calling this function is the best practice for - accessing the annotations dict of any object that supports - annotations. This function can also "un-stringize" - stringized annotations for you. - - If for some reason :func:`inspect.get_annotations` isn't - viable for your use case, you may access the - ``__annotations__`` data member manually. Best practice - for this changed in Python 3.10 as well: as of Python 3.10, - ``o.__annotations__`` is guaranteed to *always* work - on Python functions, classes, and modules. If you're - certain the object you're examining is one of these three - *specific* objects, you may simply use ``o.__annotations__`` - to get at the object's annotations dict. - - However, other types of callables--for example, - callables created by :func:`functools.partial`--may - not have an ``__annotations__`` attribute defined. When - accessing the ``__annotations__`` of a possibly unknown - object, best practice in Python versions 3.10 and - newer is to call :func:`getattr` with three arguments, - for example ``getattr(o, '__annotations__', None)``. - - Before Python 3.10, accessing ``__annotations__`` on a class that - defines no annotations but that has a parent class with - annotations would return the parent's ``__annotations__``. - In Python 3.10 and newer, the child class's annotations - will be an empty dict instead. +Python 3.10 adds a new function to the standard library: +:func:`inspect.get_annotations`. In Python versions 3.10 +and newer, calling this function is the best practice for +accessing the annotations dict of any object that supports +annotations. This function can also "un-stringize" +stringized annotations for you. + +If for some reason :func:`inspect.get_annotations` isn't +viable for your use case, you may access the +``__annotations__`` data member manually. Best practice +for this changed in Python 3.10 as well: as of Python 3.10, +``o.__annotations__`` is guaranteed to *always* work +on Python functions, classes, and modules. If you're +certain the object you're examining is one of these three +*specific* objects, you may simply use ``o.__annotations__`` +to get at the object's annotations dict. + +However, other types of callables--for example, +callables created by :func:`functools.partial`--may +not have an ``__annotations__`` attribute defined. When +accessing the ``__annotations__`` of a possibly unknown +object, best practice in Python versions 3.10 and +newer is to call :func:`getattr` with three arguments, +for example ``getattr(o, '__annotations__', None)``. + +Before Python 3.10, accessing ``__annotations__`` on a class that +defines no annotations but that has a parent class with +annotations would return the parent's ``__annotations__``. +In Python 3.10 and newer, the child class's annotations +will be an empty dict instead. Accessing The Annotations Dict Of An Object In Python 3.9 And Older =================================================================== - In Python 3.9 and older, accessing the annotations dict - of an object is much more complicated than in newer versions. - The problem is a design flaw in these older versions of Python, - specifically to do with class annotations. +In Python 3.9 and older, accessing the annotations dict +of an object is much more complicated than in newer versions. +The problem is a design flaw in these older versions of Python, +specifically to do with class annotations. - Best practice for accessing the annotations dict of other - objects--functions, other callables, and modules--is the same - as best practice for 3.10, assuming you aren't calling - :func:`inspect.get_annotations`: you should use three-argument - :func:`getattr` to access the object's ``__annotations__`` - attribute. +Best practice for accessing the annotations dict of other +objects--functions, other callables, and modules--is the same +as best practice for 3.10, assuming you aren't calling +:func:`inspect.get_annotations`: you should use three-argument +:func:`getattr` to access the object's ``__annotations__`` +attribute. - Unfortunately, this isn't best practice for classes. The problem - is that, since ``__annotations__`` is optional on classes, and - because classes can inherit attributes from their base classes, - accessing the ``__annotations__`` attribute of a class may - inadvertently return the annotations dict of a *base class.* - As an example:: +Unfortunately, this isn't best practice for classes. The problem +is that, since ``__annotations__`` is optional on classes, and +because classes can inherit attributes from their base classes, +accessing the ``__annotations__`` attribute of a class may +inadvertently return the annotations dict of a *base class.* +As an example:: - class Base: - a: int = 3 - b: str = 'abc' + class Base: + a: int = 3 + b: str = 'abc' - class Derived(Base): - pass + class Derived(Base): + pass - print(Derived.__annotations__) + print(Derived.__annotations__) - This will print the annotations dict from ``Base``, not - ``Derived``. +This will print the annotations dict from ``Base``, not +``Derived``. - Your code will have to have a separate code path if the object - you're examining is a class (``isinstance(o, type)``). - In that case, best practice relies on an implementation detail - of Python 3.9 and before: if a class has annotations defined, - they are stored in the class's ``__dict__`` dictionary. Since - the class may or may not have annotations defined, best practice - is to call the ``get`` method on the class dict. +Your code will have to have a separate code path if the object +you're examining is a class (``isinstance(o, type)``). +In that case, best practice relies on an implementation detail +of Python 3.9 and before: if a class has annotations defined, +they are stored in the class's ``__dict__`` dictionary. Since +the class may or may not have annotations defined, best practice +is to call the ``get`` method on the class dict. - To put it all together, here is some sample code that safely - accesses the ``__annotations__`` attribute on an arbitrary - object in Python 3.9 and before:: +To put it all together, here is some sample code that safely +accesses the ``__annotations__`` attribute on an arbitrary +object in Python 3.9 and before:: - if isinstance(o, type): - ann = o.__dict__.get('__annotations__', None) - else: - ann = getattr(o, '__annotations__', None) + if isinstance(o, type): + ann = o.__dict__.get('__annotations__', None) + else: + ann = getattr(o, '__annotations__', None) - After running this code, ``ann`` should be either a - dictionary or ``None``. You're encouraged to double-check - the type of ``ann`` using :func:`isinstance` before further - examination. +After running this code, ``ann`` should be either a +dictionary or ``None``. You're encouraged to double-check +the type of ``ann`` using :func:`isinstance` before further +examination. - Note that some exotic or malformed type objects may not have - a ``__dict__`` attribute, so for extra safety you may also wish - to use :func:`getattr` to access ``__dict__``. +Note that some exotic or malformed type objects may not have +a ``__dict__`` attribute, so for extra safety you may also wish +to use :func:`getattr` to access ``__dict__``. Manually Un-Stringizing Stringized Annotations ============================================== - In situations where some annotations may be "stringized", - and you wish to evaluate those strings to produce the - Python values they represent, it really is best to - call :func:`inspect.get_annotations` to do this work - for you. - - If you're using Python 3.9 or older, or if for some reason - you can't use :func:`inspect.get_annotations`, you'll need - to duplicate its logic. You're encouraged to examine the - implementation of :func:`inspect.get_annotations` in the - current Python version and follow a similar approach. - - In a nutshell, if you wish to evaluate a stringized annotation - on an arbitrary object ``o``: - - * If ``o`` is a module, use ``o.__dict__`` as the - ``globals`` when calling :func:`eval`. - * If ``o`` is a class, use ``sys.modules[o.__module__].__dict__`` - as the ``globals``, and ``dict(vars(o))`` as the ``locals``, - when calling :func:`eval`. - * If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, - :func:`functools.wraps`, or :func:`functools.partial`, iteratively - unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as - appropriate, until you have found the root unwrapped function. - * If ``o`` is a callable (but not a class), use - ``o.__globals__`` as the globals when calling :func:`eval`. - - However, not all string values used as annotations can - be successfully turned into Python values by :func:`eval`. - String values could theoretically contain any valid string, - and in practice there are valid use cases for type hints that - require annotating with string values that specifically - *can't* be evaluated. For example: - - * :pep:`604` union types using ``|``, before support for this - was added to Python 3.10. - * Definitions that aren't needed at runtime, only imported - when :const:`typing.TYPE_CHECKING` is true. - - If :func:`eval` attempts to evaluate such values, it will - fail and raise an exception. So, when designing a library - API that works with annotations, it's recommended to only - attempt to evaluate string values when explicitly requested - to by the caller. +In situations where some annotations may be "stringized", +and you wish to evaluate those strings to produce the +Python values they represent, it really is best to +call :func:`inspect.get_annotations` to do this work +for you. + +If you're using Python 3.9 or older, or if for some reason +you can't use :func:`inspect.get_annotations`, you'll need +to duplicate its logic. You're encouraged to examine the +implementation of :func:`inspect.get_annotations` in the +current Python version and follow a similar approach. + +In a nutshell, if you wish to evaluate a stringized annotation +on an arbitrary object ``o``: + +* If ``o`` is a module, use ``o.__dict__`` as the + ``globals`` when calling :func:`eval`. +* If ``o`` is a class, use ``sys.modules[o.__module__].__dict__`` + as the ``globals``, and ``dict(vars(o))`` as the ``locals``, + when calling :func:`eval`. +* If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, + :func:`functools.wraps`, or :func:`functools.partial`, iteratively + unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as + appropriate, until you have found the root unwrapped function. +* If ``o`` is a callable (but not a class), use + ``o.__globals__`` as the globals when calling :func:`eval`. + +However, not all string values used as annotations can +be successfully turned into Python values by :func:`eval`. +String values could theoretically contain any valid string, +and in practice there are valid use cases for type hints that +require annotating with string values that specifically +*can't* be evaluated. For example: + +* :pep:`604` union types using ``|``, before support for this + was added to Python 3.10. +* Definitions that aren't needed at runtime, only imported + when :const:`typing.TYPE_CHECKING` is true. + +If :func:`eval` attempts to evaluate such values, it will +fail and raise an exception. So, when designing a library +API that works with annotations, it's recommended to only +attempt to evaluate string values when explicitly requested +to by the caller. Best Practices For ``__annotations__`` In Any Python Version ============================================================ - * You should avoid assigning to the ``__annotations__`` member - of objects directly. Let Python manage setting ``__annotations__``. +* You should avoid assigning to the ``__annotations__`` member + of objects directly. Let Python manage setting ``__annotations__``. - * If you do assign directly to the ``__annotations__`` member - of an object, you should always set it to a ``dict`` object. +* If you do assign directly to the ``__annotations__`` member + of an object, you should always set it to a ``dict`` object. - * If you directly access the ``__annotations__`` member - of an object, you should ensure that it's a - dictionary before attempting to examine its contents. +* If you directly access the ``__annotations__`` member + of an object, you should ensure that it's a + dictionary before attempting to examine its contents. - * You should avoid modifying ``__annotations__`` dicts. +* You should avoid modifying ``__annotations__`` dicts. - * You should avoid deleting the ``__annotations__`` attribute - of an object. +* You should avoid deleting the ``__annotations__`` attribute + of an object. ``__annotations__`` Quirks ========================== - In all versions of Python 3, function - objects lazy-create an annotations dict if no annotations - are defined on that object. You can delete the ``__annotations__`` - attribute using ``del fn.__annotations__``, but if you then - access ``fn.__annotations__`` the object will create a new empty dict - that it will store and return as its annotations. Deleting the - annotations on a function before it has lazily created its annotations - dict will throw an ``AttributeError``; using ``del fn.__annotations__`` - twice in a row is guaranteed to always throw an ``AttributeError``. - - Everything in the above paragraph also applies to class and module - objects in Python 3.10 and newer. - - In all versions of Python 3, you can set ``__annotations__`` - on a function object to ``None``. However, subsequently - accessing the annotations on that object using ``fn.__annotations__`` - will lazy-create an empty dictionary as per the first paragraph of - this section. This is *not* true of modules and classes, in any Python - version; those objects permit setting ``__annotations__`` to any - Python value, and will retain whatever value is set. - - If Python stringizes your annotations for you - (using ``from __future__ import annotations``), and you - specify a string as an annotation, the string will - itself be quoted. In effect the annotation is quoted - *twice.* For example:: - - from __future__ import annotations - def foo(a: "str"): pass - - print(foo.__annotations__) - - This prints ``{'a': "'str'"}``. This shouldn't really be considered - a "quirk"; it's mentioned here simply because it might be surprising. +In all versions of Python 3, function +objects lazy-create an annotations dict if no annotations +are defined on that object. You can delete the ``__annotations__`` +attribute using ``del fn.__annotations__``, but if you then +access ``fn.__annotations__`` the object will create a new empty dict +that it will store and return as its annotations. Deleting the +annotations on a function before it has lazily created its annotations +dict will throw an ``AttributeError``; using ``del fn.__annotations__`` +twice in a row is guaranteed to always throw an ``AttributeError``. + +Everything in the above paragraph also applies to class and module +objects in Python 3.10 and newer. + +In all versions of Python 3, you can set ``__annotations__`` +on a function object to ``None``. However, subsequently +accessing the annotations on that object using ``fn.__annotations__`` +will lazy-create an empty dictionary as per the first paragraph of +this section. This is *not* true of modules and classes, in any Python +version; those objects permit setting ``__annotations__`` to any +Python value, and will retain whatever value is set. + +If Python stringizes your annotations for you +(using ``from __future__ import annotations``), and you +specify a string as an annotation, the string will +itself be quoted. In effect the annotation is quoted +*twice.* For example:: + + from __future__ import annotations + def foo(a: "str"): pass + + print(foo.__annotations__) + +This prints ``{'a': "'str'"}``. This shouldn't really be considered +a "quirk"; it's mentioned here simply because it might be surprising. From webhook-mailer at python.org Sat Aug 5 08:18:19 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Sat, 05 Aug 2023 12:18:19 -0000 Subject: [Python-checkins] GH-106684: raise `ResourceWarning` when `asyncio.StreamWriter` is not closed (#107650) Message-ID: https://github.com/python/cpython/commit/41178e41995992bbe417f94bce158de93f9e3188 commit: 41178e41995992bbe417f94bce158de93f9e3188 branch: main author: Kumar Aditya committer: kumaraditya303 date: 2023-08-05T17:48:15+05:30 summary: GH-106684: raise `ResourceWarning` when `asyncio.StreamWriter` is not closed (#107650) files: A Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst M Lib/asyncio/streams.py M Lib/test/test_asyncio/test_streams.py diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index bf15f517e50dc..b7ad365709b19 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -5,6 +5,7 @@ import collections import socket import sys +import warnings import weakref if hasattr(socket, 'AF_UNIX'): @@ -392,6 +393,11 @@ async def start_tls(self, sslcontext, *, self._transport = new_transport protocol._replace_writer(self) + def __del__(self, warnings=warnings): + if not self._transport.is_closing(): + self.close() + warnings.warn(f"unclosed {self!r}", ResourceWarning) + class StreamReader: diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 7f9dc62180835..9c92e75886c59 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -1074,6 +1074,29 @@ def test_eof_feed_when_closing_writer(self): self.assertEqual(messages, []) + def test_unclosed_resource_warnings(self): + async def inner(httpd): + rd, wr = await asyncio.open_connection(*httpd.address) + + wr.write(b'GET / HTTP/1.0\r\n\r\n') + data = await rd.readline() + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + data = await rd.read() + self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + with self.assertWarns(ResourceWarning): + del wr + gc.collect() + + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + with test_utils.run_test_server() as httpd: + self.loop.run_until_complete(inner(httpd)) + + self.assertEqual(messages, []) + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst b/Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst new file mode 100644 index 0000000000000..02c52d714e9df --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst @@ -0,0 +1 @@ +Raise :exc:`ResourceWarning` when :class:`asyncio.StreamWriter` is not closed leading to memory leaks. Patch by Kumar Aditya. From webhook-mailer at python.org Sat Aug 5 10:38:23 2023 From: webhook-mailer at python.org (terryjreedy) Date: Sat, 05 Aug 2023 14:38:23 -0000 Subject: [Python-checkins] gh-107662: Switch 'any' and 'anext' in functions.rst (#107663) Message-ID: https://github.com/python/cpython/commit/9ebc6ecbc336d7b17cd158d1a4522f832df3e6e2 commit: 9ebc6ecbc336d7b17cd158d1a4522f832df3e6e2 branch: main author: Terry Jan Reedy committer: terryjreedy date: 2023-08-05T14:38:20Z summary: gh-107662: Switch 'any' and 'anext' in functions.rst (#107663) Order was reversed in index at top, not in body. files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 2688f43f9b4ff..b271067ae639c 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -14,8 +14,8 @@ are always available. They are listed here in alphabetical order. | | :func:`abs` | | :func:`enumerate` | | :func:`len` | | |func-range|_ | | | :func:`aiter` | | :func:`eval` | | |func-list|_ | | :func:`repr` | | | :func:`all` | | :func:`exec` | | :func:`locals` | | :func:`reversed` | -| | :func:`any` | | | | | | :func:`round` | -| | :func:`anext` | | **F** | | **M** | | | +| | :func:`anext` | | | | | | :func:`round` | +| | :func:`any` | | **F** | | **M** | | | | | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | | | | | :func:`float` | | :func:`max` | | |func-set|_ | | | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | From webhook-mailer at python.org Sat Aug 5 13:25:25 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Sat, 05 Aug 2023 17:25:25 -0000 Subject: [Python-checkins] [3.11] gh-107662: Switch 'any' and 'anext' in functions.rst (GH-107663) (#107665) Message-ID: https://github.com/python/cpython/commit/b89feac7592c1dc2f8632f12fb1ff82c5602819f commit: b89feac7592c1dc2f8632f12fb1ff82c5602819f branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: AlexWaygood date: 2023-08-05T18:25:21+01:00 summary: [3.11] gh-107662: Switch 'any' and 'anext' in functions.rst (GH-107663) (#107665) gh-107662: Switch 'any' and 'anext' in functions.rst (GH-107663) Order was reversed in index at top, not in body. (cherry picked from commit 9ebc6ecbc336d7b17cd158d1a4522f832df3e6e2) Co-authored-by: Terry Jan Reedy files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index e00d079ce4387..2468e53cd596a 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -14,8 +14,8 @@ are always available. They are listed here in alphabetical order. | | :func:`abs` | | :func:`enumerate` | | :func:`len` | | |func-range|_ | | | :func:`aiter` | | :func:`eval` | | |func-list|_ | | :func:`repr` | | | :func:`all` | | :func:`exec` | | :func:`locals` | | :func:`reversed` | -| | :func:`any` | | | | | | :func:`round` | -| | :func:`anext` | | **F** | | **M** | | | +| | :func:`anext` | | | | | | :func:`round` | +| | :func:`any` | | **F** | | **M** | | | | | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | | | | | :func:`float` | | :func:`max` | | |func-set|_ | | | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | From webhook-mailer at python.org Sat Aug 5 16:19:41 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Sat, 05 Aug 2023 20:19:41 -0000 Subject: [Python-checkins] Docs: Argument Clinic: Improve 'How to write a custom converter' (#107328) Message-ID: https://github.com/python/cpython/commit/4a5b4221e381c541f3f73537b7b87580d100158b commit: 4a5b4221e381c541f3f73537b7b87580d100158b branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-05T20:19:37Z summary: Docs: Argument Clinic: Improve 'How to write a custom converter' (#107328) - Omit unneccesary wording and sentences - Don't mention implementation details (no digression, explanation) Co-authored-by: Ezio Melotti files: M Doc/howto/clinic.rst diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 9c9a4f45dd0f5..dcede13a03b4a 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1343,35 +1343,37 @@ state. Example from the ``setattro`` slot method in See also :pep:`573`. +.. _clinic-howto-custom-converter: + How to write a custom converter ------------------------------- -As we hinted at in the previous section... you can write your own converters! -A converter is simply a Python class that inherits from :py:class:`!CConverter`. -The main purpose of a custom converter is if you have a parameter using -the ``O&`` format unit?parsing this parameter means calling +A converter is a Python class that inherits from :py:class:`!CConverter`. +The main purpose of a custom converter, is for parameters parsed with +the ``O&`` format unit --- parsing such a parameter means calling a :c:func:`PyArg_ParseTuple` "converter function". -Your converter class should be named ``*something*_converter``. -If the name follows this convention, then your converter class -will be automatically registered with Argument Clinic; its name -will be the name of your class with the ``_converter`` suffix -stripped off. (This is accomplished with a metaclass.) - -You shouldn't subclass :py:meth:`!CConverter.__init__`. Instead, you should -write a :py:meth:`!converter_init` function. :py:meth:`!converter_init` -always accepts a *self* parameter; after that, all additional -parameters *must* be keyword-only. Any arguments passed in to -the converter in Argument Clinic will be passed along to your -:py:meth:`!converter_init`. +Your converter class should be named :samp:`{ConverterName}_converter`. +By following this convention, your converter class will be automatically +registered with Argument Clinic, with its *converter name* being the name of +your converter class with the ``_converter`` suffix stripped off. -There are some additional members of :py:class:`!CConverter` you may wish -to specify in your subclass. Here's the current list: +Instead of subclassing :py:meth:`!CConverter.__init__`, +write a :py:meth:`!converter_init` method. +Apart for the *self* parameter, all additional :py:meth:`!converter_init` +parameters **must** be keyword-only. +Any arguments passed to the converter in Argument Clinic +will be passed along to your :py:meth:`!converter_init` method. +See :py:class:`!CConverter` for a list of members you may wish to specify in +your subclass. .. module:: clinic .. class:: CConverter + The base class for all converters. + See :ref:`clinic-howto-custom-converter` for how to subclass this class. + .. attribute:: type The C type to use for this variable. @@ -1436,16 +1438,16 @@ Here's the simplest example of a custom converter, from :source:`Modules/zlibmod [python start generated code]*/ /*[python end generated code: output=da39a3ee5e6b4b0d input=35521e4e733823c7]*/ -This block adds a converter to Argument Clinic named ``ssize_t``. Parameters -declared as ``ssize_t`` will be declared as type :c:type:`Py_ssize_t`, and will -be parsed by the ``'O&'`` format unit, which will call the -``ssize_t_converter`` converter function. ``ssize_t`` variables -automatically support default values. +This block adds a converter named ``ssize_t`` to Argument Clinic. +Parameters declared as ``ssize_t`` will be declared with type :c:type:`Py_ssize_t`, +and will be parsed by the ``'O&'`` format unit, +which will call the :c:func:`!ssize_t_converter` converter C function. +``ssize_t`` variables automatically support default values. More sophisticated custom converters can insert custom C code to handle initialization and cleanup. You can see more examples of custom converters in the CPython -source tree; grep the C files for the string :py:class:`!CConverter`. +source tree; grep the C files for the string ``CConverter``. How to write a custom return converter From webhook-mailer at python.org Sat Aug 5 16:29:35 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Sat, 05 Aug 2023 20:29:35 -0000 Subject: [Python-checkins] [3.11] Docs: Argument Clinic: Improve 'How to write a custom converter' (GH-107328) (#107670) Message-ID: https://github.com/python/cpython/commit/e4b5ec71fe44e4a4a393acf98075ab76be2bbb6a commit: e4b5ec71fe44e4a4a393acf98075ab76be2bbb6a branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: erlend-aasland date: 2023-08-05T20:29:31Z summary: [3.11] Docs: Argument Clinic: Improve 'How to write a custom converter' (GH-107328) (#107670) - Omit unneccesary wording and sentences - Don't mention implementation details (no digression, explanation) (cherry picked from commit 4a5b4221e381c541f3f73537b7b87580d100158b) Co-authored-by: Erlend E. Aasland Co-authored-by: Ezio Melotti files: M Doc/howto/clinic.rst diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 913a3df7e3c48..48c8f3c4e01c0 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1338,35 +1338,37 @@ state. Example from the ``setattro`` slot method in See also :pep:`573`. +.. _clinic-howto-custom-converter: + How to write a custom converter ------------------------------- -As we hinted at in the previous section... you can write your own converters! -A converter is simply a Python class that inherits from :py:class:`!CConverter`. -The main purpose of a custom converter is if you have a parameter using -the ``O&`` format unit?parsing this parameter means calling +A converter is a Python class that inherits from :py:class:`!CConverter`. +The main purpose of a custom converter, is for parameters parsed with +the ``O&`` format unit --- parsing such a parameter means calling a :c:func:`PyArg_ParseTuple` "converter function". -Your converter class should be named ``*something*_converter``. -If the name follows this convention, then your converter class -will be automatically registered with Argument Clinic; its name -will be the name of your class with the ``_converter`` suffix -stripped off. (This is accomplished with a metaclass.) - -You shouldn't subclass :py:meth:`!CConverter.__init__`. Instead, you should -write a :py:meth:`!converter_init` function. :py:meth:`!converter_init` -always accepts a *self* parameter; after that, all additional -parameters *must* be keyword-only. Any arguments passed in to -the converter in Argument Clinic will be passed along to your -:py:meth:`!converter_init`. +Your converter class should be named :samp:`{ConverterName}_converter`. +By following this convention, your converter class will be automatically +registered with Argument Clinic, with its *converter name* being the name of +your converter class with the ``_converter`` suffix stripped off. -There are some additional members of :py:class:`!CConverter` you may wish -to specify in your subclass. Here's the current list: +Instead of subclassing :py:meth:`!CConverter.__init__`, +write a :py:meth:`!converter_init` method. +Apart for the *self* parameter, all additional :py:meth:`!converter_init` +parameters **must** be keyword-only. +Any arguments passed to the converter in Argument Clinic +will be passed along to your :py:meth:`!converter_init` method. +See :py:class:`!CConverter` for a list of members you may wish to specify in +your subclass. .. module:: clinic .. class:: CConverter + The base class for all converters. + See :ref:`clinic-howto-custom-converter` for how to subclass this class. + .. attribute:: type The C type to use for this variable. @@ -1431,16 +1433,16 @@ Here's the simplest example of a custom converter, from :source:`Modules/zlibmod [python start generated code]*/ /*[python end generated code: output=da39a3ee5e6b4b0d input=35521e4e733823c7]*/ -This block adds a converter to Argument Clinic named ``ssize_t``. Parameters -declared as ``ssize_t`` will be declared as type :c:type:`Py_ssize_t`, and will -be parsed by the ``'O&'`` format unit, which will call the -``ssize_t_converter`` converter function. ``ssize_t`` variables -automatically support default values. +This block adds a converter named ``ssize_t`` to Argument Clinic. +Parameters declared as ``ssize_t`` will be declared with type :c:type:`Py_ssize_t`, +and will be parsed by the ``'O&'`` format unit, +which will call the :c:func:`!ssize_t_converter` converter C function. +``ssize_t`` variables automatically support default values. More sophisticated custom converters can insert custom C code to handle initialization and cleanup. You can see more examples of custom converters in the CPython -source tree; grep the C files for the string :py:class:`!CConverter`. +source tree; grep the C files for the string ``CConverter``. How to write a custom return converter From webhook-mailer at python.org Sat Aug 5 16:58:42 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Sat, 05 Aug 2023 20:58:42 -0000 Subject: [Python-checkins] gh-104683: Improve consistency and test coverage of argument-clinic `__repr__` functions (#107667) Message-ID: https://github.com/python/cpython/commit/6996b406bcf9f6d85a59e539c743ef9126c3cc5d commit: 6996b406bcf9f6d85a59e539c743ef9126c3cc5d branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-05T21:58:38+01:00 summary: gh-104683: Improve consistency and test coverage of argument-clinic `__repr__` functions (#107667) files: M Lib/test/test_clinic.py M Tools/clinic/clinic.py M Tools/clinic/cpp.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 84b6a193ecf91..f30fad2126940 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -69,7 +69,7 @@ def __init__(self): self.converters = FakeConvertersDict() self.legacy_converters = FakeConvertersDict() self.language = clinic.CLanguage(None) - self.filename = None + self.filename = "clinic_tests" self.destination_buffers = {} self.block_parser = clinic.BlockParser('', self.language) self.modules = collections.OrderedDict() @@ -1849,10 +1849,10 @@ def test_non_ascii_character_in_docstring(self): self.parse(block) # The line numbers are off; this is a known limitation. expected = dedent("""\ - Warning on line 0: + Warning in file 'clinic_tests' on line 0: Non-ascii characters are not allowed in docstrings: '?' - Warning on line 0: + Warning in file 'clinic_tests' on line 0: Non-ascii characters are not allowed in docstrings: '?', '?', '?' """) @@ -3030,5 +3030,93 @@ def test_suffix_all_lines(self): self.assertEqual(out, expected) +class ClinicReprTests(unittest.TestCase): + def test_Block_repr(self): + block = clinic.Block("foo") + expected_repr = "" + self.assertEqual(repr(block), expected_repr) + + block2 = clinic.Block("bar", "baz", [], "eggs", "spam") + expected_repr_2 = "" + self.assertEqual(repr(block2), expected_repr_2) + + block3 = clinic.Block( + input="longboi_" * 100, + dsl_name="wow_so_long", + signatures=[], + output="very_long_" * 100, + indent="" + ) + expected_repr_3 = ( + "" + ) + self.assertEqual(repr(block3), expected_repr_3) + + def test_Destination_repr(self): + destination = clinic.Destination( + "foo", type="file", clinic=FakeClinic(), args=("eggs",) + ) + self.assertEqual( + repr(destination), "" + ) + + destination2 = clinic.Destination("bar", type="buffer", clinic=FakeClinic()) + self.assertEqual(repr(destination2), "") + + def test_Module_repr(self): + module = clinic.Module("foo", FakeClinic()) + self.assertRegex(repr(module), r"") + + def test_Class_repr(self): + cls = clinic.Class("foo", FakeClinic(), None, 'some_typedef', 'some_type_object') + self.assertRegex(repr(cls), r"") + + def test_FunctionKind_repr(self): + self.assertEqual( + repr(clinic.FunctionKind.INVALID), "" + ) + self.assertEqual( + repr(clinic.FunctionKind.CLASS_METHOD), "" + ) + + def test_Function_and_Parameter_reprs(self): + function = clinic.Function( + name='foo', + module=FakeClinic(), + cls=None, + c_basename=None, + full_name='foofoo', + return_converter=clinic.init_return_converter(), + kind=clinic.FunctionKind.METHOD_INIT, + coexist=False + ) + self.assertEqual(repr(function), "") + + converter = clinic.self_converter('bar', 'bar', function) + parameter = clinic.Parameter( + "bar", + kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, + function=function, + converter=converter + ) + self.assertEqual(repr(parameter), "") + + def test_Monitor_repr(self): + monitor = clinic.cpp.Monitor() + self.assertRegex(repr(monitor), r"") + + monitor.line_number = 42 + monitor.stack.append(("token1", "condition1")) + self.assertRegex( + repr(monitor), r"" + ) + + monitor.stack.append(("token2", "condition2")) + self.assertRegex( + repr(monitor), + r"" + ) + + if __name__ == "__main__": unittest.main() diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 7fbae1e0d870a..423cdfb939c16 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1728,8 +1728,12 @@ def summarize(s: object) -> str: if len(s) > 30: return s[:26] + "..." + s[0] return s - return "".join(( - "")) + parts = ( + repr(dsl_name), + f"input={summarize(self.input)}", + f"output={summarize(self.output)}" + ) + return f"" class BlockParser: @@ -2037,10 +2041,10 @@ def __post_init__(self, args: tuple[str, ...]) -> None: def __repr__(self) -> str: if self.type == 'file': - file_repr = " " + repr(self.filename) + type_repr = f"type='file' file={self.filename!r}" else: - file_repr = '' - return "".join(("")) + type_repr = f"type={self.type!r}" + return f"" def clear(self) -> None: if self.type != 'buffer': @@ -2500,7 +2504,7 @@ def new_or_init(self) -> bool: return self in {FunctionKind.METHOD_INIT, FunctionKind.METHOD_NEW} def __repr__(self) -> str: - return f"" + return f"" INVALID: Final = FunctionKind.INVALID @@ -2577,7 +2581,7 @@ def methoddef_flags(self) -> str | None: return '|'.join(flags) def __repr__(self) -> str: - return '' + return f'' def copy(self, **overrides: Any) -> Function: f = dc.replace(self, **overrides) @@ -2605,7 +2609,7 @@ class Parameter: right_bracket_count: int = dc.field(init=False, default=0) def __repr__(self) -> str: - return '' + return f'' def is_keyword_only(self) -> bool: return self.kind == inspect.Parameter.KEYWORD_ONLY diff --git a/Tools/clinic/cpp.py b/Tools/clinic/cpp.py index 21a1b02e862c1..876105120c97f 100644 --- a/Tools/clinic/cpp.py +++ b/Tools/clinic/cpp.py @@ -43,9 +43,12 @@ def __post_init__(self) -> None: self.line_number = 0 def __repr__(self) -> str: - return ( - f"" + parts = ( + str(id(self)), + f"line={self.line_number}", + f"condition={self.condition()!r}" ) + return f"" def status(self) -> str: return str(self.line_number).rjust(4) + ": " + self.condition() From webhook-mailer at python.org Sun Aug 6 01:10:54 2023 From: webhook-mailer at python.org (hugovk) Date: Sun, 06 Aug 2023 05:10:54 -0000 Subject: [Python-checkins] GH-84435: Make pyspecific directives translatable (#19470) Message-ID: https://github.com/python/cpython/commit/ecb05e0b9842ba03b42b4dec8767b1c18a4e28b3 commit: ecb05e0b9842ba03b42b4dec8767b1c18a4e28b3 branch: main author: cocoatomo committer: hugovk date: 2023-08-05T23:10:50-06:00 summary: GH-84435: Make pyspecific directives translatable (#19470) Co-authored-by: Jelle Zijlstra Co-authored-by: Adam Turner <9087854+aa-turner at users.noreply.github.com> files: M Doc/tools/extensions/pyspecific.py diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 8d99b0bfa4f38..765e6383ac6f5 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -98,14 +98,13 @@ class ImplementationDetail(Directive): final_argument_whitespace = True # This text is copied to templates/dummy.html - label_text = 'CPython implementation detail:' + label_text = sphinx_gettext('CPython implementation detail:') def run(self): self.assert_has_content() pnode = nodes.compound(classes=['impl-detail']) - label = sphinx_gettext(self.label_text) content = self.content - add_text = nodes.strong(label, label) + add_text = nodes.strong(self.label_text, self.label_text) self.state.nested_parse(content, self.content_offset, pnode) content = nodes.inline(pnode[0].rawsource, translatable=True) content.source = pnode[0].source @@ -234,9 +233,9 @@ class AuditEvent(Directive): final_argument_whitespace = True _label = [ - "Raises an :ref:`auditing event ` {name} with no arguments.", - "Raises an :ref:`auditing event ` {name} with argument {args}.", - "Raises an :ref:`auditing event ` {name} with arguments {args}.", + sphinx_gettext("Raises an :ref:`auditing event ` {name} with no arguments."), + sphinx_gettext("Raises an :ref:`auditing event ` {name} with argument {args}."), + sphinx_gettext("Raises an :ref:`auditing event ` {name} with arguments {args}."), ] @property @@ -252,7 +251,7 @@ def run(self): else: args = [] - label = sphinx_gettext(self._label[min(2, len(args))]) + label = self._label[min(2, len(args))] text = label.format(name="``{}``".format(name), args=", ".join("``{}``".format(a) for a in args if a)) @@ -414,8 +413,8 @@ class DeprecatedRemoved(Directive): final_argument_whitespace = True option_spec = {} - _deprecated_label = 'Deprecated since version {deprecated}, will be removed in version {removed}' - _removed_label = 'Deprecated since version {deprecated}, removed in version {removed}' + _deprecated_label = sphinx_gettext('Deprecated since version {deprecated}, will be removed in version {removed}') + _removed_label = sphinx_gettext('Deprecated since version {deprecated}, removed in version {removed}') def run(self): node = addnodes.versionmodified() @@ -431,7 +430,6 @@ def run(self): else: label = self._removed_label - label = sphinx_gettext(label) text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) if len(self.arguments) == 3: inodes, messages = self.state.inline_text(self.arguments[2], From webhook-mailer at python.org Sun Aug 6 04:23:54 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Sun, 06 Aug 2023 08:23:54 -0000 Subject: [Python-checkins] Docs: Fix Sphinx annotations in Doc/library/ctypes.rst (#107672) Message-ID: https://github.com/python/cpython/commit/71a7c96ffeb0d7fef06be3e57468896e030967a5 commit: 71a7c96ffeb0d7fef06be3e57468896e030967a5 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-06T10:23:50+02:00 summary: Docs: Fix Sphinx annotations in Doc/library/ctypes.rst (#107672) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/conf.py M Doc/library/ctypes.rst diff --git a/Doc/conf.py b/Doc/conf.py index 1b30b862f9a86..49108eac58fc1 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -110,6 +110,8 @@ ('c:type', 'uintptr_t'), ('c:type', 'va_list'), ('c:type', 'wchar_t'), + ('c:type', '__int64'), + ('c:type', 'unsigned __int64'), # Standard C structures ('c:struct', 'in6_addr'), ('c:struct', 'in_addr'), diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index ea7873a885172..ec4b0909181d3 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -72,8 +72,9 @@ Windows appends the usual ``.dll`` file suffix automatically. On Linux, it is required to specify the filename *including* the extension to load a library, so attribute access can not be used to load libraries. Either the -:meth:`LoadLibrary` method of the dll loaders should be used, or you should load -the library by creating an instance of CDLL by calling the constructor:: +:meth:`~LibraryLoader.LoadLibrary` method of the dll loaders should be used, +or you should load the library by creating an instance of CDLL by calling +the constructor:: >>> cdll.LoadLibrary("libc.so.6") # doctest: +LINUX @@ -333,7 +334,7 @@ property:: 10 b'Hi\x00lo\x00\x00\x00\x00\x00' >>> -The :func:`create_string_buffer` function replaces the old :func:`c_buffer` +The :func:`create_string_buffer` function replaces the old :func:`!c_buffer` function (which is still available as an alias). To create a mutable memory block containing unicode characters of the C type :c:type:`wchar_t`, use the :func:`create_unicode_buffer` function. @@ -383,15 +384,15 @@ as calling functions with a fixed number of parameters. On some platforms, and i particular ARM64 for Apple Platforms, the calling convention for variadic functions is different than that for regular functions. -On those platforms it is required to specify the *argtypes* attribute for the -regular, non-variadic, function arguments: +On those platforms it is required to specify the :attr:`~_FuncPtr.argtypes` +attribute for the regular, non-variadic, function arguments: .. code-block:: python3 libc.printf.argtypes = [ctypes.c_char_p] Because specifying the attribute does not inhibit portability it is advised to always -specify ``argtypes`` for all variadic functions. +specify :attr:`~_FuncPtr.argtypes` for all variadic functions. .. _ctypes-calling-functions-with-own-custom-data-types: @@ -401,7 +402,7 @@ Calling functions with your own custom data types You can also customize :mod:`ctypes` argument conversion to allow instances of your own classes be used as function arguments. :mod:`ctypes` looks for an -:attr:`_as_parameter_` attribute and uses this as the function argument. Of +:attr:`!_as_parameter_` attribute and uses this as the function argument. Of course, it must be one of integer, string, or bytes:: >>> class Bottles: @@ -414,7 +415,7 @@ course, it must be one of integer, string, or bytes:: 19 >>> -If you don't want to store the instance's data in the :attr:`_as_parameter_` +If you don't want to store the instance's data in the :attr:`!_as_parameter_` instance variable, you could define a :class:`property` which makes the attribute available on request. @@ -425,9 +426,9 @@ Specifying the required argument types (function prototypes) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is possible to specify the required argument types of functions exported from -DLLs by setting the :attr:`argtypes` attribute. +DLLs by setting the :attr:`~_FuncPtr.argtypes` attribute. -:attr:`argtypes` must be a sequence of C data types (the ``printf`` function is +:attr:`~_FuncPtr.argtypes` must be a sequence of C data types (the :func:`!printf` function is probably not a good example here, because it takes a variable number and different types of parameters depending on the format string, on the other hand this is quite handy to experiment with this feature):: @@ -451,14 +452,14 @@ prototype for a C function), and tries to convert the arguments to valid types:: >>> If you have defined your own classes which you pass to function calls, you have -to implement a :meth:`from_param` class method for them to be able to use them -in the :attr:`argtypes` sequence. The :meth:`from_param` class method receives +to implement a :meth:`~_CData.from_param` class method for them to be able to use them +in the :attr:`~_FuncPtr.argtypes` sequence. The :meth:`~_CData.from_param` class method receives the Python object passed to the function call, it should do a typecheck or whatever is needed to make sure this object is acceptable, and then return the -object itself, its :attr:`_as_parameter_` attribute, or whatever you want to +object itself, its :attr:`!_as_parameter_` attribute, or whatever you want to pass as the C function argument in this case. Again, the result should be an integer, string, bytes, a :mod:`ctypes` instance, or an object with an -:attr:`_as_parameter_` attribute. +:attr:`!_as_parameter_` attribute. .. _ctypes-return-types: @@ -478,13 +479,13 @@ By default functions are assumed to return the C :c:expr:`int` type. Other return types can be specified by setting the :attr:`restype` attribute of the function object. -The C prototype of ``time()`` is ``time_t time(time_t *)``. Because :c:type:`time_t` -might be of a different type than the default return type ``int``, you should -specify the ``restype``:: +The C prototype of :c:func:`time` is ``time_t time(time_t *)``. Because :c:type:`time_t` +might be of a different type than the default return type :c:expr:`int`, you should +specify the :attr:`!restype` attribute:: >>> libc.time.restype = c_time_t -The argument types can be specified using ``argtypes``:: +The argument types can be specified using :attr:`~_FuncPtr.argtypes`:: >>> libc.time.argtypes = (POINTER(c_time_t),) @@ -493,7 +494,7 @@ To call the function with a ``NULL`` pointer as first argument, use ``None``:: >>> print(libc.time(None)) # doctest: +SKIP 1150640792 -Here is a more advanced example, it uses the ``strchr`` function, which expects +Here is a more advanced example, it uses the :func:`strchr` function, which expects a string pointer and a char, and returns a pointer to a string:: >>> strchr = libc.strchr @@ -506,8 +507,8 @@ a string pointer and a char, and returns a pointer to a string:: None >>> -If you want to avoid the ``ord("x")`` calls above, you can set the -:attr:`argtypes` attribute, and the second argument will be converted from a +If you want to avoid the :func:`ord("x") ` calls above, you can set the +:attr:`~_FuncPtr.argtypes` attribute, and the second argument will be converted from a single character Python bytes object into a C char: .. doctest:: @@ -853,7 +854,7 @@ Type conversions ^^^^^^^^^^^^^^^^ Usually, ctypes does strict type checking. This means, if you have -``POINTER(c_int)`` in the :attr:`argtypes` list of a function or as the type of +``POINTER(c_int)`` in the :attr:`~_FuncPtr.argtypes` list of a function or as the type of a member field in a structure definition, only instances of exactly the same type are accepted. There are some exceptions to this rule, where ctypes accepts other objects. For example, you can pass compatible array instances instead of @@ -874,7 +875,7 @@ pointer types. So, for ``POINTER(c_int)``, ctypes accepts an array of c_int:: >>> In addition, if a function argument is explicitly declared to be a pointer type -(such as ``POINTER(c_int)``) in :attr:`argtypes`, an object of the pointed +(such as ``POINTER(c_int)``) in :attr:`_FuncPtr.argtypes`, an object of the pointed type (``c_int`` in this case) can be passed to the function. ctypes will apply the required :func:`byref` conversion in this case automatically. @@ -1437,7 +1438,7 @@ function exported by these libraries, and reacquired afterwards. All these classes can be instantiated by calling them with at least one argument, the pathname of the shared library. If you have an existing handle to an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platforms ``dlopen`` or ``LoadLibrary`` +parameter, otherwise the underlying platforms :c:func:`!dlopen` or :c:func:`LoadLibrary` function is used to load the library into the process, and to get a handle to it. @@ -1522,8 +1523,8 @@ underscore to not clash with exported function names: Shared libraries can also be loaded by using one of the prefabricated objects, which are instances of the :class:`LibraryLoader` class, either by calling the -:meth:`LoadLibrary` method, or by retrieving the library as attribute of the -loader instance. +:meth:`~LibraryLoader.LoadLibrary` method, or by retrieving the library as +attribute of the loader instance. .. class:: LibraryLoader(dlltype) @@ -1639,14 +1640,14 @@ They are instances of a private class: unspecified arguments as well. When a foreign function is called, each actual argument is passed to the - :meth:`from_param` class method of the items in the :attr:`argtypes` + :meth:`~_CData.from_param` class method of the items in the :attr:`argtypes` tuple, this method allows adapting the actual argument to an object that the foreign function accepts. For example, a :class:`c_char_p` item in the :attr:`argtypes` tuple will convert a string passed as argument into a bytes object using ctypes conversion rules. New: It is now possible to put items in argtypes which are not ctypes - types, but each item must have a :meth:`from_param` method which returns a + types, but each item must have a :meth:`~_CData.from_param` method which returns a value usable as argument (integer, string, ctypes instance). This allows defining adapters that can adapt custom objects as function parameters. @@ -1770,12 +1771,12 @@ different ways, depending on the type and number of the parameters in the call: COM methods use a special calling convention: They require a pointer to the COM interface as first argument, in addition to those parameters that - are specified in the :attr:`argtypes` tuple. + are specified in the :attr:`~_FuncPtr.argtypes` tuple. The optional *paramflags* parameter creates foreign function wrappers with much more functionality than the features described above. - *paramflags* must be a tuple of the same length as :attr:`argtypes`. + *paramflags* must be a tuple of the same length as :attr:`~_FuncPtr.argtypes`. Each item in this tuple contains further information about a parameter, it must be a tuple containing one, two, or three items. @@ -2157,8 +2158,8 @@ Data types This method adapts *obj* to a ctypes type. It is called with the actual object used in a foreign function call when the type is present in the - foreign function's :attr:`argtypes` tuple; it must return an object that - can be used as a function call parameter. + foreign function's :attr:`~_FuncPtr.argtypes` tuple; + it must return an object that can be used as a function call parameter. All ctypes data types have a default implementation of this classmethod that normally returns *obj* if that is an instance of the type. Some From webhook-mailer at python.org Sun Aug 6 07:08:35 2023 From: webhook-mailer at python.org (hugovk) Date: Sun, 06 Aug 2023 11:08:35 -0000 Subject: [Python-checkins] Docs: skip python-docs-theme 2023.7 to fix mobile menu (#107666) Message-ID: https://github.com/python/cpython/commit/9641c4d8e2bdf9b00dd9f373d4a74dfad000afd1 commit: 9641c4d8e2bdf9b00dd9f373d4a74dfad000afd1 branch: main author: Hugo van Kemenade committer: hugovk date: 2023-08-06T13:08:32+02:00 summary: Docs: skip python-docs-theme 2023.7 to fix mobile menu (#107666) files: M Doc/requirements.txt diff --git a/Doc/requirements.txt b/Doc/requirements.txt index e9b6e56b12289..d4f23ea8c400f 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -15,6 +15,6 @@ sphinxext-opengraph==0.7.5 # The theme used by the documentation is stored separately, so we need # to install that as well. -python-docs-theme>=2023.7 +python-docs-theme>=2023.3.1,!=2023.7 -c constraints.txt From webhook-mailer at python.org Sun Aug 6 07:53:29 2023 From: webhook-mailer at python.org (hugovk) Date: Sun, 06 Aug 2023 11:53:29 -0000 Subject: [Python-checkins] [3.11] Docs: skip python-docs-theme 2023.7 to fix mobile menu (GH-107666) (#107691) Message-ID: https://github.com/python/cpython/commit/58b31612e67eb9ef064acfee2b204095144a1408 commit: 58b31612e67eb9ef064acfee2b204095144a1408 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: hugovk date: 2023-08-06T13:53:25+02:00 summary: [3.11] Docs: skip python-docs-theme 2023.7 to fix mobile menu (GH-107666) (#107691) Docs: skip python-docs-theme 2023.7 to fix mobile menu (GH-107666) (cherry picked from commit 9641c4d8e2bdf9b00dd9f373d4a74dfad000afd1) Co-authored-by: Hugo van Kemenade files: M Doc/requirements.txt diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 2b8d4df214730..3c0e0a7733673 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -15,6 +15,6 @@ sphinxext-opengraph>=0.7.1 # The theme used by the documentation is stored separately, so we need # to install that as well. -python-docs-theme>=2023.7 +python-docs-theme>=2023.3.1,!=2023.7 -c constraints.txt From webhook-mailer at python.org Sun Aug 6 08:37:16 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Sun, 06 Aug 2023 12:37:16 -0000 Subject: [Python-checkins] gh-85160: improve performance of `functools.singledispatchmethod` (#107148) Message-ID: https://github.com/python/cpython/commit/3e334ae259bd922733fac14f2ebac6e3dc93a5e1 commit: 3e334ae259bd922733fac14f2ebac6e3dc93a5e1 branch: main author: Pieter Eendebak committer: AlexWaygood date: 2023-08-06T13:37:12+01:00 summary: gh-85160: improve performance of `functools.singledispatchmethod` (#107148) Co-authored-by: mental Co-authored-by: Alex Waygood files: A Misc/NEWS.d/next/Library/2020-11-10-07-04-15.bpo-40988.5kBC-O.rst M Lib/functools.py M Lib/test/test_functools.py diff --git a/Lib/functools.py b/Lib/functools.py index 8518450a8d499..2a8a69b3c527a 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -928,11 +928,14 @@ class singledispatchmethod: """ def __init__(self, func): + import weakref # see comment in singledispatch function if not callable(func) and not hasattr(func, "__get__"): raise TypeError(f"{func!r} is not callable or a descriptor") self.dispatcher = singledispatch(func) self.func = func + self._method_cache = weakref.WeakKeyDictionary() + self._all_weakrefable_instances = True def register(self, cls, method=None): """generic_method.register(cls, func) -> func @@ -942,13 +945,27 @@ def register(self, cls, method=None): return self.dispatcher.register(cls, func=method) def __get__(self, obj, cls=None): + if self._all_weakrefable_instances: + try: + _method = self._method_cache[obj] + except TypeError: + self._all_weakrefable_instances = False + except KeyError: + pass + else: + return _method + + dispatch = self.dispatcher.dispatch def _method(*args, **kwargs): - method = self.dispatcher.dispatch(args[0].__class__) - return method.__get__(obj, cls)(*args, **kwargs) + return dispatch(args[0].__class__).__get__(obj, cls)(*args, **kwargs) _method.__isabstractmethod__ = self.__isabstractmethod__ _method.register = self.register update_wrapper(_method, self.func) + + if self._all_weakrefable_instances: + self._method_cache[obj] = _method + return _method @property diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index c4eca0f5b7951..50770f066a5e1 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -2474,6 +2474,74 @@ def _(arg): self.assertTrue(A.t('')) self.assertEqual(A.t(0.0), 0.0) + def test_slotted_class(self): + class Slot: + __slots__ = ('a', 'b') + @functools.singledispatchmethod + def go(self, item, arg): + pass + + @go.register + def _(self, item: int, arg): + return item + arg + + s = Slot() + self.assertEqual(s.go(1, 1), 2) + + def test_classmethod_slotted_class(self): + class Slot: + __slots__ = ('a', 'b') + @functools.singledispatchmethod + @classmethod + def go(cls, item, arg): + pass + + @go.register + @classmethod + def _(cls, item: int, arg): + return item + arg + + s = Slot() + self.assertEqual(s.go(1, 1), 2) + self.assertEqual(Slot.go(1, 1), 2) + + def test_staticmethod_slotted_class(self): + class A: + __slots__ = ['a'] + @functools.singledispatchmethod + @staticmethod + def t(arg): + return arg + @t.register(int) + @staticmethod + def _(arg): + return isinstance(arg, int) + @t.register(str) + @staticmethod + def _(arg): + return isinstance(arg, str) + a = A() + + self.assertTrue(A.t(0)) + self.assertTrue(A.t('')) + self.assertEqual(A.t(0.0), 0.0) + self.assertTrue(a.t(0)) + self.assertTrue(a.t('')) + self.assertEqual(a.t(0.0), 0.0) + + def test_assignment_behavior(self): + # see gh-106448 + class A: + @functools.singledispatchmethod + def t(arg): + return arg + + a = A() + a.t.foo = 'bar' + a2 = A() + with self.assertRaises(AttributeError): + a2.t.foo + def test_classmethod_register(self): class A: def __init__(self, arg): diff --git a/Misc/NEWS.d/next/Library/2020-11-10-07-04-15.bpo-40988.5kBC-O.rst b/Misc/NEWS.d/next/Library/2020-11-10-07-04-15.bpo-40988.5kBC-O.rst new file mode 100644 index 0000000000000..9323d93c59b05 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-11-10-07-04-15.bpo-40988.5kBC-O.rst @@ -0,0 +1,3 @@ +Improve performance of :class:`functools.singledispatchmethod` by caching the +generated dispatch wrapper. Optimization suggested by frederico. Patch by + at mental32, Alex Waygood and Pieter Eendebak. From webhook-mailer at python.org Sun Aug 6 08:51:12 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 06 Aug 2023 12:51:12 -0000 Subject: [Python-checkins] Python 3.12.0rc1 Message-ID: https://github.com/python/cpython/commit/63bcd91daccc7c873d57e21406d038f8216b6ddf commit: 63bcd91daccc7c873d57e21406d038f8216b6ddf branch: 3.12 author: Thomas Wouters committer: Yhg1s date: 2023-08-05T14:11:50+02:00 summary: Python 3.12.0rc1 files: A Misc/NEWS.d/3.12.0rc1.rst D Misc/NEWS.d/next/Build/2023-02-03-21-36-42.gh-issue-101538.sF5F6S.rst D Misc/NEWS.d/next/Build/2023-07-23-00-38-51.gh-issue-106962.VVYrWB.rst D Misc/NEWS.d/next/Build/2023-07-28-18-17-33.gh-issue-106881.U3Ezdq.rst D Misc/NEWS.d/next/C API/2023-07-25-13-41-09.gh-issue-107226.N919zH.rst D Misc/NEWS.d/next/Core and Builtins/2023-06-02-19-37-29.gh-issue-105235.fgFGTi.rst D Misc/NEWS.d/next/Core and Builtins/2023-07-13-14-55-45.gh-issue-106723.KsMufQ.rst D Misc/NEWS.d/next/Core and Builtins/2023-07-13-15-59-07.gh-issue-106719.jmVrsv.rst D Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst D Misc/NEWS.d/next/Core and Builtins/2023-07-20-12-21-37.gh-issue-105699.08ywGV.rst D Misc/NEWS.d/next/Core and Builtins/2023-07-20-15-15-57.gh-issue-105699.DdqHFg.rst D Misc/NEWS.d/next/Core and Builtins/2023-07-21-14-37-48.gh-issue-106917.1jWp_m.rst D Misc/NEWS.d/next/Core and Builtins/2023-07-24-11-11-41.gh-issue-104621.vM8Y_l.rst D Misc/NEWS.d/next/Core and Builtins/2023-07-26-12-18-10.gh-issue-106897.EsGurc.rst D Misc/NEWS.d/next/Core and Builtins/2023-07-26-18-53-34.gh-issue-106895.DdEwV8.rst D Misc/NEWS.d/next/Core and Builtins/2023-07-26-21-28-06.gh-issue-106898.8Wjuiv.rst D Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-47-29.gh-issue-104432.oGHF-z.rst D Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst D Misc/NEWS.d/next/Documentation/2023-05-16-22-08-24.gh-issue-54738.mJvCnj.rst D Misc/NEWS.d/next/Documentation/2023-07-21-11-51-57.gh-issue-106948.K_JQ7j.rst D Misc/NEWS.d/next/Documentation/2023-07-22-15-14-13.gh-issue-107008.3JQ1Vt.rst D Misc/NEWS.d/next/Documentation/2023-07-26-16-33-04.gh-issue-107305.qB2LS4.rst D Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst D Misc/NEWS.d/next/Library/2023-06-10-12-20-17.gh-issue-105626.XyZein.rst D Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst D Misc/NEWS.d/next/Library/2023-07-03-03-46-20.gh-issue-106350.LLcTEe.rst D Misc/NEWS.d/next/Library/2023-07-04-07-25-30.gh-issue-106403.GmefbV.rst D Misc/NEWS.d/next/Library/2023-07-11-09-25-40.gh-issue-106530.VgXrMx.rst D Misc/NEWS.d/next/Library/2023-07-12-04-58-45.gh-issue-106602.dGCcXe.rst D Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst D Misc/NEWS.d/next/Library/2023-07-15-10-24-56.gh-issue-106774.FJcqCj.rst D Misc/NEWS.d/next/Library/2023-07-17-21-45-15.gh-issue-106831.RqVq9X.rst D Misc/NEWS.d/next/Library/2023-07-22-12-53-53.gh-issue-105002.gkfsW0.rst D Misc/NEWS.d/next/Library/2023-07-22-13-09-28.gh-issue-106186.EIsUNG.rst D Misc/NEWS.d/next/Library/2023-07-22-15-51-33.gh-issue-83006.21zaCz.rst D Misc/NEWS.d/next/Library/2023-07-23-12-26-23.gh-issue-62519.w8-81X.rst D Misc/NEWS.d/next/Library/2023-07-24-01-21-16.gh-issue-46376.w-xuDL.rst D Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst D Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst D Misc/NEWS.d/next/Security/2023-03-07-21-46-29.gh-issue-102509.5ouaH_.rst D Misc/NEWS.d/next/Security/2023-06-13-20-52-24.gh-issue-102988.Kei7Vf.rst D Misc/NEWS.d/next/Tests/2023-07-14-16-20-06.gh-issue-106752.gd1i6D.rst D Misc/NEWS.d/next/Tests/2023-07-16-02-57-08.gh-issue-104090.cKtK7g.rst D Misc/NEWS.d/next/Tests/2023-07-22-13-49-40.gh-issue-106714.btYI5S.rst D Misc/NEWS.d/next/Tests/2023-07-25-14-36-33.gh-issue-107237.y1pY79.rst D Misc/NEWS.d/next/Tools-Demos/2023-04-05-07-19-36.gh-issue-103186.yEozgK.rst D Misc/NEWS.d/next/Tools-Demos/2023-07-21-23-16-05.gh-issue-106970.NLRnml.rst D Misc/NEWS.d/next/Windows/2023-07-11-20-48-17.gh-issue-99079.CIMftz.rst D Misc/NEWS.d/next/macOS/2023-07-30-23-42-20.gh-issue-99079.JAtoh1.rst M Include/patchlevel.h M Lib/pydoc_data/topics.py M README.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index e3d7ef51ba60c..51e0e9b6d758f 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -19,11 +19,11 @@ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 12 #define PY_MICRO_VERSION 0 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_BETA -#define PY_RELEASE_SERIAL 4 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA +#define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.12.0b4+" +#define PY_VERSION "3.12.0rc1" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 9603975729cfa..8d19a85963fe4 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Jul 11 14:22:58 2023 +# Autogenerated by Sphinx on Sat Aug 5 14:10:40 2023 # as part of the release process. topics = {'assert': 'The "assert" statement\n' '**********************\n' @@ -9689,7 +9689,8 @@ ' still alive. The list is in definition order. Example:\n' '\n' ' >>> int.__subclasses__()\n' - " []\n", + " [, , , " + "]\n", 'specialnames': 'Special method names\n' '********************\n' '\n' @@ -12558,7 +12559,7 @@ 'followed by\n' ' the string itself.\n' '\n' - 'str.rsplit(sep=None, maxsplit=- 1)\n' + 'str.rsplit(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -12599,7 +12600,7 @@ " >>> 'Monty Python'.removesuffix(' Python')\n" " 'Monty'\n" '\n' - 'str.split(sep=None, maxsplit=- 1)\n' + 'str.split(sep=None, maxsplit=-1)\n' '\n' ' Return a list of the words in the string, using *sep* ' 'as the\n' @@ -13009,6 +13010,10 @@ 'the\n' 'literal, i.e. either "\'" or """.)\n' '\n' + '\n' + 'Escape sequences\n' + '================\n' + '\n' 'Unless an "\'r\'" or "\'R\'" prefix is present, escape sequences ' 'in string\n' 'and bytes literals are interpreted according to rules similar to ' @@ -15184,10 +15189,12 @@ ' >>> # set operations\n' " >>> keys & {'eggs', 'bacon', 'salad'}\n" " {'bacon'}\n" - " >>> keys ^ {'sausage', 'juice'}\n" - " {'juice', 'sausage', 'bacon', 'spam'}\n" - " >>> keys | ['juice', 'juice', 'juice']\n" - " {'juice', 'sausage', 'bacon', 'spam', 'eggs'}\n" + " >>> keys ^ {'sausage', 'juice'} == {'juice', 'sausage', " + "'bacon', 'spam'}\n" + ' True\n' + " >>> keys | ['juice', 'juice', 'juice'] == {'bacon', " + "'spam', 'juice'}\n" + ' True\n' '\n' ' >>> # get back a read-only proxy for the original ' 'dictionary\n' diff --git a/Misc/NEWS.d/3.12.0rc1.rst b/Misc/NEWS.d/3.12.0rc1.rst new file mode 100644 index 0000000000000..ca826cf24258c --- /dev/null +++ b/Misc/NEWS.d/3.12.0rc1.rst @@ -0,0 +1,495 @@ +.. date: 2023-06-13-20-52-24 +.. gh-issue: 102988 +.. nonce: Kei7Vf +.. release date: 2023-08-05 +.. section: Security + +Reverted the :mod:`email.utils` security improvement change released in +3.12beta4 that unintentionally caused :mod:`email.utils.getaddresses` to +fail to parse email addresses with a comma in the quoted name field. See +:gh:`106669`. + +.. + +.. date: 2023-03-07-21-46-29 +.. gh-issue: 102509 +.. nonce: 5ouaH_ +.. section: Security + +Start initializing ``ob_digit`` during creation of :c:type:`PyLongObject` +objects. Patch by Illia Volochii. + +.. + +.. date: 2023-07-30-05-20-16 +.. gh-issue: 107263 +.. nonce: q0IU2M +.. section: Core and Builtins + +Increase C recursion limit for functions other than the main interpreter +from 800 to 1500. This should allow functions like ``list.__repr__`` and +``json.dumps`` to handle all the inputs that they could prior to 3.12 + +.. + +.. date: 2023-07-27-11-47-29 +.. gh-issue: 104432 +.. nonce: oGHF-z +.. section: Core and Builtins + +Fix potential unaligned memory access on C APIs involving returned sequences +of `char *` pointers within the :mod:`grp` and :mod:`socket` modules. These +were revealed using a ``-fsaniziter=alignment`` build on ARM macOS. Patch by +Christopher Chavez. + +.. + +.. date: 2023-07-26-21-28-06 +.. gh-issue: 106898 +.. nonce: 8Wjuiv +.. section: Core and Builtins + +Add the exception as the third argument to ``PY_UNIND`` callbacks in +``sys.monitoring``. This makes the ``PY_UNWIND`` callback consistent with +the other exception hanlding callbacks. + +.. + +.. date: 2023-07-26-18-53-34 +.. gh-issue: 106895 +.. nonce: DdEwV8 +.. section: Core and Builtins + +Raise a ``ValueError`` when a monitoring callback funtion returns +``DISABLE`` for events that cannot be disabled locally. + +.. + +.. date: 2023-07-26-12-18-10 +.. gh-issue: 106897 +.. nonce: EsGurc +.. section: Core and Builtins + +Add a ``RERAISE`` event to ``sys.monitoring``, which occurs when an +exception is reraised, either explicitly by a plain ``raise`` statement, or +implicitly in an ``except`` or ``finally`` block. + +.. + +.. date: 2023-07-24-11-11-41 +.. gh-issue: 104621 +.. nonce: vM8Y_l +.. section: Core and Builtins + +Unsupported modules now always fail to be imported. + +.. + +.. date: 2023-07-21-14-37-48 +.. gh-issue: 106917 +.. nonce: 1jWp_m +.. section: Core and Builtins + +Fix classmethod-style :func:`super` method calls (i.e., where the second +argument to :func:`super`, or the implied second argument drawn from +``self/cls`` in the case of zero-arg super, is a type) when the target of +the call is not a classmethod. + +.. + +.. date: 2023-07-20-15-15-57 +.. gh-issue: 105699 +.. nonce: DdqHFg +.. section: Core and Builtins + +Python no longer crashes due an infrequent race when initialzing +per-interpreter interned strings. The crash would manifest when the +interpreter was finalized. + +.. + +.. date: 2023-07-20-12-21-37 +.. gh-issue: 105699 +.. nonce: 08ywGV +.. section: Core and Builtins + +Python no longer crashes due to an infrequent race in setting +``Py_FileSystemDefaultEncoding`` and ``Py_FileSystemDefaultEncodeErrors`` +(both deprecated), when simultaneously initializing two isolated +subinterpreters. Now they are only set during runtime initialization. + +.. + +.. date: 2023-07-18-16-13-51 +.. gh-issue: 106092 +.. nonce: bObgRM +.. section: Core and Builtins + +Fix a segmentation fault caused by a use-after-free bug in ``frame_dealloc`` +when the trashcan delays the deallocation of a ``PyFrameObject``. + +.. + +.. date: 2023-07-13-15-59-07 +.. gh-issue: 106719 +.. nonce: jmVrsv +.. section: Core and Builtins + +No longer suppress arbitrary errors in the ``__annotations__`` getter and +setter in the type and module types. + +.. + +.. date: 2023-07-13-14-55-45 +.. gh-issue: 106723 +.. nonce: KsMufQ +.. section: Core and Builtins + +Propagate ``frozen_modules`` to multiprocessing spawned process +interpreters. + +.. + +.. date: 2023-06-02-19-37-29 +.. gh-issue: 105235 +.. nonce: fgFGTi +.. section: Core and Builtins + +Prevent out-of-bounds memory access during ``mmap.find()`` calls. + +.. + +.. date: 2023-08-03-12-52-19 +.. gh-issue: 107077 +.. nonce: -pzHD6 +.. section: Library + +Seems that in some conditions, OpenSSL will return ``SSL_ERROR_SYSCALL`` +instead of ``SSL_ERROR_SSL`` when a certification verification has failed, +but the error parameters will still contain ``ERR_LIB_SSL`` and +``SSL_R_CERTIFICATE_VERIFY_FAILED``. We are now detecting this situation and +raising the appropiate ``ssl.SSLCertVerificationError``. Patch by Pablo +Galindo + +.. + +.. date: 2023-08-03-11-31-11 +.. gh-issue: 107576 +.. nonce: pO_s9I +.. section: Library + +Fix :func:`types.get_original_bases` to only return :attr:`!__orig_bases__` +if it is present on ``cls`` directly. Patch by James Hilton-Balfe. + +.. + +.. date: 2023-07-24-01-21-16 +.. gh-issue: 46376 +.. nonce: w-xuDL +.. section: Library + +Prevent memory leak and use-after-free when using pointers to pointers with +ctypes + +.. + +.. date: 2023-07-23-12-26-23 +.. gh-issue: 62519 +.. nonce: w8-81X +.. section: Library + +Make :func:`gettext.pgettext` search plural definitions when translation is +not found. + +.. + +.. date: 2023-07-22-15-51-33 +.. gh-issue: 83006 +.. nonce: 21zaCz +.. section: Library + +Document behavior of :func:`shutil.disk_usage` for non-mounted filesystems +on Unix. + +.. + +.. date: 2023-07-22-13-09-28 +.. gh-issue: 106186 +.. nonce: EIsUNG +.. section: Library + +Do not report ``MultipartInvariantViolationDefect`` defect when the +:class:`email.parser.Parser` class is used to parse emails with +``headersonly=True``. + +.. + +.. date: 2023-07-22-12-53-53 +.. gh-issue: 105002 +.. nonce: gkfsW0 +.. section: Library + +Fix invalid result from :meth:`PurePath.relative_to` method when attempting +to walk a "``..``" segment in *other* with *walk_up* enabled. A +:exc:`ValueError` exception is now raised in this case. + +.. + +.. date: 2023-07-17-21-45-15 +.. gh-issue: 106831 +.. nonce: RqVq9X +.. section: Library + +Fix potential missing ``NULL`` check of ``d2i_SSL_SESSION`` result in +``_ssl.c``. + +.. + +.. date: 2023-07-15-10-24-56 +.. gh-issue: 106774 +.. nonce: FJcqCj +.. section: Library + +Update the bundled copy of pip to version 23.2.1. + +.. + +.. date: 2023-07-14-16-54-13 +.. gh-issue: 106752 +.. nonce: BT1Yxw +.. section: Library + +Fixed several bugs in zipfile.Path, including: in ``Path.match`, Windows +separators are no longer honored (and never were meant to be); Fixed +``name``/``suffix``/``suffixes``/``stem`` operations when no filename is +present and the Path is not at the root of the zipfile; Reworked glob for +performance and more correct matching behavior. + +.. + +.. date: 2023-07-12-04-58-45 +.. gh-issue: 106602 +.. nonce: dGCcXe +.. section: Library + +Add __copy__ and __deepcopy__ in :mod:`enum` + +.. + +.. date: 2023-07-11-09-25-40 +.. gh-issue: 106530 +.. nonce: VgXrMx +.. section: Library + +Revert a change to :func:`colorsys.rgb_to_hls` that caused division by zero +for certain almost-white inputs. Patch by Terry Jan Reedy. + +.. + +.. date: 2023-07-04-07-25-30 +.. gh-issue: 106403 +.. nonce: GmefbV +.. section: Library + +Instances of :class:`typing.TypeVar`, :class:`typing.ParamSpec`, +:class:`typing.ParamSpecArgs`, :class:`typing.ParamSpecKwargs`, and +:class:`typing.TypeVarTuple` once again support weak references, fixing a +regression introduced in Python 3.12.0 beta 1. Patch by Jelle Zijlstra. + +.. + +.. date: 2023-07-03-03-46-20 +.. gh-issue: 106350 +.. nonce: LLcTEe +.. section: Library + +Detect possible memory allocation failure in the libtommath function +:c:func:`mp_init` used by the ``_tkinter`` module. + +.. + +.. date: 2023-06-30-16-42-44 +.. gh-issue: 106263 +.. nonce: tk-t93 +.. section: Library + +Fix crash when calling ``repr`` with a manually constructed SignalDict +object. Patch by Charlie Zhao. + +.. + +.. date: 2023-06-10-12-20-17 +.. gh-issue: 105626 +.. nonce: XyZein +.. section: Library + +Change the default return value of +:meth:`http.client.HTTPConnection.get_proxy_response_headers` to be ``None`` +and not ``{}``. + +.. + +.. bpo: 18319 +.. date: 2020-05-03-00-33-15 +.. nonce: faPTlx +.. section: Library + +Ensure `gettext(msg)` retrieve translations even if a plural form exists. In +other words: `gettext(msg) == ngettext(msg, '', 1)`. + +.. + +.. date: 2023-07-26-16-33-04 +.. gh-issue: 107305 +.. nonce: qB2LS4 +.. section: Documentation + +Add documentation for :c:type:`PyInterpreterConfig` and +:c:func:`Py_NewInterpreterFromConfig`. Also clarify some of the nearby docs +relative to per-interpreter GIL. + +.. + +.. date: 2023-07-22-15-14-13 +.. gh-issue: 107008 +.. nonce: 3JQ1Vt +.. section: Documentation + +Document the :mod:`curses` module variables :const:`~curses.LINES` and +:const:`~curses.COLS`. + +.. + +.. date: 2023-07-21-11-51-57 +.. gh-issue: 106948 +.. nonce: K_JQ7j +.. section: Documentation + +Add a number of standard external names to ``nitpick_ignore``. + +.. + +.. date: 2023-05-16-22-08-24 +.. gh-issue: 54738 +.. nonce: mJvCnj +.. section: Documentation + +Add documentation on how to localize the :mod:`argparse` module. + +.. + +.. date: 2023-07-25-14-36-33 +.. gh-issue: 107237 +.. nonce: y1pY79 +.. section: Tests + +``test_logging``: Fix ``test_udp_reconnection()`` by increasing the timeout +from 100 ms to 5 minutes (LONG_TIMEOUT). Patch by Victor Stinner. + +.. + +.. date: 2023-07-22-13-49-40 +.. gh-issue: 106714 +.. nonce: btYI5S +.. section: Tests + +test_capi: Fix test_no_FatalError_infinite_loop() to no longer write a +coredump, by using test.support.SuppressCrashReport. Patch by Victor +Stinner. + +.. + +.. date: 2023-07-16-02-57-08 +.. gh-issue: 104090 +.. nonce: cKtK7g +.. section: Tests + +Avoid creating a reference to the test object in +:meth:`~unittest.TestResult.collectedDurations`. + +.. + +.. date: 2023-07-14-16-20-06 +.. gh-issue: 106752 +.. nonce: gd1i6D +.. section: Tests + +Moved tests for ``zipfile.Path`` into ``Lib/test/test_zipfile/_path``. Made +``zipfile._path`` a package. + +.. + +.. date: 2023-07-28-18-17-33 +.. gh-issue: 106881 +.. nonce: U3Ezdq +.. section: Build + +Check for `linux/limits.h` before including it in `Modules/posixmodule.c`. + +.. + +.. date: 2023-07-23-00-38-51 +.. gh-issue: 106962 +.. nonce: VVYrWB +.. section: Build + +Detect MPI compilers in :file:`configure`. + +.. + +.. date: 2023-02-03-21-36-42 +.. gh-issue: 101538 +.. nonce: sF5F6S +.. section: Build + +Add experimental wasi-threads support. Patch by Takashi Yamamoto. + +.. + +.. date: 2023-07-11-20-48-17 +.. gh-issue: 99079 +.. nonce: CIMftz +.. section: Windows + +Update Windows build to use OpenSSL 3.0.9 + +.. + +.. date: 2023-07-30-23-42-20 +.. gh-issue: 99079 +.. nonce: JAtoh1 +.. section: macOS + +Update macOS installer to use OpenSSL 3.0.9. + +.. + +.. date: 2023-07-21-23-16-05 +.. gh-issue: 106970 +.. nonce: NLRnml +.. section: Tools/Demos + +Fix bugs in the Argument Clinic ``destination clear`` command; the +destination buffers would never be cleared, and the ``destination`` +directive parser would simply continue to the fault handler after processing +the command. Patch by Erlend E. Aasland. + +.. + +.. date: 2023-04-05-07-19-36 +.. gh-issue: 103186 +.. nonce: yEozgK +.. section: Tools/Demos + +``freeze`` now fetches ``CONFIG_ARGS`` from the original CPython instance +the Makefile uses to call utility scripts. Patch by Ijtaba Hussain. + +.. + +.. date: 2023-07-25-13-41-09 +.. gh-issue: 107226 +.. nonce: N919zH +.. section: C API + +:c:func:`PyModule_AddObjectRef` is now only available in the limited API +version 3.10 or later. diff --git a/Misc/NEWS.d/next/Build/2023-02-03-21-36-42.gh-issue-101538.sF5F6S.rst b/Misc/NEWS.d/next/Build/2023-02-03-21-36-42.gh-issue-101538.sF5F6S.rst deleted file mode 100644 index 4b83c303b3d2c..0000000000000 --- a/Misc/NEWS.d/next/Build/2023-02-03-21-36-42.gh-issue-101538.sF5F6S.rst +++ /dev/null @@ -1 +0,0 @@ -Add experimental wasi-threads support. Patch by Takashi Yamamoto. diff --git a/Misc/NEWS.d/next/Build/2023-07-23-00-38-51.gh-issue-106962.VVYrWB.rst b/Misc/NEWS.d/next/Build/2023-07-23-00-38-51.gh-issue-106962.VVYrWB.rst deleted file mode 100644 index 32e196fe26d3b..0000000000000 --- a/Misc/NEWS.d/next/Build/2023-07-23-00-38-51.gh-issue-106962.VVYrWB.rst +++ /dev/null @@ -1 +0,0 @@ -Detect MPI compilers in :file:`configure`. diff --git a/Misc/NEWS.d/next/Build/2023-07-28-18-17-33.gh-issue-106881.U3Ezdq.rst b/Misc/NEWS.d/next/Build/2023-07-28-18-17-33.gh-issue-106881.U3Ezdq.rst deleted file mode 100644 index 40b2609e95c7e..0000000000000 --- a/Misc/NEWS.d/next/Build/2023-07-28-18-17-33.gh-issue-106881.U3Ezdq.rst +++ /dev/null @@ -1 +0,0 @@ -Check for `linux/limits.h` before including it in `Modules/posixmodule.c`. diff --git a/Misc/NEWS.d/next/C API/2023-07-25-13-41-09.gh-issue-107226.N919zH.rst b/Misc/NEWS.d/next/C API/2023-07-25-13-41-09.gh-issue-107226.N919zH.rst deleted file mode 100644 index 6178f18517d48..0000000000000 --- a/Misc/NEWS.d/next/C API/2023-07-25-13-41-09.gh-issue-107226.N919zH.rst +++ /dev/null @@ -1,2 +0,0 @@ -:c:func:`PyModule_AddObjectRef` is now only available in the limited API -version 3.10 or later. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-02-19-37-29.gh-issue-105235.fgFGTi.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-02-19-37-29.gh-issue-105235.fgFGTi.rst deleted file mode 100644 index c28d0101cd4ba..0000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-06-02-19-37-29.gh-issue-105235.fgFGTi.rst +++ /dev/null @@ -1 +0,0 @@ -Prevent out-of-bounds memory access during ``mmap.find()`` calls. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-13-14-55-45.gh-issue-106723.KsMufQ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-13-14-55-45.gh-issue-106723.KsMufQ.rst deleted file mode 100644 index 207f397f17d3f..0000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-13-14-55-45.gh-issue-106723.KsMufQ.rst +++ /dev/null @@ -1 +0,0 @@ -Propagate ``frozen_modules`` to multiprocessing spawned process interpreters. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-13-15-59-07.gh-issue-106719.jmVrsv.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-13-15-59-07.gh-issue-106719.jmVrsv.rst deleted file mode 100644 index dc4bef193a322..0000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-13-15-59-07.gh-issue-106719.jmVrsv.rst +++ /dev/null @@ -1,2 +0,0 @@ -No longer suppress arbitrary errors in the ``__annotations__`` getter and -setter in the type and module types. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst deleted file mode 100644 index 7fb5b45c763e4..0000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-18-16-13-51.gh-issue-106092.bObgRM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a segmentation fault caused by a use-after-free bug in ``frame_dealloc`` -when the trashcan delays the deallocation of a ``PyFrameObject``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-20-12-21-37.gh-issue-105699.08ywGV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-20-12-21-37.gh-issue-105699.08ywGV.rst deleted file mode 100644 index 82312718cd047..0000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-20-12-21-37.gh-issue-105699.08ywGV.rst +++ /dev/null @@ -1,4 +0,0 @@ -Python no longer crashes due to an infrequent race in setting -``Py_FileSystemDefaultEncoding`` and ``Py_FileSystemDefaultEncodeErrors`` -(both deprecated), when simultaneously initializing two isolated -subinterpreters. Now they are only set during runtime initialization. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-20-15-15-57.gh-issue-105699.DdqHFg.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-20-15-15-57.gh-issue-105699.DdqHFg.rst deleted file mode 100644 index 4a257c6282220..0000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-20-15-15-57.gh-issue-105699.DdqHFg.rst +++ /dev/null @@ -1,3 +0,0 @@ -Python no longer crashes due an infrequent race when initialzing -per-interpreter interned strings. The crash would manifest when the -interpreter was finalized. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-21-14-37-48.gh-issue-106917.1jWp_m.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-21-14-37-48.gh-issue-106917.1jWp_m.rst deleted file mode 100644 index 82c74d5465458..0000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-21-14-37-48.gh-issue-106917.1jWp_m.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix classmethod-style :func:`super` method calls (i.e., where the second -argument to :func:`super`, or the implied second argument drawn from -``self/cls`` in the case of zero-arg super, is a type) when the target of -the call is not a classmethod. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-24-11-11-41.gh-issue-104621.vM8Y_l.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-24-11-11-41.gh-issue-104621.vM8Y_l.rst deleted file mode 100644 index 86c976295f262..0000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-24-11-11-41.gh-issue-104621.vM8Y_l.rst +++ /dev/null @@ -1 +0,0 @@ -Unsupported modules now always fail to be imported. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-26-12-18-10.gh-issue-106897.EsGurc.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-26-12-18-10.gh-issue-106897.EsGurc.rst deleted file mode 100644 index 52c49c3c0f5e7..0000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-26-12-18-10.gh-issue-106897.EsGurc.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add a ``RERAISE`` event to ``sys.monitoring``, which occurs when an -exception is reraised, either explicitly by a plain ``raise`` statement, or -implicitly in an ``except`` or ``finally`` block. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-26-18-53-34.gh-issue-106895.DdEwV8.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-26-18-53-34.gh-issue-106895.DdEwV8.rst deleted file mode 100644 index 370a29d34c860..0000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-26-18-53-34.gh-issue-106895.DdEwV8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Raise a ``ValueError`` when a monitoring callback funtion returns -``DISABLE`` for events that cannot be disabled locally. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-26-21-28-06.gh-issue-106898.8Wjuiv.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-26-21-28-06.gh-issue-106898.8Wjuiv.rst deleted file mode 100644 index f1b1c4c64b4ac..0000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-26-21-28-06.gh-issue-106898.8Wjuiv.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add the exception as the third argument to ``PY_UNIND`` callbacks in -``sys.monitoring``. This makes the ``PY_UNWIND`` callback consistent with -the other exception hanlding callbacks. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-47-29.gh-issue-104432.oGHF-z.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-47-29.gh-issue-104432.oGHF-z.rst deleted file mode 100644 index e47927b4e1188..0000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-47-29.gh-issue-104432.oGHF-z.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix potential unaligned memory access on C APIs involving returned sequences -of `char *` pointers within the :mod:`grp` and :mod:`socket` modules. These -were revealed using a ``-fsaniziter=alignment`` build on ARM macOS. Patch by -Christopher Chavez. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst deleted file mode 100644 index fb0940b456dae..0000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-30-05-20-16.gh-issue-107263.q0IU2M.rst +++ /dev/null @@ -1,3 +0,0 @@ -Increase C recursion limit for functions other than the main interpreter -from 800 to 1500. This should allow functions like ``list.__repr__`` and -``json.dumps`` to handle all the inputs that they could prior to 3.12 diff --git a/Misc/NEWS.d/next/Documentation/2023-05-16-22-08-24.gh-issue-54738.mJvCnj.rst b/Misc/NEWS.d/next/Documentation/2023-05-16-22-08-24.gh-issue-54738.mJvCnj.rst deleted file mode 100644 index 4da58fc982b6d..0000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-05-16-22-08-24.gh-issue-54738.mJvCnj.rst +++ /dev/null @@ -1 +0,0 @@ -Add documentation on how to localize the :mod:`argparse` module. diff --git a/Misc/NEWS.d/next/Documentation/2023-07-21-11-51-57.gh-issue-106948.K_JQ7j.rst b/Misc/NEWS.d/next/Documentation/2023-07-21-11-51-57.gh-issue-106948.K_JQ7j.rst deleted file mode 100644 index 42b6348153b56..0000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-07-21-11-51-57.gh-issue-106948.K_JQ7j.rst +++ /dev/null @@ -1 +0,0 @@ -Add a number of standard external names to ``nitpick_ignore``. diff --git a/Misc/NEWS.d/next/Documentation/2023-07-22-15-14-13.gh-issue-107008.3JQ1Vt.rst b/Misc/NEWS.d/next/Documentation/2023-07-22-15-14-13.gh-issue-107008.3JQ1Vt.rst deleted file mode 100644 index a0fa27ec10303..0000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-07-22-15-14-13.gh-issue-107008.3JQ1Vt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Document the :mod:`curses` module variables :const:`~curses.LINES` and -:const:`~curses.COLS`. diff --git a/Misc/NEWS.d/next/Documentation/2023-07-26-16-33-04.gh-issue-107305.qB2LS4.rst b/Misc/NEWS.d/next/Documentation/2023-07-26-16-33-04.gh-issue-107305.qB2LS4.rst deleted file mode 100644 index 038f9e68a5422..0000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-07-26-16-33-04.gh-issue-107305.qB2LS4.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add documentation for :c:type:`PyInterpreterConfig` and -:c:func:`Py_NewInterpreterFromConfig`. Also clarify some of the nearby docs -relative to per-interpreter GIL. diff --git a/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst b/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst deleted file mode 100644 index a1a4cf6d63725..0000000000000 --- a/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst +++ /dev/null @@ -1,2 +0,0 @@ -Ensure `gettext(msg)` retrieve translations even if a plural form exists. In -other words: `gettext(msg) == ngettext(msg, '', 1)`. diff --git a/Misc/NEWS.d/next/Library/2023-06-10-12-20-17.gh-issue-105626.XyZein.rst b/Misc/NEWS.d/next/Library/2023-06-10-12-20-17.gh-issue-105626.XyZein.rst deleted file mode 100644 index 2a48361fa596c..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-06-10-12-20-17.gh-issue-105626.XyZein.rst +++ /dev/null @@ -1,3 +0,0 @@ -Change the default return value of -:meth:`http.client.HTTPConnection.get_proxy_response_headers` to be ``None`` -and not ``{}``. diff --git a/Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst b/Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst deleted file mode 100644 index 23763818d84ba..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix crash when calling ``repr`` with a manually constructed SignalDict object. -Patch by Charlie Zhao. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2023-07-03-03-46-20.gh-issue-106350.LLcTEe.rst b/Misc/NEWS.d/next/Library/2023-07-03-03-46-20.gh-issue-106350.LLcTEe.rst deleted file mode 100644 index 681d63a6668be..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-03-03-46-20.gh-issue-106350.LLcTEe.rst +++ /dev/null @@ -1,2 +0,0 @@ -Detect possible memory allocation failure in the libtommath function :c:func:`mp_init` -used by the ``_tkinter`` module. diff --git a/Misc/NEWS.d/next/Library/2023-07-04-07-25-30.gh-issue-106403.GmefbV.rst b/Misc/NEWS.d/next/Library/2023-07-04-07-25-30.gh-issue-106403.GmefbV.rst deleted file mode 100644 index 4fea45f16c4f8..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-04-07-25-30.gh-issue-106403.GmefbV.rst +++ /dev/null @@ -1,4 +0,0 @@ -Instances of :class:`typing.TypeVar`, :class:`typing.ParamSpec`, -:class:`typing.ParamSpecArgs`, :class:`typing.ParamSpecKwargs`, and -:class:`typing.TypeVarTuple` once again support weak references, fixing a -regression introduced in Python 3.12.0 beta 1. Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2023-07-11-09-25-40.gh-issue-106530.VgXrMx.rst b/Misc/NEWS.d/next/Library/2023-07-11-09-25-40.gh-issue-106530.VgXrMx.rst deleted file mode 100644 index 09fc647cc01d2..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-11-09-25-40.gh-issue-106530.VgXrMx.rst +++ /dev/null @@ -1,2 +0,0 @@ -Revert a change to :func:`colorsys.rgb_to_hls` that caused division by zero -for certain almost-white inputs. Patch by Terry Jan Reedy. diff --git a/Misc/NEWS.d/next/Library/2023-07-12-04-58-45.gh-issue-106602.dGCcXe.rst b/Misc/NEWS.d/next/Library/2023-07-12-04-58-45.gh-issue-106602.dGCcXe.rst deleted file mode 100644 index d9c122f1d3c72..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-12-04-58-45.gh-issue-106602.dGCcXe.rst +++ /dev/null @@ -1 +0,0 @@ -Add __copy__ and __deepcopy__ in :mod:`enum` diff --git a/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst b/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst deleted file mode 100644 index bbc53d76decbc..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fixed several bugs in zipfile.Path, including: in ``Path.match`, Windows -separators are no longer honored (and never were meant to be); Fixed -``name``/``suffix``/``suffixes``/``stem`` operations when no filename is -present and the Path is not at the root of the zipfile; Reworked glob for -performance and more correct matching behavior. diff --git a/Misc/NEWS.d/next/Library/2023-07-15-10-24-56.gh-issue-106774.FJcqCj.rst b/Misc/NEWS.d/next/Library/2023-07-15-10-24-56.gh-issue-106774.FJcqCj.rst deleted file mode 100644 index ed467573b89e1..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-15-10-24-56.gh-issue-106774.FJcqCj.rst +++ /dev/null @@ -1 +0,0 @@ -Update the bundled copy of pip to version 23.2.1. diff --git a/Misc/NEWS.d/next/Library/2023-07-17-21-45-15.gh-issue-106831.RqVq9X.rst b/Misc/NEWS.d/next/Library/2023-07-17-21-45-15.gh-issue-106831.RqVq9X.rst deleted file mode 100644 index d3b9862684539..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-17-21-45-15.gh-issue-106831.RqVq9X.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix potential missing ``NULL`` check of ``d2i_SSL_SESSION`` result in -``_ssl.c``. diff --git a/Misc/NEWS.d/next/Library/2023-07-22-12-53-53.gh-issue-105002.gkfsW0.rst b/Misc/NEWS.d/next/Library/2023-07-22-12-53-53.gh-issue-105002.gkfsW0.rst deleted file mode 100644 index b4c133a5cb124..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-22-12-53-53.gh-issue-105002.gkfsW0.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix invalid result from :meth:`PurePath.relative_to` method when attempting to walk -a "``..``" segment in *other* with *walk_up* enabled. A :exc:`ValueError` exception -is now raised in this case. diff --git a/Misc/NEWS.d/next/Library/2023-07-22-13-09-28.gh-issue-106186.EIsUNG.rst b/Misc/NEWS.d/next/Library/2023-07-22-13-09-28.gh-issue-106186.EIsUNG.rst deleted file mode 100644 index 07fdcc96fa38a..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-22-13-09-28.gh-issue-106186.EIsUNG.rst +++ /dev/null @@ -1,3 +0,0 @@ -Do not report ``MultipartInvariantViolationDefect`` defect -when the :class:`email.parser.Parser` class is used -to parse emails with ``headersonly=True``. diff --git a/Misc/NEWS.d/next/Library/2023-07-22-15-51-33.gh-issue-83006.21zaCz.rst b/Misc/NEWS.d/next/Library/2023-07-22-15-51-33.gh-issue-83006.21zaCz.rst deleted file mode 100644 index e64d186082843..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-22-15-51-33.gh-issue-83006.21zaCz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Document behavior of :func:`shutil.disk_usage` for non-mounted filesystems -on Unix. diff --git a/Misc/NEWS.d/next/Library/2023-07-23-12-26-23.gh-issue-62519.w8-81X.rst b/Misc/NEWS.d/next/Library/2023-07-23-12-26-23.gh-issue-62519.w8-81X.rst deleted file mode 100644 index 96e2a3dcc24fb..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-23-12-26-23.gh-issue-62519.w8-81X.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make :func:`gettext.pgettext` search plural definitions when -translation is not found. diff --git a/Misc/NEWS.d/next/Library/2023-07-24-01-21-16.gh-issue-46376.w-xuDL.rst b/Misc/NEWS.d/next/Library/2023-07-24-01-21-16.gh-issue-46376.w-xuDL.rst deleted file mode 100644 index 8e8f0245b4539..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-07-24-01-21-16.gh-issue-46376.w-xuDL.rst +++ /dev/null @@ -1 +0,0 @@ -Prevent memory leak and use-after-free when using pointers to pointers with ctypes diff --git a/Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst b/Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst deleted file mode 100644 index 67677dd3c8ed2..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-08-03-11-31-11.gh-issue-107576.pO_s9I.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :func:`types.get_original_bases` to only return -:attr:`!__orig_bases__` if it is present on ``cls`` directly. Patch by -James Hilton-Balfe. diff --git a/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst b/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst deleted file mode 100644 index ecaf437a48e0a..0000000000000 --- a/Misc/NEWS.d/next/Library/2023-08-03-12-52-19.gh-issue-107077.-pzHD6.rst +++ /dev/null @@ -1,6 +0,0 @@ -Seems that in some conditions, OpenSSL will return ``SSL_ERROR_SYSCALL`` -instead of ``SSL_ERROR_SSL`` when a certification verification has failed, -but the error parameters will still contain ``ERR_LIB_SSL`` and -``SSL_R_CERTIFICATE_VERIFY_FAILED``. We are now detecting this situation and -raising the appropiate ``ssl.SSLCertVerificationError``. Patch by Pablo -Galindo diff --git a/Misc/NEWS.d/next/Security/2023-03-07-21-46-29.gh-issue-102509.5ouaH_.rst b/Misc/NEWS.d/next/Security/2023-03-07-21-46-29.gh-issue-102509.5ouaH_.rst deleted file mode 100644 index d1a8e8b5a8d3c..0000000000000 --- a/Misc/NEWS.d/next/Security/2023-03-07-21-46-29.gh-issue-102509.5ouaH_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Start initializing ``ob_digit`` during creation of :c:type:`PyLongObject` -objects. Patch by Illia Volochii. diff --git a/Misc/NEWS.d/next/Security/2023-06-13-20-52-24.gh-issue-102988.Kei7Vf.rst b/Misc/NEWS.d/next/Security/2023-06-13-20-52-24.gh-issue-102988.Kei7Vf.rst deleted file mode 100644 index c67ec45737b53..0000000000000 --- a/Misc/NEWS.d/next/Security/2023-06-13-20-52-24.gh-issue-102988.Kei7Vf.rst +++ /dev/null @@ -1,4 +0,0 @@ -Reverted the :mod:`email.utils` security improvement change released in -3.12beta4 that unintentionally caused :mod:`email.utils.getaddresses` to fail -to parse email addresses with a comma in the quoted name field. -See :gh:`106669`. diff --git a/Misc/NEWS.d/next/Tests/2023-07-14-16-20-06.gh-issue-106752.gd1i6D.rst b/Misc/NEWS.d/next/Tests/2023-07-14-16-20-06.gh-issue-106752.gd1i6D.rst deleted file mode 100644 index ba7257e361080..0000000000000 --- a/Misc/NEWS.d/next/Tests/2023-07-14-16-20-06.gh-issue-106752.gd1i6D.rst +++ /dev/null @@ -1,2 +0,0 @@ -Moved tests for ``zipfile.Path`` into ``Lib/test/test_zipfile/_path``. Made -``zipfile._path`` a package. diff --git a/Misc/NEWS.d/next/Tests/2023-07-16-02-57-08.gh-issue-104090.cKtK7g.rst b/Misc/NEWS.d/next/Tests/2023-07-16-02-57-08.gh-issue-104090.cKtK7g.rst deleted file mode 100644 index 5cc6c5bbe1544..0000000000000 --- a/Misc/NEWS.d/next/Tests/2023-07-16-02-57-08.gh-issue-104090.cKtK7g.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid creating a reference to the test object in :meth:`~unittest.TestResult.collectedDurations`. diff --git a/Misc/NEWS.d/next/Tests/2023-07-22-13-49-40.gh-issue-106714.btYI5S.rst b/Misc/NEWS.d/next/Tests/2023-07-22-13-49-40.gh-issue-106714.btYI5S.rst deleted file mode 100644 index 955620521c8f6..0000000000000 --- a/Misc/NEWS.d/next/Tests/2023-07-22-13-49-40.gh-issue-106714.btYI5S.rst +++ /dev/null @@ -1,3 +0,0 @@ -test_capi: Fix test_no_FatalError_infinite_loop() to no longer write a -coredump, by using test.support.SuppressCrashReport. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Tests/2023-07-25-14-36-33.gh-issue-107237.y1pY79.rst b/Misc/NEWS.d/next/Tests/2023-07-25-14-36-33.gh-issue-107237.y1pY79.rst deleted file mode 100644 index a04f7eeddef17..0000000000000 --- a/Misc/NEWS.d/next/Tests/2023-07-25-14-36-33.gh-issue-107237.y1pY79.rst +++ /dev/null @@ -1,2 +0,0 @@ -``test_logging``: Fix ``test_udp_reconnection()`` by increasing the timeout -from 100 ms to 5 minutes (LONG_TIMEOUT). Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-04-05-07-19-36.gh-issue-103186.yEozgK.rst b/Misc/NEWS.d/next/Tools-Demos/2023-04-05-07-19-36.gh-issue-103186.yEozgK.rst deleted file mode 100644 index 7e28ba6963216..0000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2023-04-05-07-19-36.gh-issue-103186.yEozgK.rst +++ /dev/null @@ -1,2 +0,0 @@ -``freeze`` now fetches ``CONFIG_ARGS`` from the original CPython instance -the Makefile uses to call utility scripts. Patch by Ijtaba Hussain. diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-07-21-23-16-05.gh-issue-106970.NLRnml.rst b/Misc/NEWS.d/next/Tools-Demos/2023-07-21-23-16-05.gh-issue-106970.NLRnml.rst deleted file mode 100644 index 194e3351b0470..0000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2023-07-21-23-16-05.gh-issue-106970.NLRnml.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix bugs in the Argument Clinic ``destination clear`` command; the -destination buffers would never be cleared, and the ``destination`` -directive parser would simply continue to the fault handler after processing -the command. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Windows/2023-07-11-20-48-17.gh-issue-99079.CIMftz.rst b/Misc/NEWS.d/next/Windows/2023-07-11-20-48-17.gh-issue-99079.CIMftz.rst deleted file mode 100644 index 11f411be0f17c..0000000000000 --- a/Misc/NEWS.d/next/Windows/2023-07-11-20-48-17.gh-issue-99079.CIMftz.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows build to use OpenSSL 3.0.9 diff --git a/Misc/NEWS.d/next/macOS/2023-07-30-23-42-20.gh-issue-99079.JAtoh1.rst b/Misc/NEWS.d/next/macOS/2023-07-30-23-42-20.gh-issue-99079.JAtoh1.rst deleted file mode 100644 index d0eef4ec1003c..0000000000000 --- a/Misc/NEWS.d/next/macOS/2023-07-30-23-42-20.gh-issue-99079.JAtoh1.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to use OpenSSL 3.0.9. diff --git a/README.rst b/README.rst index 5257beacd3185..7742de68d5a2f 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.12.0 beta 4 -===================================== +This is Python version 3.12.0 release candidate 1 +================================================= .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg :alt: CPython build status on GitHub Actions From webhook-mailer at python.org Sun Aug 6 09:14:45 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 06 Aug 2023 13:14:45 -0000 Subject: [Python-checkins] [3.12] gh-107662: Switch 'any' and 'anext' in functions.rst (GH-107663) (#107664) Message-ID: https://github.com/python/cpython/commit/6132f7099d9bb0138542d1e03728194bd066361c commit: 6132f7099d9bb0138542d1e03728194bd066361c branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-06T15:14:42+02:00 summary: [3.12] gh-107662: Switch 'any' and 'anext' in functions.rst (GH-107663) (#107664) gh-107662: Switch 'any' and 'anext' in functions.rst (GH-107663) Order was reversed in index at top, not in body. (cherry picked from commit 9ebc6ecbc336d7b17cd158d1a4522f832df3e6e2) Co-authored-by: Terry Jan Reedy files: M Doc/library/functions.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 2688f43f9b4ff..b271067ae639c 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -14,8 +14,8 @@ are always available. They are listed here in alphabetical order. | | :func:`abs` | | :func:`enumerate` | | :func:`len` | | |func-range|_ | | | :func:`aiter` | | :func:`eval` | | |func-list|_ | | :func:`repr` | | | :func:`all` | | :func:`exec` | | :func:`locals` | | :func:`reversed` | -| | :func:`any` | | | | | | :func:`round` | -| | :func:`anext` | | **F** | | **M** | | | +| | :func:`anext` | | | | | | :func:`round` | +| | :func:`any` | | **F** | | **M** | | | | | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | | | | | :func:`float` | | :func:`max` | | |func-set|_ | | | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | From webhook-mailer at python.org Sun Aug 6 09:15:43 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 06 Aug 2023 13:15:43 -0000 Subject: [Python-checkins] [3.12] Docs: skip python-docs-theme 2023.7 to fix mobile menu (GH-107666) (#107690) Message-ID: https://github.com/python/cpython/commit/cc766c0b15ed81ad5a6f0355da98a5e95aad7bfb commit: cc766c0b15ed81ad5a6f0355da98a5e95aad7bfb branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-06T15:15:39+02:00 summary: [3.12] Docs: skip python-docs-theme 2023.7 to fix mobile menu (GH-107666) (#107690) Docs: skip python-docs-theme 2023.7 to fix mobile menu (GH-107666) (cherry picked from commit 9641c4d8e2bdf9b00dd9f373d4a74dfad000afd1) Co-authored-by: Hugo van Kemenade files: M Doc/requirements.txt diff --git a/Doc/requirements.txt b/Doc/requirements.txt index fb558b3c5af24..4741265a0347f 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -14,6 +14,6 @@ sphinxext-opengraph==0.7.5 # The theme used by the documentation is stored separately, so we need # to install that as well. -python-docs-theme>=2023.7 +python-docs-theme>=2023.3.1,!=2023.7 -c constraints.txt From webhook-mailer at python.org Sun Aug 6 09:58:04 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Sun, 06 Aug 2023 13:58:04 -0000 Subject: [Python-checkins] Do not use deprecated ``logger.warn()`` in pyspecific (#107694) Message-ID: https://github.com/python/cpython/commit/9564e31cbc95a723f2414537231bc4611b56644f commit: 9564e31cbc95a723f2414537231bc4611b56644f branch: main author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: AlexWaygood date: 2023-08-06T13:58:00Z summary: Do not use deprecated ``logger.warn()`` in pyspecific (#107694) files: M Doc/tools/extensions/pyspecific.py diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 765e6383ac6f5..3cf4d236604bc 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -179,7 +179,7 @@ def parse_platforms(self): if unknown: cls = type(self) logger = logging.getLogger(cls.__qualname__) - logger.warn( + logger.warning( f"Unknown platform(s) or syntax '{' '.join(sorted(unknown))}' " f"in '.. availability:: {self.arguments[0]}', see " f"{__file__}:{cls.__qualname__}.known_platforms for a set " @@ -266,7 +266,7 @@ def run(self): info = env.all_audit_events.setdefault(name, new_info) if info is not new_info: if not self._do_args_match(info['args'], new_info['args']): - self.logger.warn( + self.logger.warning( "Mismatched arguments for audit-event {}: {!r} != {!r}" .format(name, info['args'], new_info['args']) ) @@ -542,7 +542,7 @@ def write(self, *ignored): 'building topics... ', length=len(pydoc_topic_labels)): if label not in self.env.domaindata['std']['labels']: - self.env.logger.warn('label %r not in documentation' % label) + self.env.logger.warning(f'label {label!r} not in documentation') continue docname, labelid, sectname = self.env.domaindata['std']['labels'][label] doctree = self.env.get_and_resolve_doctree(docname, self) From webhook-mailer at python.org Sun Aug 6 10:06:19 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Sun, 06 Aug 2023 14:06:19 -0000 Subject: [Python-checkins] [3.11] Do not use deprecated ``logger.warn()`` in pyspecific (GH-107694) (#107696) Message-ID: https://github.com/python/cpython/commit/2345a8fb0c8e660e2e3d63697e16433a312fce58 commit: 2345a8fb0c8e660e2e3d63697e16433a312fce58 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: AlexWaygood date: 2023-08-06T14:06:16Z summary: [3.11] Do not use deprecated ``logger.warn()`` in pyspecific (GH-107694) (#107696) Do not use deprecated ``logger.warn()`` in pyspecific (GH-107694) (cherry picked from commit 9564e31cbc95a723f2414537231bc4611b56644f) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/tools/extensions/pyspecific.py diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 9104be01b722e..f59331419e508 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -185,7 +185,7 @@ def parse_platforms(self): if unknown: cls = type(self) logger = logging.getLogger(cls.__qualname__) - logger.warn( + logger.warning( f"Unknown platform(s) or syntax '{' '.join(sorted(unknown))}' " f"in '.. availability:: {self.arguments[0]}', see " f"{__file__}:{cls.__qualname__}.known_platforms for a set " @@ -272,7 +272,7 @@ def run(self): info = env.all_audit_events.setdefault(name, new_info) if info is not new_info: if not self._do_args_match(info['args'], new_info['args']): - self.logger.warn( + self.logger.warning( "Mismatched arguments for audit-event {}: {!r} != {!r}" .format(name, info['args'], new_info['args']) ) @@ -549,7 +549,7 @@ def write(self, *ignored): 'building topics... ', length=len(pydoc_topic_labels)): if label not in self.env.domaindata['std']['labels']: - self.env.logger.warn('label %r not in documentation' % label) + self.env.logger.warning(f'label {label!r} not in documentation') continue docname, labelid, sectname = self.env.domaindata['std']['labels'][label] doctree = self.env.get_and_resolve_doctree(docname, self) From webhook-mailer at python.org Sun Aug 6 10:17:23 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 06 Aug 2023 14:17:23 -0000 Subject: [Python-checkins] [3.12] Do not use deprecated ``logger.warn()`` in pyspecific (GH-107694) (#107695) Message-ID: https://github.com/python/cpython/commit/c80112906a406d2eecf63729caad129c3f920f87 commit: c80112906a406d2eecf63729caad129c3f920f87 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-06T16:17:20+02:00 summary: [3.12] Do not use deprecated ``logger.warn()`` in pyspecific (GH-107694) (#107695) Do not use deprecated ``logger.warn()`` in pyspecific (GH-107694) (cherry picked from commit 9564e31cbc95a723f2414537231bc4611b56644f) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/tools/extensions/pyspecific.py diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 003229d8a5975..5ae742068e491 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -180,7 +180,7 @@ def parse_platforms(self): if unknown: cls = type(self) logger = logging.getLogger(cls.__qualname__) - logger.warn( + logger.warning( f"Unknown platform(s) or syntax '{' '.join(sorted(unknown))}' " f"in '.. availability:: {self.arguments[0]}', see " f"{__file__}:{cls.__qualname__}.known_platforms for a set " @@ -267,7 +267,7 @@ def run(self): info = env.all_audit_events.setdefault(name, new_info) if info is not new_info: if not self._do_args_match(info['args'], new_info['args']): - self.logger.warn( + self.logger.warning( "Mismatched arguments for audit-event {}: {!r} != {!r}" .format(name, info['args'], new_info['args']) ) @@ -544,7 +544,7 @@ def write(self, *ignored): 'building topics... ', length=len(pydoc_topic_labels)): if label not in self.env.domaindata['std']['labels']: - self.env.logger.warn('label %r not in documentation' % label) + self.env.logger.warning(f'label {label!r} not in documentation') continue docname, labelid, sectname = self.env.domaindata['std']['labels'][label] doctree = self.env.get_and_resolve_doctree(docname, self) From webhook-mailer at python.org Sun Aug 6 10:17:49 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 06 Aug 2023 14:17:49 -0000 Subject: [Python-checkins] [3.12] GH-84435: Make pyspecific directives translatable (GH-19470) (#107681) Message-ID: https://github.com/python/cpython/commit/6b02734af7849fd4da6215d7b4f8df2010e240bf commit: 6b02734af7849fd4da6215d7b4f8df2010e240bf branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-06T16:17:45+02:00 summary: [3.12] GH-84435: Make pyspecific directives translatable (GH-19470) (#107681) GH-84435: Make pyspecific directives translatable (GH-19470) (cherry picked from commit ecb05e0b9842ba03b42b4dec8767b1c18a4e28b3) Co-authored-by: cocoatomo Co-authored-by: Jelle Zijlstra Co-authored-by: Adam Turner <9087854+aa-turner at users.noreply.github.com> files: M Doc/tools/extensions/pyspecific.py diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 5ae742068e491..c286bcf34fa8b 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -98,14 +98,13 @@ class ImplementationDetail(Directive): final_argument_whitespace = True # This text is copied to templates/dummy.html - label_text = 'CPython implementation detail:' + label_text = sphinx_gettext('CPython implementation detail:') def run(self): self.assert_has_content() pnode = nodes.compound(classes=['impl-detail']) - label = sphinx_gettext(self.label_text) content = self.content - add_text = nodes.strong(label, label) + add_text = nodes.strong(self.label_text, self.label_text) self.state.nested_parse(content, self.content_offset, pnode) content = nodes.inline(pnode[0].rawsource, translatable=True) content.source = pnode[0].source @@ -234,9 +233,9 @@ class AuditEvent(Directive): final_argument_whitespace = True _label = [ - "Raises an :ref:`auditing event ` {name} with no arguments.", - "Raises an :ref:`auditing event ` {name} with argument {args}.", - "Raises an :ref:`auditing event ` {name} with arguments {args}.", + sphinx_gettext("Raises an :ref:`auditing event ` {name} with no arguments."), + sphinx_gettext("Raises an :ref:`auditing event ` {name} with argument {args}."), + sphinx_gettext("Raises an :ref:`auditing event ` {name} with arguments {args}."), ] @property @@ -252,7 +251,7 @@ def run(self): else: args = [] - label = sphinx_gettext(self._label[min(2, len(args))]) + label = self._label[min(2, len(args))] text = label.format(name="``{}``".format(name), args=", ".join("``{}``".format(a) for a in args if a)) @@ -414,8 +413,8 @@ class DeprecatedRemoved(Directive): final_argument_whitespace = True option_spec = {} - _deprecated_label = 'Deprecated since version {deprecated}, will be removed in version {removed}' - _removed_label = 'Deprecated since version {deprecated}, removed in version {removed}' + _deprecated_label = sphinx_gettext('Deprecated since version {deprecated}, will be removed in version {removed}') + _removed_label = sphinx_gettext('Deprecated since version {deprecated}, removed in version {removed}') def run(self): node = addnodes.versionmodified() @@ -431,7 +430,6 @@ def run(self): else: label = self._removed_label - label = sphinx_gettext(label) text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) if len(self.arguments) == 3: inodes, messages = self.state.inline_text(self.arguments[2], From webhook-mailer at python.org Sun Aug 6 15:40:59 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Sun, 06 Aug 2023 19:40:59 -0000 Subject: [Python-checkins] gh-106368: Improve coverage reports for argument clinic (#107693) Message-ID: https://github.com/python/cpython/commit/ee3bf45e5e253d393f8553e58715c31e470800cd commit: ee3bf45e5e253d393f8553e58715c31e470800cd branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-06T20:40:55+01:00 summary: gh-106368: Improve coverage reports for argument clinic (#107693) files: M .coveragerc M Tools/clinic/clinic.py M Tools/clinic/cpp.py diff --git a/.coveragerc b/.coveragerc index 18bf2f40fe523..b5d94317e8aa8 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,6 +7,11 @@ exclude_lines = # Don't complain if non-runnable code isn't run: if 0: if __name__ == .__main__.: + raise AssertionError\( + + # Empty bodies in protocols or abstract methods + ^\s*def [a-zA-Z0-9_]+\(.*\)(\s*->.*)?:\s*\.\.\.(\s*#.*)?$ + ^\s*\.\.\.(\s*#.*)?$ .*# pragma: no cover .*# pragma: no branch diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 423cdfb939c16..47b5f5ae32f58 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -469,7 +469,7 @@ class Language(metaclass=abc.ABCMeta): checksum_line = "" def __init__(self, filename: str) -> None: - pass + ... @abc.abstractmethod def render( @@ -477,10 +477,10 @@ def render( clinic: Clinic | None, signatures: Iterable[Module | Class | Function] ) -> str: - pass + ... def parse_line(self, line: str) -> None: - pass + ... def validate(self) -> None: def assert_only_one( @@ -2862,6 +2862,9 @@ def __getattr__(self, attr): f"Note: accessing self.function inside converter_init is disallowed!" ) return super().__getattr__(attr) + # this branch is just here for coverage reporting + else: # pragma: no cover + pass def converter_init(self) -> None: pass @@ -3990,7 +3993,7 @@ def correct_name_for_self( return "void *", "null" if f.kind in (CLASS_METHOD, METHOD_NEW): return "PyTypeObject *", "type" - raise RuntimeError("Unhandled type of function f: " + repr(f.kind)) + raise AssertionError(f"Unhandled type of function f: {f.kind!r}") def required_type_for_self_for_parser( f: Function diff --git a/Tools/clinic/cpp.py b/Tools/clinic/cpp.py index 876105120c97f..3d9c61ac67896 100644 --- a/Tools/clinic/cpp.py +++ b/Tools/clinic/cpp.py @@ -178,11 +178,17 @@ def pop_stack() -> TokenAndCondition: if self.verbose: print(self.status()) -if __name__ == '__main__': - for filename in sys.argv[1:]: + +def _main(filenames: list[str] | None = None) -> None: + filenames = filenames or sys.argv[1:] + for filename in filenames: with open(filename) as f: cpp = Monitor(filename, verbose=True) print() print(filename) for line_number, line in enumerate(f.read().split('\n'), 1): cpp.writeline(line) + + +if __name__ == '__main__': + _main() From webhook-mailer at python.org Sun Aug 6 16:08:41 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 06 Aug 2023 20:08:41 -0000 Subject: [Python-checkins] [3.12] Docs: Fix Sphinx annotations in Doc/library/ctypes.rst (GH-107672) (#107685) Message-ID: https://github.com/python/cpython/commit/e950d7045161b3dd10185400799637e5f7118b3c commit: e950d7045161b3dd10185400799637e5f7118b3c branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-06T22:08:38+02:00 summary: [3.12] Docs: Fix Sphinx annotations in Doc/library/ctypes.rst (GH-107672) (#107685) Docs: Fix Sphinx annotations in Doc/library/ctypes.rst (GH-107672) (cherry picked from commit 71a7c96ffeb0d7fef06be3e57468896e030967a5) Co-authored-by: Erlend E. Aasland Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/conf.py M Doc/library/ctypes.rst diff --git a/Doc/conf.py b/Doc/conf.py index a8fd853ebb1f0..fd115f7551f44 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -110,6 +110,8 @@ ('c:type', 'uintptr_t'), ('c:type', 'va_list'), ('c:type', 'wchar_t'), + ('c:type', '__int64'), + ('c:type', 'unsigned __int64'), # Standard C structures ('c:struct', 'in6_addr'), ('c:struct', 'in_addr'), diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index c253a45e1a8b5..ff579d980d5ed 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -72,8 +72,9 @@ Windows appends the usual ``.dll`` file suffix automatically. On Linux, it is required to specify the filename *including* the extension to load a library, so attribute access can not be used to load libraries. Either the -:meth:`LoadLibrary` method of the dll loaders should be used, or you should load -the library by creating an instance of CDLL by calling the constructor:: +:meth:`~LibraryLoader.LoadLibrary` method of the dll loaders should be used, +or you should load the library by creating an instance of CDLL by calling +the constructor:: >>> cdll.LoadLibrary("libc.so.6") # doctest: +LINUX @@ -333,7 +334,7 @@ property:: 10 b'Hi\x00lo\x00\x00\x00\x00\x00' >>> -The :func:`create_string_buffer` function replaces the old :func:`c_buffer` +The :func:`create_string_buffer` function replaces the old :func:`!c_buffer` function (which is still available as an alias). To create a mutable memory block containing unicode characters of the C type :c:type:`wchar_t`, use the :func:`create_unicode_buffer` function. @@ -383,15 +384,15 @@ as calling functions with a fixed number of parameters. On some platforms, and i particular ARM64 for Apple Platforms, the calling convention for variadic functions is different than that for regular functions. -On those platforms it is required to specify the *argtypes* attribute for the -regular, non-variadic, function arguments: +On those platforms it is required to specify the :attr:`~_FuncPtr.argtypes` +attribute for the regular, non-variadic, function arguments: .. code-block:: python3 libc.printf.argtypes = [ctypes.c_char_p] Because specifying the attribute does not inhibit portability it is advised to always -specify ``argtypes`` for all variadic functions. +specify :attr:`~_FuncPtr.argtypes` for all variadic functions. .. _ctypes-calling-functions-with-own-custom-data-types: @@ -401,7 +402,7 @@ Calling functions with your own custom data types You can also customize :mod:`ctypes` argument conversion to allow instances of your own classes be used as function arguments. :mod:`ctypes` looks for an -:attr:`_as_parameter_` attribute and uses this as the function argument. Of +:attr:`!_as_parameter_` attribute and uses this as the function argument. Of course, it must be one of integer, string, or bytes:: >>> class Bottles: @@ -414,7 +415,7 @@ course, it must be one of integer, string, or bytes:: 19 >>> -If you don't want to store the instance's data in the :attr:`_as_parameter_` +If you don't want to store the instance's data in the :attr:`!_as_parameter_` instance variable, you could define a :class:`property` which makes the attribute available on request. @@ -425,9 +426,9 @@ Specifying the required argument types (function prototypes) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ It is possible to specify the required argument types of functions exported from -DLLs by setting the :attr:`argtypes` attribute. +DLLs by setting the :attr:`~_FuncPtr.argtypes` attribute. -:attr:`argtypes` must be a sequence of C data types (the ``printf`` function is +:attr:`~_FuncPtr.argtypes` must be a sequence of C data types (the :func:`!printf` function is probably not a good example here, because it takes a variable number and different types of parameters depending on the format string, on the other hand this is quite handy to experiment with this feature):: @@ -451,14 +452,14 @@ prototype for a C function), and tries to convert the arguments to valid types:: >>> If you have defined your own classes which you pass to function calls, you have -to implement a :meth:`from_param` class method for them to be able to use them -in the :attr:`argtypes` sequence. The :meth:`from_param` class method receives +to implement a :meth:`~_CData.from_param` class method for them to be able to use them +in the :attr:`~_FuncPtr.argtypes` sequence. The :meth:`~_CData.from_param` class method receives the Python object passed to the function call, it should do a typecheck or whatever is needed to make sure this object is acceptable, and then return the -object itself, its :attr:`_as_parameter_` attribute, or whatever you want to +object itself, its :attr:`!_as_parameter_` attribute, or whatever you want to pass as the C function argument in this case. Again, the result should be an integer, string, bytes, a :mod:`ctypes` instance, or an object with an -:attr:`_as_parameter_` attribute. +:attr:`!_as_parameter_` attribute. .. _ctypes-return-types: @@ -478,13 +479,13 @@ By default functions are assumed to return the C :c:expr:`int` type. Other return types can be specified by setting the :attr:`restype` attribute of the function object. -The C prototype of ``time()`` is ``time_t time(time_t *)``. Because :c:type:`time_t` -might be of a different type than the default return type ``int``, you should -specify the ``restype``:: +The C prototype of :c:func:`time` is ``time_t time(time_t *)``. Because :c:type:`time_t` +might be of a different type than the default return type :c:expr:`int`, you should +specify the :attr:`!restype` attribute:: >>> libc.time.restype = c_time_t -The argument types can be specified using ``argtypes``:: +The argument types can be specified using :attr:`~_FuncPtr.argtypes`:: >>> libc.time.argtypes = (POINTER(c_time_t),) @@ -493,7 +494,7 @@ To call the function with a ``NULL`` pointer as first argument, use ``None``:: >>> print(libc.time(None)) # doctest: +SKIP 1150640792 -Here is a more advanced example, it uses the ``strchr`` function, which expects +Here is a more advanced example, it uses the :func:`strchr` function, which expects a string pointer and a char, and returns a pointer to a string:: >>> strchr = libc.strchr @@ -506,8 +507,8 @@ a string pointer and a char, and returns a pointer to a string:: None >>> -If you want to avoid the ``ord("x")`` calls above, you can set the -:attr:`argtypes` attribute, and the second argument will be converted from a +If you want to avoid the :func:`ord("x") ` calls above, you can set the +:attr:`~_FuncPtr.argtypes` attribute, and the second argument will be converted from a single character Python bytes object into a C char: .. doctest:: @@ -853,7 +854,7 @@ Type conversions ^^^^^^^^^^^^^^^^ Usually, ctypes does strict type checking. This means, if you have -``POINTER(c_int)`` in the :attr:`argtypes` list of a function or as the type of +``POINTER(c_int)`` in the :attr:`~_FuncPtr.argtypes` list of a function or as the type of a member field in a structure definition, only instances of exactly the same type are accepted. There are some exceptions to this rule, where ctypes accepts other objects. For example, you can pass compatible array instances instead of @@ -874,7 +875,7 @@ pointer types. So, for ``POINTER(c_int)``, ctypes accepts an array of c_int:: >>> In addition, if a function argument is explicitly declared to be a pointer type -(such as ``POINTER(c_int)``) in :attr:`argtypes`, an object of the pointed +(such as ``POINTER(c_int)``) in :attr:`_FuncPtr.argtypes`, an object of the pointed type (``c_int`` in this case) can be passed to the function. ctypes will apply the required :func:`byref` conversion in this case automatically. @@ -1437,7 +1438,7 @@ function exported by these libraries, and reacquired afterwards. All these classes can be instantiated by calling them with at least one argument, the pathname of the shared library. If you have an existing handle to an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platforms ``dlopen`` or ``LoadLibrary`` +parameter, otherwise the underlying platforms :c:func:`!dlopen` or :c:func:`LoadLibrary` function is used to load the library into the process, and to get a handle to it. @@ -1522,8 +1523,8 @@ underscore to not clash with exported function names: Shared libraries can also be loaded by using one of the prefabricated objects, which are instances of the :class:`LibraryLoader` class, either by calling the -:meth:`LoadLibrary` method, or by retrieving the library as attribute of the -loader instance. +:meth:`~LibraryLoader.LoadLibrary` method, or by retrieving the library as +attribute of the loader instance. .. class:: LibraryLoader(dlltype) @@ -1639,14 +1640,14 @@ They are instances of a private class: unspecified arguments as well. When a foreign function is called, each actual argument is passed to the - :meth:`from_param` class method of the items in the :attr:`argtypes` + :meth:`~_CData.from_param` class method of the items in the :attr:`argtypes` tuple, this method allows adapting the actual argument to an object that the foreign function accepts. For example, a :class:`c_char_p` item in the :attr:`argtypes` tuple will convert a string passed as argument into a bytes object using ctypes conversion rules. New: It is now possible to put items in argtypes which are not ctypes - types, but each item must have a :meth:`from_param` method which returns a + types, but each item must have a :meth:`~_CData.from_param` method which returns a value usable as argument (integer, string, ctypes instance). This allows defining adapters that can adapt custom objects as function parameters. @@ -1770,12 +1771,12 @@ different ways, depending on the type and number of the parameters in the call: COM methods use a special calling convention: They require a pointer to the COM interface as first argument, in addition to those parameters that - are specified in the :attr:`argtypes` tuple. + are specified in the :attr:`~_FuncPtr.argtypes` tuple. The optional *paramflags* parameter creates foreign function wrappers with much more functionality than the features described above. - *paramflags* must be a tuple of the same length as :attr:`argtypes`. + *paramflags* must be a tuple of the same length as :attr:`~_FuncPtr.argtypes`. Each item in this tuple contains further information about a parameter, it must be a tuple containing one, two, or three items. @@ -2157,8 +2158,8 @@ Data types This method adapts *obj* to a ctypes type. It is called with the actual object used in a foreign function call when the type is present in the - foreign function's :attr:`argtypes` tuple; it must return an object that - can be used as a function call parameter. + foreign function's :attr:`~_FuncPtr.argtypes` tuple; + it must return an object that can be used as a function call parameter. All ctypes data types have a default implementation of this classmethod that normally returns *obj* if that is an instance of the type. Some From webhook-mailer at python.org Sun Aug 6 16:09:04 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 06 Aug 2023 20:09:04 -0000 Subject: [Python-checkins] [3.12] Docs: Argument Clinic: Improve 'How to write a custom converter' (GH-107328) (#107669) Message-ID: https://github.com/python/cpython/commit/e94548479aaa46e0624da98f8f6b48904860ee73 commit: e94548479aaa46e0624da98f8f6b48904860ee73 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-06T22:09:00+02:00 summary: [3.12] Docs: Argument Clinic: Improve 'How to write a custom converter' (GH-107328) (#107669) Docs: Argument Clinic: Improve 'How to write a custom converter' (GH-107328) - Omit unneccesary wording and sentences - Don't mention implementation details (no digression, explanation) (cherry picked from commit 4a5b4221e381c541f3f73537b7b87580d100158b) Co-authored-by: Erlend E. Aasland Co-authored-by: Ezio Melotti files: M Doc/howto/clinic.rst diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 9c9a4f45dd0f5..dcede13a03b4a 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1343,35 +1343,37 @@ state. Example from the ``setattro`` slot method in See also :pep:`573`. +.. _clinic-howto-custom-converter: + How to write a custom converter ------------------------------- -As we hinted at in the previous section... you can write your own converters! -A converter is simply a Python class that inherits from :py:class:`!CConverter`. -The main purpose of a custom converter is if you have a parameter using -the ``O&`` format unit?parsing this parameter means calling +A converter is a Python class that inherits from :py:class:`!CConverter`. +The main purpose of a custom converter, is for parameters parsed with +the ``O&`` format unit --- parsing such a parameter means calling a :c:func:`PyArg_ParseTuple` "converter function". -Your converter class should be named ``*something*_converter``. -If the name follows this convention, then your converter class -will be automatically registered with Argument Clinic; its name -will be the name of your class with the ``_converter`` suffix -stripped off. (This is accomplished with a metaclass.) - -You shouldn't subclass :py:meth:`!CConverter.__init__`. Instead, you should -write a :py:meth:`!converter_init` function. :py:meth:`!converter_init` -always accepts a *self* parameter; after that, all additional -parameters *must* be keyword-only. Any arguments passed in to -the converter in Argument Clinic will be passed along to your -:py:meth:`!converter_init`. +Your converter class should be named :samp:`{ConverterName}_converter`. +By following this convention, your converter class will be automatically +registered with Argument Clinic, with its *converter name* being the name of +your converter class with the ``_converter`` suffix stripped off. -There are some additional members of :py:class:`!CConverter` you may wish -to specify in your subclass. Here's the current list: +Instead of subclassing :py:meth:`!CConverter.__init__`, +write a :py:meth:`!converter_init` method. +Apart for the *self* parameter, all additional :py:meth:`!converter_init` +parameters **must** be keyword-only. +Any arguments passed to the converter in Argument Clinic +will be passed along to your :py:meth:`!converter_init` method. +See :py:class:`!CConverter` for a list of members you may wish to specify in +your subclass. .. module:: clinic .. class:: CConverter + The base class for all converters. + See :ref:`clinic-howto-custom-converter` for how to subclass this class. + .. attribute:: type The C type to use for this variable. @@ -1436,16 +1438,16 @@ Here's the simplest example of a custom converter, from :source:`Modules/zlibmod [python start generated code]*/ /*[python end generated code: output=da39a3ee5e6b4b0d input=35521e4e733823c7]*/ -This block adds a converter to Argument Clinic named ``ssize_t``. Parameters -declared as ``ssize_t`` will be declared as type :c:type:`Py_ssize_t`, and will -be parsed by the ``'O&'`` format unit, which will call the -``ssize_t_converter`` converter function. ``ssize_t`` variables -automatically support default values. +This block adds a converter named ``ssize_t`` to Argument Clinic. +Parameters declared as ``ssize_t`` will be declared with type :c:type:`Py_ssize_t`, +and will be parsed by the ``'O&'`` format unit, +which will call the :c:func:`!ssize_t_converter` converter C function. +``ssize_t`` variables automatically support default values. More sophisticated custom converters can insert custom C code to handle initialization and cleanup. You can see more examples of custom converters in the CPython -source tree; grep the C files for the string :py:class:`!CConverter`. +source tree; grep the C files for the string ``CConverter``. How to write a custom return converter From webhook-mailer at python.org Sun Aug 6 17:11:20 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Sun, 06 Aug 2023 21:11:20 -0000 Subject: [Python-checkins] Improve cross-references in `runpy` docs (#107673) Message-ID: https://github.com/python/cpython/commit/4e242d1ffb2d165443fe2680f7d1fef9fecbcfc0 commit: 4e242d1ffb2d165443fe2680f7d1fef9fecbcfc0 branch: main author: Kamil Turek committer: AlexWaygood date: 2023-08-06T21:11:16Z summary: Improve cross-references in `runpy` docs (#107673) - Add links to `__main__` and `sys.path` where appropriate - Ensure each paragraph never has more than one link to the same thing, to avoid visual clutter from too many links files: M Doc/library/runpy.rst diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index 42ed8c253b802..406b080b7be30 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -39,7 +39,7 @@ The :mod:`runpy` module provides two functions: The *mod_name* argument should be an absolute module name. If the module name refers to a package rather than a normal - module, then that package is imported and the ``__main__`` submodule within + module, then that package is imported and the :mod:`__main__` submodule within that package is then executed and the resulting module globals dictionary returned. @@ -74,7 +74,7 @@ The :mod:`runpy` module provides two functions: Note that this manipulation of :mod:`sys` is not thread-safe. Other threads may see the partially initialised module, as well as the altered list of - arguments. It is recommended that the :mod:`sys` module be left alone when + arguments. It is recommended that the ``sys`` module be left alone when invoking this function from threaded code. .. seealso:: @@ -82,7 +82,7 @@ The :mod:`runpy` module provides two functions: command line. .. versionchanged:: 3.1 - Added ability to execute packages by looking for a ``__main__`` submodule. + Added ability to execute packages by looking for a :mod:`__main__` submodule. .. versionchanged:: 3.2 Added ``__cached__`` global variable (see :pep:`3147`). @@ -106,15 +106,16 @@ The :mod:`runpy` module provides two functions: Execute the code at the named filesystem location and return the resulting module globals dictionary. As with a script name supplied to the CPython command line, the supplied path may refer to a Python source file, a - compiled bytecode file or a valid sys.path entry containing a ``__main__`` - module (e.g. a zipfile containing a top-level ``__main__.py`` file). + compiled bytecode file or a valid :data:`sys.path` entry containing a + :mod:`__main__` module + (e.g. a zipfile containing a top-level ``__main__.py`` file). For a simple script, the specified code is simply executed in a fresh - module namespace. For a valid sys.path entry (typically a zipfile or + module namespace. For a valid :data:`sys.path` entry (typically a zipfile or directory), the entry is first added to the beginning of ``sys.path``. The function then looks for and executes a :mod:`__main__` module using the updated path. Note that there is no special protection against invoking - an existing :mod:`__main__` entry located elsewhere on ``sys.path`` if + an existing ``__main__`` entry located elsewhere on ``sys.path`` if there is no such module at the specified location. The optional dictionary argument *init_globals* may be used to pre-populate @@ -137,14 +138,14 @@ The :mod:`runpy` module provides two functions: supplied path, and ``__spec__``, ``__cached__``, ``__loader__`` and ``__package__`` will all be set to :const:`None`. - If the supplied path is a reference to a valid sys.path entry, then - ``__spec__`` will be set appropriately for the imported ``__main__`` + If the supplied path is a reference to a valid :data:`sys.path` entry, then + ``__spec__`` will be set appropriately for the imported :mod:`__main__` module (that is, ``__spec__.name`` will always be ``__main__``). ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be :ref:`set as normal ` based on the module spec. A number of alterations are also made to the :mod:`sys` module. Firstly, - ``sys.path`` may be altered as described above. ``sys.argv[0]`` is updated + :data:`sys.path` may be altered as described above. ``sys.argv[0]`` is updated with the value of ``path_name`` and ``sys.modules[__name__]`` is updated with a temporary module object for the module being executed. All modifications to items in :mod:`sys` are reverted before the function @@ -152,7 +153,7 @@ The :mod:`runpy` module provides two functions: Note that, unlike :func:`run_module`, the alterations made to :mod:`sys` are not optional in this function as these adjustments are essential to - allowing the execution of sys.path entries. As the thread-safety + allowing the execution of :data:`sys.path` entries. As the thread-safety limitations still apply, use of this function in threaded code should be either serialised with the import lock or delegated to a separate process. @@ -165,7 +166,7 @@ The :mod:`runpy` module provides two functions: .. versionchanged:: 3.4 Updated to take advantage of the module spec feature added by :pep:`451`. This allows ``__cached__`` to be set correctly in the - case where ``__main__`` is imported from a valid sys.path entry rather + case where ``__main__`` is imported from a valid :data:`sys.path` entry rather than being executed directly. .. versionchanged:: 3.12 From webhook-mailer at python.org Sun Aug 6 17:20:28 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Sun, 06 Aug 2023 21:20:28 -0000 Subject: [Python-checkins] [3.11] Improve cross-references in `runpy` docs (GH-107673) (#107699) Message-ID: https://github.com/python/cpython/commit/1878419ed86fdfa32f5bcc0d79139e46d91f9545 commit: 1878419ed86fdfa32f5bcc0d79139e46d91f9545 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: AlexWaygood date: 2023-08-06T21:20:24Z summary: [3.11] Improve cross-references in `runpy` docs (GH-107673) (#107699) Improve cross-references in `runpy` docs (GH-107673) - Add links to `__main__` and `sys.path` where appropriate - Ensure each paragraph never has more than one link to the same thing, to avoid visual clutter from too many links (cherry picked from commit 4e242d1ffb2d165443fe2680f7d1fef9fecbcfc0) Co-authored-by: Kamil Turek files: M Doc/library/runpy.rst diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index e161458040f89..4b37e40bfd6b8 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -39,7 +39,7 @@ The :mod:`runpy` module provides two functions: The *mod_name* argument should be an absolute module name. If the module name refers to a package rather than a normal - module, then that package is imported and the ``__main__`` submodule within + module, then that package is imported and the :mod:`__main__` submodule within that package is then executed and the resulting module globals dictionary returned. @@ -74,7 +74,7 @@ The :mod:`runpy` module provides two functions: Note that this manipulation of :mod:`sys` is not thread-safe. Other threads may see the partially initialised module, as well as the altered list of - arguments. It is recommended that the :mod:`sys` module be left alone when + arguments. It is recommended that the ``sys`` module be left alone when invoking this function from threaded code. .. seealso:: @@ -82,7 +82,7 @@ The :mod:`runpy` module provides two functions: command line. .. versionchanged:: 3.1 - Added ability to execute packages by looking for a ``__main__`` submodule. + Added ability to execute packages by looking for a :mod:`__main__` submodule. .. versionchanged:: 3.2 Added ``__cached__`` global variable (see :pep:`3147`). @@ -101,15 +101,16 @@ The :mod:`runpy` module provides two functions: Execute the code at the named filesystem location and return the resulting module globals dictionary. As with a script name supplied to the CPython command line, the supplied path may refer to a Python source file, a - compiled bytecode file or a valid sys.path entry containing a ``__main__`` - module (e.g. a zipfile containing a top-level ``__main__.py`` file). + compiled bytecode file or a valid :data:`sys.path` entry containing a + :mod:`__main__` module + (e.g. a zipfile containing a top-level ``__main__.py`` file). For a simple script, the specified code is simply executed in a fresh - module namespace. For a valid sys.path entry (typically a zipfile or + module namespace. For a valid :data:`sys.path` entry (typically a zipfile or directory), the entry is first added to the beginning of ``sys.path``. The function then looks for and executes a :mod:`__main__` module using the updated path. Note that there is no special protection against invoking - an existing :mod:`__main__` entry located elsewhere on ``sys.path`` if + an existing ``__main__`` entry located elsewhere on ``sys.path`` if there is no such module at the specified location. The optional dictionary argument *init_globals* may be used to pre-populate @@ -132,14 +133,14 @@ The :mod:`runpy` module provides two functions: supplied path, and ``__spec__``, ``__cached__``, ``__loader__`` and ``__package__`` will all be set to :const:`None`. - If the supplied path is a reference to a valid sys.path entry, then - ``__spec__`` will be set appropriately for the imported ``__main__`` + If the supplied path is a reference to a valid :data:`sys.path` entry, then + ``__spec__`` will be set appropriately for the imported :mod:`__main__` module (that is, ``__spec__.name`` will always be ``__main__``). ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be :ref:`set as normal ` based on the module spec. A number of alterations are also made to the :mod:`sys` module. Firstly, - ``sys.path`` may be altered as described above. ``sys.argv[0]`` is updated + :data:`sys.path` may be altered as described above. ``sys.argv[0]`` is updated with the value of ``path_name`` and ``sys.modules[__name__]`` is updated with a temporary module object for the module being executed. All modifications to items in :mod:`sys` are reverted before the function @@ -147,7 +148,7 @@ The :mod:`runpy` module provides two functions: Note that, unlike :func:`run_module`, the alterations made to :mod:`sys` are not optional in this function as these adjustments are essential to - allowing the execution of sys.path entries. As the thread-safety + allowing the execution of :data:`sys.path` entries. As the thread-safety limitations still apply, use of this function in threaded code should be either serialised with the import lock or delegated to a separate process. @@ -160,7 +161,7 @@ The :mod:`runpy` module provides two functions: .. versionchanged:: 3.4 Updated to take advantage of the module spec feature added by :pep:`451`. This allows ``__cached__`` to be set correctly in the - case where ``__main__`` is imported from a valid sys.path entry rather + case where ``__main__`` is imported from a valid :data:`sys.path` entry rather than being executed directly. .. seealso:: From webhook-mailer at python.org Sun Aug 6 18:48:52 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 06 Aug 2023 22:48:52 -0000 Subject: [Python-checkins] [3.12] Improve cross-references in `runpy` docs (GH-107673) (#107698) Message-ID: https://github.com/python/cpython/commit/ab1988d5a6797a35a9d81e840a73b2e988da6dcc commit: ab1988d5a6797a35a9d81e840a73b2e988da6dcc branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-07T00:48:48+02:00 summary: [3.12] Improve cross-references in `runpy` docs (GH-107673) (#107698) Improve cross-references in `runpy` docs (GH-107673) - Add links to `__main__` and `sys.path` where appropriate - Ensure each paragraph never has more than one link to the same thing, to avoid visual clutter from too many links (cherry picked from commit 4e242d1ffb2d165443fe2680f7d1fef9fecbcfc0) Co-authored-by: Kamil Turek files: M Doc/library/runpy.rst diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index 42ed8c253b802..406b080b7be30 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -39,7 +39,7 @@ The :mod:`runpy` module provides two functions: The *mod_name* argument should be an absolute module name. If the module name refers to a package rather than a normal - module, then that package is imported and the ``__main__`` submodule within + module, then that package is imported and the :mod:`__main__` submodule within that package is then executed and the resulting module globals dictionary returned. @@ -74,7 +74,7 @@ The :mod:`runpy` module provides two functions: Note that this manipulation of :mod:`sys` is not thread-safe. Other threads may see the partially initialised module, as well as the altered list of - arguments. It is recommended that the :mod:`sys` module be left alone when + arguments. It is recommended that the ``sys`` module be left alone when invoking this function from threaded code. .. seealso:: @@ -82,7 +82,7 @@ The :mod:`runpy` module provides two functions: command line. .. versionchanged:: 3.1 - Added ability to execute packages by looking for a ``__main__`` submodule. + Added ability to execute packages by looking for a :mod:`__main__` submodule. .. versionchanged:: 3.2 Added ``__cached__`` global variable (see :pep:`3147`). @@ -106,15 +106,16 @@ The :mod:`runpy` module provides two functions: Execute the code at the named filesystem location and return the resulting module globals dictionary. As with a script name supplied to the CPython command line, the supplied path may refer to a Python source file, a - compiled bytecode file or a valid sys.path entry containing a ``__main__`` - module (e.g. a zipfile containing a top-level ``__main__.py`` file). + compiled bytecode file or a valid :data:`sys.path` entry containing a + :mod:`__main__` module + (e.g. a zipfile containing a top-level ``__main__.py`` file). For a simple script, the specified code is simply executed in a fresh - module namespace. For a valid sys.path entry (typically a zipfile or + module namespace. For a valid :data:`sys.path` entry (typically a zipfile or directory), the entry is first added to the beginning of ``sys.path``. The function then looks for and executes a :mod:`__main__` module using the updated path. Note that there is no special protection against invoking - an existing :mod:`__main__` entry located elsewhere on ``sys.path`` if + an existing ``__main__`` entry located elsewhere on ``sys.path`` if there is no such module at the specified location. The optional dictionary argument *init_globals* may be used to pre-populate @@ -137,14 +138,14 @@ The :mod:`runpy` module provides two functions: supplied path, and ``__spec__``, ``__cached__``, ``__loader__`` and ``__package__`` will all be set to :const:`None`. - If the supplied path is a reference to a valid sys.path entry, then - ``__spec__`` will be set appropriately for the imported ``__main__`` + If the supplied path is a reference to a valid :data:`sys.path` entry, then + ``__spec__`` will be set appropriately for the imported :mod:`__main__` module (that is, ``__spec__.name`` will always be ``__main__``). ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be :ref:`set as normal ` based on the module spec. A number of alterations are also made to the :mod:`sys` module. Firstly, - ``sys.path`` may be altered as described above. ``sys.argv[0]`` is updated + :data:`sys.path` may be altered as described above. ``sys.argv[0]`` is updated with the value of ``path_name`` and ``sys.modules[__name__]`` is updated with a temporary module object for the module being executed. All modifications to items in :mod:`sys` are reverted before the function @@ -152,7 +153,7 @@ The :mod:`runpy` module provides two functions: Note that, unlike :func:`run_module`, the alterations made to :mod:`sys` are not optional in this function as these adjustments are essential to - allowing the execution of sys.path entries. As the thread-safety + allowing the execution of :data:`sys.path` entries. As the thread-safety limitations still apply, use of this function in threaded code should be either serialised with the import lock or delegated to a separate process. @@ -165,7 +166,7 @@ The :mod:`runpy` module provides two functions: .. versionchanged:: 3.4 Updated to take advantage of the module spec feature added by :pep:`451`. This allows ``__cached__`` to be set correctly in the - case where ``__main__`` is imported from a valid sys.path entry rather + case where ``__main__`` is imported from a valid :data:`sys.path` entry rather than being executed directly. .. versionchanged:: 3.12 From webhook-mailer at python.org Sun Aug 6 20:08:38 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Mon, 07 Aug 2023 00:08:38 -0000 Subject: [Python-checkins] Docs: Argument Clinic: Move the CConverter class to the reference (#107671) Message-ID: https://github.com/python/cpython/commit/a6675b1a597c67be972598ac8562883fabe48099 commit: a6675b1a597c67be972598ac8562883fabe48099 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-07T02:08:34+02:00 summary: Docs: Argument Clinic: Move the CConverter class to the reference (#107671) files: M Doc/howto/clinic.rst diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index dcede13a03b4a..e8e6aace350e0 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -193,6 +193,71 @@ The CLI supports the following options: The list of files to process. +.. _clinic-classes: + +Classes for extending Argument Clinic +------------------------------------- + +.. module:: clinic + +.. class:: CConverter + + The base class for all converters. + See :ref:`clinic-howto-custom-converter` for how to subclass this class. + + .. attribute:: type + + The C type to use for this variable. + :attr:`!type` should be a Python string specifying the type, + e.g. ``'int'``. + If this is a pointer type, the type string should end with ``' *'``. + + .. attribute:: default + + The Python default value for this parameter, as a Python value. + Or the magic value ``unspecified`` if there is no default. + + .. attribute:: py_default + + :attr:`!default` as it should appear in Python code, + as a string. + Or ``None`` if there is no default. + + .. attribute:: c_default + + :attr:`!default` as it should appear in C code, + as a string. + Or ``None`` if there is no default. + + .. attribute:: c_ignored_default + + The default value used to initialize the C variable when + there is no default, but not specifying a default may + result in an "uninitialized variable" warning. This can + easily happen when using option groups?although + properly written code will never actually use this value, + the variable does get passed in to the impl, and the + C compiler will complain about the "use" of the + uninitialized value. This value should always be a + non-empty string. + + .. attribute:: converter + + The name of the C converter function, as a string. + + .. attribute:: impl_by_reference + + A boolean value. If true, + Argument Clinic will add a ``&`` in front of the name of + the variable when passing it into the impl function. + + .. attribute:: parse_by_reference + + A boolean value. If true, + Argument Clinic will add a ``&`` in front of the name of + the variable when passing it into :c:func:`PyArg_ParseTuple`. + + .. _clinic-tutorial: Tutorial @@ -1348,7 +1413,7 @@ See also :pep:`573`. How to write a custom converter ------------------------------- -A converter is a Python class that inherits from :py:class:`!CConverter`. +A converter is a Python class that inherits from :py:class:`CConverter`. The main purpose of a custom converter, is for parameters parsed with the ``O&`` format unit --- parsing such a parameter means calling a :c:func:`PyArg_ParseTuple` "converter function". @@ -1360,73 +1425,13 @@ your converter class with the ``_converter`` suffix stripped off. Instead of subclassing :py:meth:`!CConverter.__init__`, write a :py:meth:`!converter_init` method. -Apart for the *self* parameter, all additional :py:meth:`!converter_init` -parameters **must** be keyword-only. +:py:meth:`!converter_init` always accepts a *self* parameter. +After *self*, all additional parameters **must** be keyword-only. Any arguments passed to the converter in Argument Clinic will be passed along to your :py:meth:`!converter_init` method. -See :py:class:`!CConverter` for a list of members you may wish to specify in +See :py:class:`CConverter` for a list of members you may wish to specify in your subclass. -.. module:: clinic - -.. class:: CConverter - - The base class for all converters. - See :ref:`clinic-howto-custom-converter` for how to subclass this class. - - .. attribute:: type - - The C type to use for this variable. - :attr:`!type` should be a Python string specifying the type, - e.g. ``'int'``. - If this is a pointer type, the type string should end with ``' *'``. - - .. attribute:: default - - The Python default value for this parameter, as a Python value. - Or the magic value ``unspecified`` if there is no default. - - .. attribute:: py_default - - :attr:`!default` as it should appear in Python code, - as a string. - Or ``None`` if there is no default. - - .. attribute:: c_default - - :attr:`!default` as it should appear in C code, - as a string. - Or ``None`` if there is no default. - - .. attribute:: c_ignored_default - - The default value used to initialize the C variable when - there is no default, but not specifying a default may - result in an "uninitialized variable" warning. This can - easily happen when using option groups?although - properly written code will never actually use this value, - the variable does get passed in to the impl, and the - C compiler will complain about the "use" of the - uninitialized value. This value should always be a - non-empty string. - - .. attribute:: converter - - The name of the C converter function, as a string. - - .. attribute:: impl_by_reference - - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into the impl function. - - .. attribute:: parse_by_reference - - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into :c:func:`PyArg_ParseTuple`. - - Here's the simplest example of a custom converter, from :source:`Modules/zlibmodule.c`:: /*[python input] From webhook-mailer at python.org Sun Aug 6 20:17:45 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Mon, 07 Aug 2023 00:17:45 -0000 Subject: [Python-checkins] [3.11] Docs: Argument Clinic: Move the CConverter class to the reference (GH-107671) (#107702) Message-ID: https://github.com/python/cpython/commit/d58c74c335bbd5904656b3fffd88fa3cdc67d92b commit: d58c74c335bbd5904656b3fffd88fa3cdc67d92b branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: erlend-aasland date: 2023-08-07T00:17:41Z summary: [3.11] Docs: Argument Clinic: Move the CConverter class to the reference (GH-107671) (#107702) (cherry picked from commit a6675b1a597c67be972598ac8562883fabe48099) Co-authored-by: Erlend E. Aasland files: M Doc/howto/clinic.rst diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 48c8f3c4e01c0..d75ba8a19cff2 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -193,6 +193,71 @@ The CLI supports the following options: The list of files to process. +.. _clinic-classes: + +Classes for extending Argument Clinic +------------------------------------- + +.. module:: clinic + +.. class:: CConverter + + The base class for all converters. + See :ref:`clinic-howto-custom-converter` for how to subclass this class. + + .. attribute:: type + + The C type to use for this variable. + :attr:`!type` should be a Python string specifying the type, + e.g. ``'int'``. + If this is a pointer type, the type string should end with ``' *'``. + + .. attribute:: default + + The Python default value for this parameter, as a Python value. + Or the magic value ``unspecified`` if there is no default. + + .. attribute:: py_default + + :attr:`!default` as it should appear in Python code, + as a string. + Or ``None`` if there is no default. + + .. attribute:: c_default + + :attr:`!default` as it should appear in C code, + as a string. + Or ``None`` if there is no default. + + .. attribute:: c_ignored_default + + The default value used to initialize the C variable when + there is no default, but not specifying a default may + result in an "uninitialized variable" warning. This can + easily happen when using option groups?although + properly written code will never actually use this value, + the variable does get passed in to the impl, and the + C compiler will complain about the "use" of the + uninitialized value. This value should always be a + non-empty string. + + .. attribute:: converter + + The name of the C converter function, as a string. + + .. attribute:: impl_by_reference + + A boolean value. If true, + Argument Clinic will add a ``&`` in front of the name of + the variable when passing it into the impl function. + + .. attribute:: parse_by_reference + + A boolean value. If true, + Argument Clinic will add a ``&`` in front of the name of + the variable when passing it into :c:func:`PyArg_ParseTuple`. + + .. _clinic-tutorial: Tutorial @@ -1343,7 +1408,7 @@ See also :pep:`573`. How to write a custom converter ------------------------------- -A converter is a Python class that inherits from :py:class:`!CConverter`. +A converter is a Python class that inherits from :py:class:`CConverter`. The main purpose of a custom converter, is for parameters parsed with the ``O&`` format unit --- parsing such a parameter means calling a :c:func:`PyArg_ParseTuple` "converter function". @@ -1355,73 +1420,13 @@ your converter class with the ``_converter`` suffix stripped off. Instead of subclassing :py:meth:`!CConverter.__init__`, write a :py:meth:`!converter_init` method. -Apart for the *self* parameter, all additional :py:meth:`!converter_init` -parameters **must** be keyword-only. +:py:meth:`!converter_init` always accepts a *self* parameter. +After *self*, all additional parameters **must** be keyword-only. Any arguments passed to the converter in Argument Clinic will be passed along to your :py:meth:`!converter_init` method. -See :py:class:`!CConverter` for a list of members you may wish to specify in +See :py:class:`CConverter` for a list of members you may wish to specify in your subclass. -.. module:: clinic - -.. class:: CConverter - - The base class for all converters. - See :ref:`clinic-howto-custom-converter` for how to subclass this class. - - .. attribute:: type - - The C type to use for this variable. - :attr:`!type` should be a Python string specifying the type, - e.g. ``'int'``. - If this is a pointer type, the type string should end with ``' *'``. - - .. attribute:: default - - The Python default value for this parameter, as a Python value. - Or the magic value ``unspecified`` if there is no default. - - .. attribute:: py_default - - :attr:`!default` as it should appear in Python code, - as a string. - Or ``None`` if there is no default. - - .. attribute:: c_default - - :attr:`!default` as it should appear in C code, - as a string. - Or ``None`` if there is no default. - - .. attribute:: c_ignored_default - - The default value used to initialize the C variable when - there is no default, but not specifying a default may - result in an "uninitialized variable" warning. This can - easily happen when using option groups?although - properly written code will never actually use this value, - the variable does get passed in to the impl, and the - C compiler will complain about the "use" of the - uninitialized value. This value should always be a - non-empty string. - - .. attribute:: converter - - The name of the C converter function, as a string. - - .. attribute:: impl_by_reference - - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into the impl function. - - .. attribute:: parse_by_reference - - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into :c:func:`PyArg_ParseTuple`. - - Here's the simplest example of a custom converter, from :source:`Modules/zlibmodule.c`:: /*[python input] From webhook-mailer at python.org Mon Aug 7 06:03:11 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 07 Aug 2023 10:03:11 -0000 Subject: [Python-checkins] [3.12] Docs: Argument Clinic: Move the CConverter class to the reference (GH-107671) (#107701) Message-ID: https://github.com/python/cpython/commit/19863c36d2c9172b5c6c930e3c54ab8d63271993 commit: 19863c36d2c9172b5c6c930e3c54ab8d63271993 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-07T12:03:07+02:00 summary: [3.12] Docs: Argument Clinic: Move the CConverter class to the reference (GH-107671) (#107701) Docs: Argument Clinic: Move the CConverter class to the reference (GH-107671) (cherry picked from commit a6675b1a597c67be972598ac8562883fabe48099) Co-authored-by: Erlend E. Aasland files: M Doc/howto/clinic.rst diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index dcede13a03b4a..e8e6aace350e0 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -193,6 +193,71 @@ The CLI supports the following options: The list of files to process. +.. _clinic-classes: + +Classes for extending Argument Clinic +------------------------------------- + +.. module:: clinic + +.. class:: CConverter + + The base class for all converters. + See :ref:`clinic-howto-custom-converter` for how to subclass this class. + + .. attribute:: type + + The C type to use for this variable. + :attr:`!type` should be a Python string specifying the type, + e.g. ``'int'``. + If this is a pointer type, the type string should end with ``' *'``. + + .. attribute:: default + + The Python default value for this parameter, as a Python value. + Or the magic value ``unspecified`` if there is no default. + + .. attribute:: py_default + + :attr:`!default` as it should appear in Python code, + as a string. + Or ``None`` if there is no default. + + .. attribute:: c_default + + :attr:`!default` as it should appear in C code, + as a string. + Or ``None`` if there is no default. + + .. attribute:: c_ignored_default + + The default value used to initialize the C variable when + there is no default, but not specifying a default may + result in an "uninitialized variable" warning. This can + easily happen when using option groups?although + properly written code will never actually use this value, + the variable does get passed in to the impl, and the + C compiler will complain about the "use" of the + uninitialized value. This value should always be a + non-empty string. + + .. attribute:: converter + + The name of the C converter function, as a string. + + .. attribute:: impl_by_reference + + A boolean value. If true, + Argument Clinic will add a ``&`` in front of the name of + the variable when passing it into the impl function. + + .. attribute:: parse_by_reference + + A boolean value. If true, + Argument Clinic will add a ``&`` in front of the name of + the variable when passing it into :c:func:`PyArg_ParseTuple`. + + .. _clinic-tutorial: Tutorial @@ -1348,7 +1413,7 @@ See also :pep:`573`. How to write a custom converter ------------------------------- -A converter is a Python class that inherits from :py:class:`!CConverter`. +A converter is a Python class that inherits from :py:class:`CConverter`. The main purpose of a custom converter, is for parameters parsed with the ``O&`` format unit --- parsing such a parameter means calling a :c:func:`PyArg_ParseTuple` "converter function". @@ -1360,73 +1425,13 @@ your converter class with the ``_converter`` suffix stripped off. Instead of subclassing :py:meth:`!CConverter.__init__`, write a :py:meth:`!converter_init` method. -Apart for the *self* parameter, all additional :py:meth:`!converter_init` -parameters **must** be keyword-only. +:py:meth:`!converter_init` always accepts a *self* parameter. +After *self*, all additional parameters **must** be keyword-only. Any arguments passed to the converter in Argument Clinic will be passed along to your :py:meth:`!converter_init` method. -See :py:class:`!CConverter` for a list of members you may wish to specify in +See :py:class:`CConverter` for a list of members you may wish to specify in your subclass. -.. module:: clinic - -.. class:: CConverter - - The base class for all converters. - See :ref:`clinic-howto-custom-converter` for how to subclass this class. - - .. attribute:: type - - The C type to use for this variable. - :attr:`!type` should be a Python string specifying the type, - e.g. ``'int'``. - If this is a pointer type, the type string should end with ``' *'``. - - .. attribute:: default - - The Python default value for this parameter, as a Python value. - Or the magic value ``unspecified`` if there is no default. - - .. attribute:: py_default - - :attr:`!default` as it should appear in Python code, - as a string. - Or ``None`` if there is no default. - - .. attribute:: c_default - - :attr:`!default` as it should appear in C code, - as a string. - Or ``None`` if there is no default. - - .. attribute:: c_ignored_default - - The default value used to initialize the C variable when - there is no default, but not specifying a default may - result in an "uninitialized variable" warning. This can - easily happen when using option groups?although - properly written code will never actually use this value, - the variable does get passed in to the impl, and the - C compiler will complain about the "use" of the - uninitialized value. This value should always be a - non-empty string. - - .. attribute:: converter - - The name of the C converter function, as a string. - - .. attribute:: impl_by_reference - - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into the impl function. - - .. attribute:: parse_by_reference - - A boolean value. If true, - Argument Clinic will add a ``&`` in front of the name of - the variable when passing it into :c:func:`PyArg_ParseTuple`. - - Here's the simplest example of a custom converter, from :source:`Modules/zlibmodule.c`:: /*[python input] From webhook-mailer at python.org Mon Aug 7 06:41:44 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Mon, 07 Aug 2023 10:41:44 -0000 Subject: [Python-checkins] gh-107442: Document all valid types for ctypes _as_parameter_ (#107443) Message-ID: https://github.com/python/cpython/commit/6925c578a0e3cbb00858e64da813a7ffe79623c4 commit: 6925c578a0e3cbb00858e64da813a7ffe79623c4 branch: main author: Tomas R committer: erlend-aasland date: 2023-08-07T12:41:39+02:00 summary: gh-107442: Document all valid types for ctypes _as_parameter_ (#107443) files: M Doc/library/ctypes.rst diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index ec4b0909181d3..fcf711efe0eb7 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -401,9 +401,10 @@ Calling functions with your own custom data types ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can also customize :mod:`ctypes` argument conversion to allow instances of -your own classes be used as function arguments. :mod:`ctypes` looks for an -:attr:`!_as_parameter_` attribute and uses this as the function argument. Of -course, it must be one of integer, string, or bytes:: +your own classes be used as function arguments. :mod:`ctypes` looks for an +:attr:`!_as_parameter_` attribute and uses this as the function argument. The +attribute must be an integer, string, bytes, a :mod:`ctypes` instance, or an +object with an :attr:`!_as_parameter_` attribute:: >>> class Bottles: ... def __init__(self, number): From webhook-mailer at python.org Mon Aug 7 07:11:42 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 07 Aug 2023 11:11:42 -0000 Subject: [Python-checkins] gh-104496: Use correct Tcl or Tk version in Tkinter tests (GH-107688) Message-ID: https://github.com/python/cpython/commit/3c8e8f3ceeae08fc43d885f5a4c65a3ee4b1a2c8 commit: 3c8e8f3ceeae08fc43d885f5a4c65a3ee4b1a2c8 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-07T14:11:39+03:00 summary: gh-104496: Use correct Tcl or Tk version in Tkinter tests (GH-107688) In future Tcl and Tk versions can be desynchronized. files: M Lib/test/test_tcl.py M Lib/test/test_tkinter/support.py M Lib/test/test_tkinter/test_images.py M Lib/test/test_tkinter/test_widgets.py M Lib/test/test_tkinter/widget_tests.py M Lib/test/test_ttk/test_style.py M Lib/test/test_ttk/test_widgets.py diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index d07b83acb1b50..ebdb58f91d3d8 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -20,14 +20,6 @@ tcl_version = tuple(map(int, _tkinter.TCL_VERSION.split('.'))) -_tk_patchlevel = None -def get_tk_patchlevel(): - global _tk_patchlevel - if _tk_patchlevel is None: - tcl = Tcl() - _tk_patchlevel = tcl.info_patchlevel() - return _tk_patchlevel - class TkinterTest(unittest.TestCase): @@ -571,7 +563,6 @@ def test_splitlist(self): (1, '2', (3.4,)) if self.wantobjects else ('1', '2', '3.4')), ] - tk_patchlevel = get_tk_patchlevel() if not self.wantobjects: expected = ('12', '\u20ac', '\xe2\x82\xac', '3.4') else: @@ -580,8 +571,8 @@ def test_splitlist(self): (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)), expected), ] - dbg_info = ('want objects? %s, Tcl version: %s, Tk patchlevel: %s' - % (self.wantobjects, tcl_version, tk_patchlevel)) + dbg_info = ('want objects? %s, Tcl version: %s, Tcl patchlevel: %s' + % (self.wantobjects, tcl_version, self.interp.info_patchlevel())) for arg, res in testcases: self.assertEqual(splitlist(arg), res, 'arg=%a, %s' % (arg, dbg_info)) diff --git a/Lib/test/test_tkinter/support.py b/Lib/test/test_tkinter/support.py index 9154ebac5c48f..10e64bf40a4af 100644 --- a/Lib/test/test_tkinter/support.py +++ b/Lib/test/test_tkinter/support.py @@ -79,28 +79,28 @@ def simulate_mouse_click(widget, x, y): import _tkinter tcl_version = tuple(map(int, _tkinter.TCL_VERSION.split('.'))) +tk_version = tuple(map(int, _tkinter.TK_VERSION.split('.'))) -def requires_tcl(*version): - if len(version) <= 2: - return unittest.skipUnless(tcl_version >= version, - 'requires Tcl version >= ' + '.'.join(map(str, version))) +def requires_tk(*version): + if len(version) <= 2 and tk_version >= version: + return lambda test: test def deco(test): @functools.wraps(test) def newtest(self): - if get_tk_patchlevel() < version: - self.skipTest('requires Tcl version >= ' + + root = getattr(self, 'root', None) + if get_tk_patchlevel(root) < version: + self.skipTest('requires Tk version >= ' + '.'.join(map(str, version))) test(self) return newtest return deco _tk_patchlevel = None -def get_tk_patchlevel(): +def get_tk_patchlevel(root): global _tk_patchlevel if _tk_patchlevel is None: - tcl = tkinter.Tcl() - _tk_patchlevel = tcl.info_patchlevel() + _tk_patchlevel = tkinter._parse_version(root.tk.globalgetvar('tk_patchLevel')) return _tk_patchlevel units = { diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index c07de867ce04b..b6ba1c22dcbac 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -2,7 +2,7 @@ import tkinter from test import support from test.support import os_helper -from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest, requires_tcl +from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest, requires_tk support.requires('gui') @@ -213,11 +213,11 @@ def test_create_from_gif_file(self): def test_create_from_gif_data(self): self.check_create_from_data('gif') - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_create_from_png_file(self): self.check_create_from_file('png') - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_create_from_png_data(self): self.check_create_from_data('png') diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index 34e67c0cbc44a..d3f942db7baf9 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -4,7 +4,7 @@ import os from test.support import requires -from test.test_tkinter.support import (requires_tcl, +from test.test_tkinter.support import (requires_tk, get_tk_patchlevel, widget_eq, AbstractDefaultRootTest) from test.test_tkinter.widget_tests import ( @@ -613,7 +613,7 @@ def test_configure_inactiveselectbackground(self): widget = self.create() self.checkColorParam(widget, 'inactiveselectbackground') - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_configure_insertunfocussed(self): widget = self.create() self.checkEnumParam(widget, 'insertunfocussed', @@ -924,7 +924,7 @@ def test_coords(self): for i in range(4): self.assertIsInstance(coords[i], float) - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_moveto(self): widget = self.create() i1 = widget.create_rectangle(1, 1, 20, 20, tags='group') @@ -969,7 +969,7 @@ def test_configure_activestyle(self): self.checkEnumParam(widget, 'activestyle', 'dotbox', 'none', 'underline') - test_configure_justify = requires_tcl(8, 6, 5)(StandardOptionsTests.test_configure_justify) + test_configure_justify = requires_tk(8, 6, 5)(StandardOptionsTests.test_configure_justify) def test_configure_listvariable(self): widget = self.create() @@ -1108,7 +1108,7 @@ def test_configure_digits(self): def test_configure_from(self): widget = self.create() - conv = float if get_tk_patchlevel() >= (8, 6, 10) else float_round + conv = float if get_tk_patchlevel(self.root) >= (8, 6, 10) else float_round self.checkFloatParam(widget, 'from', 100, 14.9, 15.1, conv=conv) def test_configure_label(self): @@ -1235,19 +1235,19 @@ def test_configure_opaqueresize(self): widget = self.create() self.checkBooleanParam(widget, 'opaqueresize') - @requires_tcl(8, 6, 5) + @requires_tk(8, 6, 5) def test_configure_proxybackground(self): widget = self.create() self.checkColorParam(widget, 'proxybackground') - @requires_tcl(8, 6, 5) + @requires_tk(8, 6, 5) def test_configure_proxyborderwidth(self): widget = self.create() self.checkPixelsParam(widget, 'proxyborderwidth', 0, 1.3, 2.9, 6, -2, '10p', conv=False) - @requires_tcl(8, 6, 5) + @requires_tk(8, 6, 5) def test_configure_proxyrelief(self): widget = self.create() self.checkReliefParam(widget, 'proxyrelief') diff --git a/Lib/test/test_tkinter/widget_tests.py b/Lib/test/test_tkinter/widget_tests.py index f60087a6e9f38..31f82f459beef 100644 --- a/Lib/test/test_tkinter/widget_tests.py +++ b/Lib/test/test_tkinter/widget_tests.py @@ -1,7 +1,7 @@ # Common tests for test_tkinter/test_widgets.py and test_ttk/test_widgets.py import tkinter -from test.test_tkinter.support import (AbstractTkTest, tcl_version, +from test.test_tkinter.support import (AbstractTkTest, tk_version, pixels_conv, tcl_obj_eq) import test.support @@ -22,7 +22,7 @@ def scaling(self): return self._scaling def _str(self, value): - if not self._stringify and self.wantobjects and tcl_version >= (8, 6): + if not self._stringify and self.wantobjects and tk_version >= (8, 6): return value if isinstance(value, tuple): return ' '.join(map(self._str, value)) @@ -156,7 +156,7 @@ def checkReliefParam(self, widget, name): 'flat', 'groove', 'raised', 'ridge', 'solid', 'sunken') errmsg='bad relief "spam": must be '\ 'flat, groove, raised, ridge, solid, or sunken' - if tcl_version < (8, 6): + if tk_version < (8, 6): errmsg = None self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) diff --git a/Lib/test/test_ttk/test_style.py b/Lib/test/test_ttk/test_style.py index 0ec95cf6b5ffc..f9c56ec235745 100644 --- a/Lib/test/test_ttk/test_style.py +++ b/Lib/test/test_ttk/test_style.py @@ -170,7 +170,7 @@ def test_map_custom_copy(self): newname = f'C.{name}' self.assertEqual(style.map(newname), {}) style.map(newname, **default) - if theme == 'alt' and name == '.' and get_tk_patchlevel() < (8, 6, 1): + if theme == 'alt' and name == '.' and get_tk_patchlevel(self.root) < (8, 6, 1): default['embossed'] = [('disabled', '1')] self.assertEqual(style.map(newname), default) for key, value in default.items(): diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py index 79d65b496abdc..fd1a748a498ac 100644 --- a/Lib/test/test_ttk/test_widgets.py +++ b/Lib/test/test_ttk/test_widgets.py @@ -5,7 +5,7 @@ import sys from test.test_ttk_textonly import MockTclObj -from test.test_tkinter.support import (AbstractTkTest, tcl_version, get_tk_patchlevel, +from test.test_tkinter.support import (AbstractTkTest, tk_version, get_tk_patchlevel, simulate_mouse_click, AbstractDefaultRootTest) from test.test_tkinter.widget_tests import (add_standard_options, AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) @@ -19,7 +19,7 @@ def test_configure_class(self): widget = self.create() self.assertEqual(widget['class'], '') errmsg='attempt to change read-only option' - if get_tk_patchlevel() < (8, 6, 0, 'beta', 3): + if get_tk_patchlevel(self.root) < (8, 6, 0, 'beta', 3): errmsg='Attempt to change read-only option' self.checkInvalidParam(widget, 'class', 'Foo', errmsg=errmsg) widget2 = self.create(class_='Foo') @@ -560,7 +560,7 @@ def test_configure_orient(self): widget = self.create() self.assertEqual(str(widget['orient']), 'vertical') errmsg='attempt to change read-only option' - if get_tk_patchlevel() < (8, 6, 0, 'beta', 3): + if get_tk_patchlevel(self.root) < (8, 6, 0, 'beta', 3): errmsg='Attempt to change read-only option' self.checkInvalidParam(widget, 'orient', 'horizontal', errmsg=errmsg) @@ -1526,7 +1526,7 @@ def test_heading(self): def test_heading_callback(self): def simulate_heading_click(x, y): - if tcl_version >= (8, 6): + if tk_version >= (8, 6): self.assertEqual(self.tv.identify_column(x), '#0') self.assertEqual(self.tv.identify_region(x, y), 'heading') simulate_mouse_click(self.tv, x, y) From webhook-mailer at python.org Mon Aug 7 07:28:12 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Mon, 07 Aug 2023 11:28:12 -0000 Subject: [Python-checkins] gh-95065: Add Argument Clinic support for deprecating positional use of parameters (#95151) Message-ID: https://github.com/python/cpython/commit/33cb0b06efe33968eb32463fa1b02b5a729a17f8 commit: 33cb0b06efe33968eb32463fa1b02b5a729a17f8 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-07T11:28:08Z summary: gh-95065: Add Argument Clinic support for deprecating positional use of parameters (#95151) It is now possible to deprecate passing parameters positionally with Argument Clinic, using the new '* [from X.Y]' syntax. (To be read as "keyword-only from Python version X.Y") Co-authored-by: Alex Waygood Co-authored-by: Serhiy Storchaka files: A Misc/NEWS.d/next/Tools-Demos/2022-07-23-00-33-28.gh-issue-95065.NfCCpp.rst M Doc/howto/clinic.rst M Lib/test/clinic.test.c M Lib/test/test_clinic.py M Tools/clinic/clinic.py diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index e8e6aace350e0..286623c241014 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1898,3 +1898,91 @@ blocks embedded in Python files look slightly different. They look like this: #[python start generated code]*/ def foo(): pass #/*[python checksum:...]*/ + + +.. _clinic-howto-deprecate-positional: + +How to deprecate passing parameters positionally +------------------------------------------------ + +Argument Clinic provides syntax that makes it possible to generate code that +deprecates passing :term:`arguments ` positionally. +For example, say we've got a module-level function :py:func:`!foo.myfunc` +that has three :term:`parameters `: +positional-or-keyword parameters *a* and *b*, and a keyword-only parameter *c*:: + + /*[clinic input] + module foo + myfunc + a: int + b: int + * + c: int + [clinic start generated output]*/ + +We now want to make the *b* parameter keyword-only; +however, we'll have to wait two releases before making this change, +as mandated by Python's backwards-compatibility policy (see :pep:`387`). +For this example, imagine we're in the development phase for Python 3.12: +that means we'll be allowed to introduce deprecation warnings in Python 3.12 +whenever the *b* parameter is passed positionally, +and we'll be allowed to make it keyword-only in Python 3.14 at the earliest. + +We can use Argument Clinic to emit the desired deprecation warnings +using the ``* [from ...]``` syntax, +by adding the line ``* [from 3.14]`` right above the *b* parameter:: + + /*[clinic input] + module foo + myfunc + a: int + * [from 3.14] + b: int + * + c: int + [clinic start generated output]*/ + +Next, regenerate Argument Clinic code (``make clinic``), +and add unit tests for the new behaviour. + +The generated code will now emit a :exc:`DeprecationWarning` +when an :term:`argument` for the :term:`parameter` *b* is passed positionally. +C preprocessor directives are also generated for emitting +compiler warnings if the ``* [from ...]`` line has not been removed +from the Argument Clinic input when the deprecation period is over, +which means when the alpha phase of the specified Python version kicks in. + +Let's return to our example and skip ahead two years: +Python 3.14 development has now entered the alpha phase, +but we forgot all about updating the Argument Clinic code +for :py:func:`!myfunc`! +Luckily for us, compiler warnings are now generated: + +.. code-block:: none + + In file included from Modules/foomodule.c:139: + Modules/clinic/foomodule.c.h:83:8: warning: Update 'b' in 'myfunc' in 'foomodule.c' to be keyword-only. [-W#warnings] + # warning "Update 'b' in 'myfunc' in 'foomodule.c' to be keyword-only." + ^ + +We now close the deprecation phase by making *b* keyword-only; +replace the ``* [from ...]``` line above *b* +with the ``*`` from the line above *c*:: + + /*[clinic input] + module foo + myfunc + a: int + * + b: int + c: int + [clinic start generated output]*/ + +Finally, run ``make clinic`` to regenerate the Argument Clinic code, +and update your unit tests to reflect the new behaviour. + +.. note:: + + If you forget to update your input block during the alpha and beta phases, + the compiler warning will turn into a compiler error when the + release candidate phase begins. diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index d2ad1a0482c30..321ac69273189 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5380,6 +5380,7 @@ static PyObject * fn_with_default_binop_expr_impl(PyObject *module, PyObject *arg) /*[clinic end generated code: output=018672772e4092ff input=1b55c8ae68d89453]*/ + /*[python input] class Custom_converter(CConverter): type = "str" @@ -5464,3 +5465,812 @@ docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py static PyObject * docstr_fallback_to_converter_default_impl(PyObject *module, str a) /*[clinic end generated code: output=ae24a9c6f60ee8a6 input=0cbe6a4d24bc2274]*/ + + +/*[clinic input] +test_deprecate_positional_pos1_len1_optional + a: object + * [from 3.14] + b: object = None +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos1_len1_optional__doc__, +"test_deprecate_positional_pos1_len1_optional($module, /, a, b=None)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS1_LEN1_OPTIONAL_METHODDEF \ + {"test_deprecate_positional_pos1_len1_optional", _PyCFunction_CAST(test_deprecate_positional_pos1_len1_optional), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len1_optional__doc__}, + +static PyObject * +test_deprecate_positional_pos1_len1_optional_impl(PyObject *module, + PyObject *a, PyObject *b); + +static PyObject * +test_deprecate_positional_pos1_len1_optional(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos1_len1_optional", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1_optional' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1_optional' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1_optional' to be keyword-only." + # endif + #endif + if (nargs == 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing 2 positional arguments to test_deprecate_positional_pos1_len1_optional() is deprecated. Parameter 'b' will become a keyword-only parameter in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + b = args[1]; +skip_optional_pos: + return_value = test_deprecate_positional_pos1_len1_optional_impl(module, a, b); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos1_len1_optional_impl(PyObject *module, + PyObject *a, PyObject *b) +/*[clinic end generated code: output=20bdea6a2960ddf3 input=89099f3dacd757da]*/ + + +/*[clinic input] +test_deprecate_positional_pos1_len1 + a: object + * [from 3.14] + b: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos1_len1__doc__, +"test_deprecate_positional_pos1_len1($module, /, a, b)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS1_LEN1_METHODDEF \ + {"test_deprecate_positional_pos1_len1", _PyCFunction_CAST(test_deprecate_positional_pos1_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len1__doc__}, + +static PyObject * +test_deprecate_positional_pos1_len1_impl(PyObject *module, PyObject *a, + PyObject *b); + +static PyObject * +test_deprecate_positional_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos1_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1' to be keyword-only." + # endif + #endif + if (nargs == 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing 2 positional arguments to test_deprecate_positional_pos1_len1() is deprecated. Parameter 'b' will become a keyword-only parameter in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = test_deprecate_positional_pos1_len1_impl(module, a, b); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos1_len1_impl(PyObject *module, PyObject *a, + PyObject *b) +/*[clinic end generated code: output=22c70f8b36085758 input=1702bbab1e9b3b99]*/ + + +/*[clinic input] +test_deprecate_positional_pos1_len2_with_kwd + a: object + * [from 3.14] + b: object + c: object + * + d: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos1_len2_with_kwd__doc__, +"test_deprecate_positional_pos1_len2_with_kwd($module, /, a, b, c, *, d)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS1_LEN2_WITH_KWD_METHODDEF \ + {"test_deprecate_positional_pos1_len2_with_kwd", _PyCFunction_CAST(test_deprecate_positional_pos1_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len2_with_kwd__doc__}, + +static PyObject * +test_deprecate_positional_pos1_len2_with_kwd_impl(PyObject *module, + PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +test_deprecate_positional_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos1_len2_with_kwd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos1_len2_with_kwd' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos1_len2_with_kwd' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos1_len2_with_kwd' to be keyword-only." + # endif + #endif + if (nargs > 1 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 1 positional argument to test_deprecate_positional_pos1_len2_with_kwd() is deprecated. Parameters 'b' and 'c' will become keyword-only parameters in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = test_deprecate_positional_pos1_len2_with_kwd_impl(module, a, b, c, d); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos1_len2_with_kwd_impl(PyObject *module, + PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=79c5f04220a1f3aa input=28cdb885f6c34eab]*/ + + +/*[clinic input] +test_deprecate_positional_pos0_len1 + * [from 3.14] + a: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos0_len1__doc__, +"test_deprecate_positional_pos0_len1($module, /, a)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS0_LEN1_METHODDEF \ + {"test_deprecate_positional_pos0_len1", _PyCFunction_CAST(test_deprecate_positional_pos0_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len1__doc__}, + +static PyObject * +test_deprecate_positional_pos0_len1_impl(PyObject *module, PyObject *a); + +static PyObject * +test_deprecate_positional_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos0_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *a; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'a' in the clinic input of 'test_deprecate_positional_pos0_len1' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'a' in the clinic input of 'test_deprecate_positional_pos0_len1' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'a' in the clinic input of 'test_deprecate_positional_pos0_len1' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to test_deprecate_positional_pos0_len1() is deprecated. Parameter 'a' will become a keyword-only parameter in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + return_value = test_deprecate_positional_pos0_len1_impl(module, a); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos0_len1_impl(PyObject *module, PyObject *a) +/*[clinic end generated code: output=1b7f23b9ffca431b input=678206db25c0652c]*/ + + +/*[clinic input] +test_deprecate_positional_pos0_len2 + * [from 3.14] + a: object + b: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos0_len2__doc__, +"test_deprecate_positional_pos0_len2($module, /, a, b)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS0_LEN2_METHODDEF \ + {"test_deprecate_positional_pos0_len2", _PyCFunction_CAST(test_deprecate_positional_pos0_len2), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len2__doc__}, + +static PyObject * +test_deprecate_positional_pos0_len2_impl(PyObject *module, PyObject *a, + PyObject *b); + +static PyObject * +test_deprecate_positional_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos0_len2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic input of 'test_deprecate_positional_pos0_len2' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic input of 'test_deprecate_positional_pos0_len2' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic input of 'test_deprecate_positional_pos0_len2' to be keyword-only." + # endif + #endif + if (nargs > 0 && nargs <= 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to test_deprecate_positional_pos0_len2() is deprecated. Parameters 'a' and 'b' will become keyword-only parameters in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = test_deprecate_positional_pos0_len2_impl(module, a, b); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos0_len2_impl(PyObject *module, PyObject *a, + PyObject *b) +/*[clinic end generated code: output=31b494f2dcc016af input=fae0d0b1d480c939]*/ + + +/*[clinic input] +test_deprecate_positional_pos0_len3_with_kwdonly + * [from 3.14] + a: object + b: object + c: object + * + e: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos0_len3_with_kwdonly__doc__, +"test_deprecate_positional_pos0_len3_with_kwdonly($module, /, a, b, c,\n" +" *, e)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS0_LEN3_WITH_KWDONLY_METHODDEF \ + {"test_deprecate_positional_pos0_len3_with_kwdonly", _PyCFunction_CAST(test_deprecate_positional_pos0_len3_with_kwdonly), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len3_with_kwdonly__doc__}, + +static PyObject * +test_deprecate_positional_pos0_len3_with_kwdonly_impl(PyObject *module, + PyObject *a, + PyObject *b, + PyObject *c, + PyObject *e); + +static PyObject * +test_deprecate_positional_pos0_len3_with_kwdonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(e), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "e", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos0_len3_with_kwdonly", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *e; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos0_len3_with_kwdonly' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos0_len3_with_kwdonly' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos0_len3_with_kwdonly' to be keyword-only." + # endif + #endif + if (nargs > 0 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to test_deprecate_positional_pos0_len3_with_kwdonly() is deprecated. Parameters 'a', 'b' and 'c' will become keyword-only parameters in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + e = args[3]; + return_value = test_deprecate_positional_pos0_len3_with_kwdonly_impl(module, a, b, c, e); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos0_len3_with_kwdonly_impl(PyObject *module, + PyObject *a, + PyObject *b, + PyObject *c, + PyObject *e) +/*[clinic end generated code: output=96978e786acfbc7b input=1b0121770c0c52e0]*/ + + +/*[clinic input] +test_deprecate_positional_pos2_len1 + a: object + b: object + * [from 3.14] + c: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos2_len1__doc__, +"test_deprecate_positional_pos2_len1($module, /, a, b, c)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS2_LEN1_METHODDEF \ + {"test_deprecate_positional_pos2_len1", _PyCFunction_CAST(test_deprecate_positional_pos2_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len1__doc__}, + +static PyObject * +test_deprecate_positional_pos2_len1_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c); + +static PyObject * +test_deprecate_positional_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos2_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *c; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'c' in the clinic input of 'test_deprecate_positional_pos2_len1' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'c' in the clinic input of 'test_deprecate_positional_pos2_len1' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'c' in the clinic input of 'test_deprecate_positional_pos2_len1' to be keyword-only." + # endif + #endif + if (nargs == 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing 3 positional arguments to test_deprecate_positional_pos2_len1() is deprecated. Parameter 'c' will become a keyword-only parameter in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + return_value = test_deprecate_positional_pos2_len1_impl(module, a, b, c); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos2_len1_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c) +/*[clinic end generated code: output=ceadd05f11f7f491 input=e1d129689e69ec7c]*/ + + +/*[clinic input] +test_deprecate_positional_pos2_len2 + a: object + b: object + * [from 3.14] + c: object + d: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos2_len2__doc__, +"test_deprecate_positional_pos2_len2($module, /, a, b, c, d)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS2_LEN2_METHODDEF \ + {"test_deprecate_positional_pos2_len2", _PyCFunction_CAST(test_deprecate_positional_pos2_len2), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len2__doc__}, + +static PyObject * +test_deprecate_positional_pos2_len2_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, + PyObject *d); + +static PyObject * +test_deprecate_positional_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos2_len2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len2' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len2' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len2' to be keyword-only." + # endif + #endif + if (nargs > 2 && nargs <= 4) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 2 positional arguments to test_deprecate_positional_pos2_len2() is deprecated. Parameters 'c' and 'd' will become keyword-only parameters in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = test_deprecate_positional_pos2_len2_impl(module, a, b, c, d); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos2_len2_impl(PyObject *module, PyObject *a, + PyObject *b, PyObject *c, + PyObject *d) +/*[clinic end generated code: output=5693682e3fa1188b input=0d53533463a12792]*/ + + +/*[clinic input] +test_deprecate_positional_pos2_len3_with_kwdonly + a: object + b: object + * [from 3.14] + c: object + d: object + * + e: object +[clinic start generated code]*/ + +PyDoc_STRVAR(test_deprecate_positional_pos2_len3_with_kwdonly__doc__, +"test_deprecate_positional_pos2_len3_with_kwdonly($module, /, a, b, c,\n" +" d, *, e)\n" +"--\n" +"\n"); + +#define TEST_DEPRECATE_POSITIONAL_POS2_LEN3_WITH_KWDONLY_METHODDEF \ + {"test_deprecate_positional_pos2_len3_with_kwdonly", _PyCFunction_CAST(test_deprecate_positional_pos2_len3_with_kwdonly), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len3_with_kwdonly__doc__}, + +static PyObject * +test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, + PyObject *a, + PyObject *b, + PyObject *c, + PyObject *d, + PyObject *e); + +static PyObject * +test_deprecate_positional_pos2_len3_with_kwdonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 5 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", "e", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "test_deprecate_positional_pos2_len3_with_kwdonly", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[5]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + PyObject *e; + + #if PY_VERSION_HEX >= 0x030e00C0 + # error "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ("In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to be keyword-only.") + # else + # warning "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to be keyword-only." + # endif + #endif + if (nargs > 2 && nargs <= 4) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 2 positional arguments to test_deprecate_positional_pos2_len3_with_kwdonly() is deprecated. Parameters 'c' and 'd' will become keyword-only parameters in Python 3.14.", 1)) { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + e = args[4]; + return_value = test_deprecate_positional_pos2_len3_with_kwdonly_impl(module, a, b, c, d, e); + +exit: + return return_value; +} + +static PyObject * +test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, + PyObject *a, + PyObject *b, + PyObject *c, + PyObject *d, + PyObject *e) +/*[clinic end generated code: output=00d436de747a00f3 input=154fd450448d8935]*/ diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index f30fad2126940..f594e39a90546 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1478,11 +1478,105 @@ def test_parameters_required_after_star(self): "module foo\nfoo.bar\n this: int\n *", "module foo\nfoo.bar\n this: int\n *\nDocstring.", ) - err = "Function 'bar' specifies '*' without any parameters afterwards." + err = "Function 'foo.bar' specifies '*' without any parameters afterwards." for block in dataset: with self.subTest(block=block): self.expect_failure(block, err) + def test_parameters_required_after_depr_star(self): + dataset = ( + "module foo\nfoo.bar\n * [from 3.14]", + "module foo\nfoo.bar\n * [from 3.14]\nDocstring here.", + "module foo\nfoo.bar\n this: int\n * [from 3.14]", + "module foo\nfoo.bar\n this: int\n * [from 3.14]\nDocstring.", + ) + err = "Function 'foo.bar' specifies '* [from 3.14]' without any parameters afterwards." + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err) + + def test_depr_star_invalid_format_1(self): + block = """ + module foo + foo.bar + this: int + * [from 3] + Docstring. + """ + err = ( + "Function 'foo.bar': expected format '* [from major.minor]' " + "where 'major' and 'minor' are integers; got '3'" + ) + self.expect_failure(block, err, lineno=3) + + def test_depr_star_invalid_format_2(self): + block = """ + module foo + foo.bar + this: int + * [from a.b] + Docstring. + """ + err = ( + "Function 'foo.bar': expected format '* [from major.minor]' " + "where 'major' and 'minor' are integers; got 'a.b'" + ) + self.expect_failure(block, err, lineno=3) + + def test_depr_star_invalid_format_3(self): + block = """ + module foo + foo.bar + this: int + * [from 1.2.3] + Docstring. + """ + err = ( + "Function 'foo.bar': expected format '* [from major.minor]' " + "where 'major' and 'minor' are integers; got '1.2.3'" + ) + self.expect_failure(block, err, lineno=3) + + def test_parameters_required_after_depr_star(self): + block = """ + module foo + foo.bar + this: int + * [from 3.14] + Docstring. + """ + err = ( + "Function 'foo.bar' specifies '* [from ...]' without " + "any parameters afterwards" + ) + self.expect_failure(block, err, lineno=4) + + def test_depr_star_must_come_before_star(self): + block = """ + module foo + foo.bar + this: int + * + * [from 3.14] + Docstring. + """ + err = "Function 'foo.bar': '* [from ...]' must come before '*'" + self.expect_failure(block, err, lineno=4) + + def test_depr_star_duplicate(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + b: int + * [from 3.14] + c: int + Docstring. + """ + err = "Function 'foo.bar' uses '[from ...]' more than once" + self.expect_failure(block, err, lineno=5) + def test_single_slash(self): block = """ module foo diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-07-23-00-33-28.gh-issue-95065.NfCCpp.rst b/Misc/NEWS.d/next/Tools-Demos/2022-07-23-00-33-28.gh-issue-95065.NfCCpp.rst new file mode 100644 index 0000000000000..3641716769cd5 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-07-23-00-33-28.gh-issue-95065.NfCCpp.rst @@ -0,0 +1,6 @@ +It is now possible to deprecate passing parameters positionally with +Argument Clinic, using the new ``* [from X.Y]`` syntax. +(To be read as *"keyword-only from Python version X.Y"*.) +See :ref:`clinic-howto-deprecate-positional` for more information. +Patch by Erlend E. Aasland with help from Alex Waygood, +Nikita Sobolev, and Serhiy Storchaka. diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 47b5f5ae32f58..4dfe90b314f54 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -347,6 +347,13 @@ def suffix_all_lines(s: str, suffix: str) -> str: return ''.join(final) +def pprint_words(items: list[str]) -> str: + if len(items) <= 2: + return " and ".join(items) + else: + return ", ".join(items[:-1]) + " and " + items[-1] + + def version_splitter(s: str) -> tuple[int, ...]: """Splits a version string into a tuple of integers. @@ -828,6 +835,22 @@ class CLanguage(Language): #define {methoddef_name} #endif /* !defined({methoddef_name}) */ """) + DEPRECATED_POSITIONAL_PROTOTYPE: Final[str] = r""" + #if PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00C0 + # error "{cpp_message}" + #elif PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00A0 + # ifdef _MSC_VER + # pragma message ("{cpp_message}") + # else + # warning "{cpp_message}" + # endif + #endif + if ({condition}) {{{{ + if (PyErr_WarnEx(PyExc_DeprecationWarning, "{depr_message}", 1)) {{{{ + goto exit; + }}}} + }}}} + """ def __init__(self, filename: str) -> None: super().__init__(filename) @@ -850,6 +873,64 @@ def render( function = o return self.render_function(clinic, function) + def deprecate_positional_use( + self, + func: Function, + params: dict[int, Parameter], + ) -> str: + assert len(params) > 0 + names = [repr(p.name) for p in params.values()] + first_pos, first_param = next(iter(params.items())) + last_pos, last_param = next(reversed(params.items())) + + # Pretty-print list of names. + pstr = pprint_words(names) + + # For now, assume there's only one deprecation level. + assert first_param.deprecated_positional == last_param.deprecated_positional + thenceforth = first_param.deprecated_positional + assert thenceforth is not None + + # Format the preprocessor warning and error messages. + assert isinstance(self.cpp.filename, str) + source = os.path.basename(self.cpp.filename) + major, minor = thenceforth + cpp_message = ( + f"In {source}, update parameter(s) {pstr} in the clinic " + f"input of {func.full_name!r} to be keyword-only." + ) + # Format the deprecation message. + if first_pos == 0: + preamble = "Passing positional arguments to " + if len(params) == 1: + condition = f"nargs == {first_pos+1}" + if first_pos: + preamble = f"Passing {first_pos+1} positional arguments to " + depr_message = preamble + ( + f"{func.full_name}() is deprecated. Parameter {pstr} will " + f"become a keyword-only parameter in Python {major}.{minor}." + ) + else: + condition = f"nargs > {first_pos} && nargs <= {last_pos+1}" + if first_pos: + preamble = ( + f"Passing more than {first_pos} positional " + f"argument{'s' if first_pos != 1 else ''} to " + ) + depr_message = preamble + ( + f"{func.full_name}() is deprecated. Parameters {pstr} will " + f"become keyword-only parameters in Python {major}.{minor}." + ) + # Format and return the code block. + code = self.DEPRECATED_POSITIONAL_PROTOTYPE.format( + condition=condition, + major=major, + minor=minor, + cpp_message=cpp_message, + depr_message=depr_message, + ) + return normalize_snippet(code, indent=4) + def docstring_for_c_string( self, f: Function @@ -1199,6 +1280,7 @@ def parser_body( flags = 'METH_METHOD|' + flags parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS + deprecated_positionals: dict[int, Parameter] = {} add_label: str | None = None for i, p in enumerate(parameters): if isinstance(p.converter, defining_class_converter): @@ -1213,6 +1295,8 @@ def parser_body( parser_code.append("%s:" % add_label) add_label = None if not p.is_optional(): + if p.deprecated_positional: + deprecated_positionals[i] = p parser_code.append(normalize_snippet(parsearg, indent=4)) elif i < pos_only: add_label = 'skip_optional_posonly' @@ -1242,6 +1326,8 @@ def parser_body( goto %s; }} """ % add_label, indent=4)) + if p.deprecated_positional: + deprecated_positionals[i] = p if i + 1 == len(parameters): parser_code.append(normalize_snippet(parsearg, indent=4)) else: @@ -1257,6 +1343,12 @@ def parser_body( }} """ % add_label, indent=4)) + if deprecated_positionals: + code = self.deprecate_positional_use(f, deprecated_positionals) + assert parser_code is not None + # Insert the deprecation code before parameter parsing. + parser_code.insert(0, code) + if parser_code is not None: if add_label: parser_code.append("%s:" % add_label) @@ -2592,6 +2684,9 @@ def copy(self, **overrides: Any) -> Function: return f +VersionTuple = tuple[int, int] + + @dc.dataclass(repr=False, slots=True) class Parameter: """ @@ -2606,6 +2701,8 @@ class Parameter: annotation: object = inspect.Parameter.empty docstring: str = '' group: int = 0 + # (`None` signifies that there is no deprecation) + deprecated_positional: VersionTuple | None = None right_bracket_count: int = dc.field(init=False, default=0) def __repr__(self) -> str: @@ -4430,6 +4527,7 @@ class DSLParser: state: StateKeeper keyword_only: bool positional_only: bool + deprecated_positional: VersionTuple | None group: int parameter_state: ParamState indent: IndentStack @@ -4437,6 +4535,11 @@ class DSLParser: coexist: bool parameter_continuation: str preserve_output: bool + star_from_version_re = create_regex( + before="* [from ", + after="]", + word=False, + ) def __init__(self, clinic: Clinic) -> None: self.clinic = clinic @@ -4460,6 +4563,7 @@ def reset(self) -> None: self.state = self.state_dsl_start self.keyword_only = False self.positional_only = False + self.deprecated_positional = None self.group = 0 self.parameter_state: ParamState = ParamState.START self.indent = IndentStack() @@ -4622,7 +4726,7 @@ def parse(self, block: Block) -> None: exc.lineno = line_number raise - self.do_post_block_processing_cleanup() + self.do_post_block_processing_cleanup(line_number) block.output.extend(self.clinic.language.render(self.clinic, block.signatures)) if self.preserve_output: @@ -4908,8 +5012,14 @@ def state_parameter(self, line: str) -> None: self.parameter_continuation = line[:-1] return + line = line.lstrip() + match = self.star_from_version_re.match(line) + if match: + self.parse_deprecated_positional(match.group(1)) + return + func = self.function - match line.lstrip(): + match line: case '*': self.parse_star(func) case '[': @@ -5182,7 +5292,9 @@ def bad_node(self, node: ast.AST) -> None: "after 'self'.") - p = Parameter(parameter_name, kind, function=self.function, converter=converter, default=value, group=self.group) + p = Parameter(parameter_name, kind, function=self.function, + converter=converter, default=value, group=self.group, + deprecated_positional=self.deprecated_positional) names = [k.name for k in self.function.parameters.values()] if parameter_name in names[1:]: @@ -5215,10 +5327,28 @@ def parse_converter( "Annotations must be either a name, a function call, or a string." ) + def parse_deprecated_positional(self, thenceforth: str) -> None: + assert isinstance(self.function, Function) + fname = self.function.full_name + + if self.keyword_only: + fail(f"Function {fname!r}: '* [from ...]' must come before '*'") + if self.deprecated_positional: + fail(f"Function {fname!r} uses '[from ...]' more than once.") + try: + major, minor = thenceforth.split(".") + self.deprecated_positional = int(major), int(minor) + except ValueError: + fail( + f"Function {fname!r}: expected format '* [from major.minor]' " + f"where 'major' and 'minor' are integers; got {thenceforth!r}" + ) + def parse_star(self, function: Function) -> None: """Parse keyword-only parameter marker '*'.""" if self.keyword_only: fail(f"Function {function.name!r} uses '*' more than once.") + self.deprecated_positional = None self.keyword_only = True def parse_opening_square_bracket(self, function: Function) -> None: @@ -5586,23 +5716,34 @@ def add_parameter(text: str) -> None: return docstring - def do_post_block_processing_cleanup(self) -> None: + def do_post_block_processing_cleanup(self, lineno: int) -> None: """ Called when processing the block is done. """ if not self.function: return - if self.keyword_only: - values = self.function.parameters.values() - if not values: - no_parameter_after_star = True + def check_remaining( + symbol: str, + condition: Callable[[Parameter], bool] + ) -> None: + assert isinstance(self.function, Function) + + if values := self.function.parameters.values(): + last_param = next(reversed(values)) + no_param_after_symbol = condition(last_param) else: - last_parameter = next(reversed(list(values))) - no_parameter_after_star = last_parameter.kind != inspect.Parameter.KEYWORD_ONLY - if no_parameter_after_star: - fail(f"Function {self.function.name!r} specifies '*' " - "without any parameters afterwards.") + no_param_after_symbol = True + if no_param_after_symbol: + fname = self.function.full_name + fail(f"Function {fname!r} specifies {symbol!r} " + "without any parameters afterwards.", line_number=lineno) + + if self.keyword_only: + check_remaining("*", lambda p: p.kind != inspect.Parameter.KEYWORD_ONLY) + + if self.deprecated_positional: + check_remaining("* [from ...]", lambda p: not p.deprecated_positional) self.function.docstring = self.format_docstring() From webhook-mailer at python.org Mon Aug 7 08:33:29 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 07 Aug 2023 12:33:29 -0000 Subject: [Python-checkins] [3.12] gh-107442: Document all valid types for ctypes _as_parameter_ (GH-107443) (#107707) Message-ID: https://github.com/python/cpython/commit/e5582bdbcf56033a445c7a96eca8503968771b33 commit: e5582bdbcf56033a445c7a96eca8503968771b33 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-07T14:33:25+02:00 summary: [3.12] gh-107442: Document all valid types for ctypes _as_parameter_ (GH-107443) (#107707) gh-107442: Document all valid types for ctypes _as_parameter_ (GH-107443) (cherry picked from commit 6925c578a0e3cbb00858e64da813a7ffe79623c4) Co-authored-by: Tomas R files: M Doc/library/ctypes.rst diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index ff579d980d5ed..3c794beca32c1 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -401,9 +401,10 @@ Calling functions with your own custom data types ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can also customize :mod:`ctypes` argument conversion to allow instances of -your own classes be used as function arguments. :mod:`ctypes` looks for an -:attr:`!_as_parameter_` attribute and uses this as the function argument. Of -course, it must be one of integer, string, or bytes:: +your own classes be used as function arguments. :mod:`ctypes` looks for an +:attr:`!_as_parameter_` attribute and uses this as the function argument. The +attribute must be an integer, string, bytes, a :mod:`ctypes` instance, or an +object with an :attr:`!_as_parameter_` attribute:: >>> class Bottles: ... def __init__(self, number): From webhook-mailer at python.org Mon Aug 7 08:46:40 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Mon, 07 Aug 2023 12:46:40 -0000 Subject: [Python-checkins] gh-85160: Reduce memory usage of `singledispatchmethod` (#107706) Message-ID: https://github.com/python/cpython/commit/2ac103c346ffe9d0e4c146402ce215c5ce6c1ef2 commit: 2ac103c346ffe9d0e4c146402ce215c5ce6c1ef2 branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-07T13:46:36+01:00 summary: gh-85160: Reduce memory usage of `singledispatchmethod` (#107706) A small followup to #107148 Co-authored-by: Serhiy Storchaka files: M Lib/functools.py diff --git a/Lib/functools.py b/Lib/functools.py index 2a8a69b3c527a..be44ccdae6b69 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -928,14 +928,14 @@ class singledispatchmethod: """ def __init__(self, func): - import weakref # see comment in singledispatch function if not callable(func) and not hasattr(func, "__get__"): raise TypeError(f"{func!r} is not callable or a descriptor") self.dispatcher = singledispatch(func) self.func = func + + import weakref # see comment in singledispatch function self._method_cache = weakref.WeakKeyDictionary() - self._all_weakrefable_instances = True def register(self, cls, method=None): """generic_method.register(cls, func) -> func @@ -945,11 +945,11 @@ def register(self, cls, method=None): return self.dispatcher.register(cls, func=method) def __get__(self, obj, cls=None): - if self._all_weakrefable_instances: + if self._method_cache is not None: try: _method = self._method_cache[obj] except TypeError: - self._all_weakrefable_instances = False + self._method_cache = None except KeyError: pass else: @@ -963,7 +963,7 @@ def _method(*args, **kwargs): _method.register = self.register update_wrapper(_method, self.func) - if self._all_weakrefable_instances: + if self._method_cache is not None: self._method_cache[obj] = _method return _method From webhook-mailer at python.org Mon Aug 7 09:11:09 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Mon, 07 Aug 2023 13:11:09 -0000 Subject: [Python-checkins] Docs: Fix more Sphinx annotations in ctypes.rst (#107708) Message-ID: https://github.com/python/cpython/commit/8c9af6b9a0d6fc9cb237e96588d8dcab727e32b8 commit: 8c9af6b9a0d6fc9cb237e96588d8dcab727e32b8 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-07T15:11:05+02:00 summary: Docs: Fix more Sphinx annotations in ctypes.rst (#107708) files: M Doc/conf.py M Doc/library/ctypes.rst M Doc/tools/.nitignore diff --git a/Doc/conf.py b/Doc/conf.py index 49108eac58fc1..19e05e1aa8fe1 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -94,6 +94,7 @@ ('c:func', 'sprintf'), ('c:func', 'stat'), ('c:func', 'system'), + ('c:func', 'time'), ('c:func', 'vsnprintf'), # Standard C types ('c:type', 'FILE'), diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index fcf711efe0eb7..474359a5cd98b 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -41,7 +41,7 @@ You load libraries by accessing them as attributes of these objects. *cdll* loads libraries which export functions using the standard ``cdecl`` calling convention, while *windll* libraries call functions using the ``stdcall`` calling convention. *oledll* also uses the ``stdcall`` calling convention, and -assumes the functions return a Windows :c:type:`HRESULT` error code. The error +assumes the functions return a Windows :c:type:`!HRESULT` error code. The error code is used to automatically raise an :class:`OSError` exception when the function call fails. @@ -477,7 +477,7 @@ Return types By default functions are assumed to return the C :c:expr:`int` type. Other -return types can be specified by setting the :attr:`restype` attribute of the +return types can be specified by setting the :attr:`~_FuncPtr.restype` attribute of the function object. The C prototype of :c:func:`time` is ``time_t time(time_t *)``. Because :c:type:`time_t` @@ -495,7 +495,7 @@ To call the function with a ``NULL`` pointer as first argument, use ``None``:: >>> print(libc.time(None)) # doctest: +SKIP 1150640792 -Here is a more advanced example, it uses the :func:`strchr` function, which expects +Here is a more advanced example, it uses the :func:`!strchr` function, which expects a string pointer and a char, and returns a pointer to a string:: >>> strchr = libc.strchr @@ -528,7 +528,7 @@ single character Python bytes object into a C char: >>> You can also use a callable Python object (a function or a class for example) as -the :attr:`restype` attribute, if the foreign function returns an integer. The +the :attr:`~_FuncPtr.restype` attribute, if the foreign function returns an integer. The callable will be called with the *integer* the C function returns, and the result of this call will be used as the result of your function call. This is useful to check for error return values and automatically raise an exception:: @@ -556,7 +556,8 @@ get the string representation of an error code, and *returns* an exception. :func:`GetLastError` to retrieve it. Please note that a much more powerful error checking mechanism is available -through the :attr:`errcheck` attribute; see the reference manual for details. +through the :attr:`~_FuncPtr.errcheck` attribute; +see the reference manual for details. .. _ctypes-passing-pointers: @@ -594,7 +595,7 @@ Structures and unions Structures and unions must derive from the :class:`Structure` and :class:`Union` base classes which are defined in the :mod:`ctypes` module. Each subclass must -define a :attr:`_fields_` attribute. :attr:`_fields_` must be a list of +define a :attr:`~Structure._fields_` attribute. :attr:`!_fields_` must be a list of *2-tuples*, containing a *field name* and a *field type*. The field type must be a :mod:`ctypes` type like :class:`c_int`, or any other @@ -666,9 +667,9 @@ Structure/union alignment and byte order By default, Structure and Union fields are aligned in the same way the C compiler does it. It is possible to override this behavior by specifying a -:attr:`_pack_` class attribute in the subclass definition. This must be set to a -positive integer and specifies the maximum alignment for the fields. This is -what ``#pragma pack(n)`` also does in MSVC. +:attr:`~Structure._pack_` class attribute in the subclass definition. +This must be set to a positive integer and specifies the maximum alignment for the fields. +This is what ``#pragma pack(n)`` also does in MSVC. :mod:`ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the @@ -684,7 +685,7 @@ Bit fields in structures and unions It is possible to create structures and unions containing bit fields. Bit fields are only possible for integer fields, the bit width is specified as the third -item in the :attr:`_fields_` tuples:: +item in the :attr:`~Structure._fields_` tuples:: >>> class Int(Structure): ... _fields_ = [("first_16", c_int, 16), @@ -876,7 +877,7 @@ pointer types. So, for ``POINTER(c_int)``, ctypes accepts an array of c_int:: >>> In addition, if a function argument is explicitly declared to be a pointer type -(such as ``POINTER(c_int)``) in :attr:`_FuncPtr.argtypes`, an object of the pointed +(such as ``POINTER(c_int)``) in :attr:`~_FuncPtr.argtypes`, an object of the pointed type (``c_int`` in this case) can be passed to the function. ctypes will apply the required :func:`byref` conversion in this case automatically. @@ -952,8 +953,8 @@ work:: >>> because the new ``class cell`` is not available in the class statement itself. -In :mod:`ctypes`, we can define the ``cell`` class and set the :attr:`_fields_` -attribute later, after the class statement:: +In :mod:`ctypes`, we can define the ``cell`` class and set the +:attr:`~Structure._fields_` attribute later, after the class statement:: >>> from ctypes import * >>> class cell(Structure): @@ -1003,8 +1004,8 @@ argument, and the callback functions expected argument types as the remaining arguments. I will present an example here which uses the standard C library's -:c:func:`qsort` function, that is used to sort items with the help of a callback -function. :c:func:`qsort` will be used to sort an array of integers:: +:c:func:`!qsort` function, that is used to sort items with the help of a callback +function. :c:func:`!qsort` will be used to sort an array of integers:: >>> IntArray5 = c_int * 5 >>> ia = IntArray5(5, 1, 7, 33, 99) @@ -1012,7 +1013,7 @@ function. :c:func:`qsort` will be used to sort an array of integers:: >>> qsort.restype = None >>> -:func:`qsort` must be called with a pointer to the data to sort, the number of +:func:`!qsort` must be called with a pointer to the data to sort, the number of items in the data array, the size of one item, and a pointer to the comparison function, the callback. The callback will then be called with two pointers to items, and it must return a negative integer if the first item is smaller than @@ -1104,7 +1105,7 @@ Some shared libraries not only export functions, they also export variables. An example in the Python library itself is the :c:data:`Py_Version`, Python runtime version number encoded in a single constant integer. -:mod:`ctypes` can access values like this with the :meth:`in_dll` class methods of +:mod:`ctypes` can access values like this with the :meth:`~_CData.in_dll` class methods of the type. *pythonapi* is a predefined symbol giving access to the Python C api:: @@ -1294,13 +1295,13 @@ Finding shared libraries When programming in a compiled language, shared libraries are accessed when compiling/linking a program, and when the program is run. -The purpose of the :func:`find_library` function is to locate a library in a way +The purpose of the :func:`~ctypes.util.find_library` function is to locate a library in a way similar to what the compiler or runtime loader does (on platforms with several versions of a shared library the most recent should be loaded), while the ctypes library loaders act like when a program is run, and call the runtime loader directly. -The :mod:`ctypes.util` module provides a function which can help to determine +The :mod:`!ctypes.util` module provides a function which can help to determine the library to load. @@ -1315,7 +1316,7 @@ the library to load. The exact functionality is system dependent. -On Linux, :func:`find_library` tries to run external programs +On Linux, :func:`~ctypes.util.find_library` tries to run external programs (``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the library file. It returns the filename of the library file. @@ -1334,7 +1335,7 @@ Here are some examples:: 'libbz2.so.1.0' >>> -On macOS, :func:`find_library` tries several predefined naming schemes and paths +On macOS, :func:`~ctypes.util.find_library` tries several predefined naming schemes and paths to locate the library, and returns a full pathname if successful:: >>> from ctypes.util import find_library @@ -1348,13 +1349,13 @@ to locate the library, and returns a full pathname if successful:: '/System/Library/Frameworks/AGL.framework/AGL' >>> -On Windows, :func:`find_library` searches along the system search path, and +On Windows, :func:`~ctypes.util.find_library` searches along the system search path, and returns the full pathname, but since there is no predefined naming scheme a call like ``find_library("c")`` will fail and return ``None``. If wrapping a shared library with :mod:`ctypes`, it *may* be better to determine the shared library name at development time, and hardcode that into the wrapper -module instead of using :func:`find_library` to locate the library at runtime. +module instead of using :func:`~ctypes.util.find_library` to locate the library at runtime. .. _ctypes-loading-shared-libraries: @@ -1439,9 +1440,9 @@ function exported by these libraries, and reacquired afterwards. All these classes can be instantiated by calling them with at least one argument, the pathname of the shared library. If you have an existing handle to an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platforms :c:func:`!dlopen` or :c:func:`LoadLibrary` -function is used to load the library into the process, and to get a handle to -it. +parameter, otherwise the underlying platforms :c:func:`!dlopen` or +:c:func:`!LoadLibrary` function is used to load the library into +the process, and to get a handle to it. The *mode* parameter can be used to specify how the library is loaded. For details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is @@ -1461,7 +1462,7 @@ to a new value and returns the former value. The *use_last_error* parameter, when set to true, enables the same mechanism for the Windows error code which is managed by the :func:`GetLastError` and -:func:`SetLastError` Windows API functions; :func:`ctypes.get_last_error` and +:func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and :func:`ctypes.set_last_error` are used to request and change the ctypes private copy of the windows error code. @@ -1533,7 +1534,7 @@ attribute of the loader instance. Class which loads shared libraries. *dlltype* should be one of the :class:`CDLL`, :class:`PyDLL`, :class:`WinDLL`, or :class:`OleDLL` types. - :meth:`__getattr__` has special behavior: It allows loading a shared library by + :meth:`!__getattr__` has special behavior: It allows loading a shared library by accessing it as attribute of a library loader instance. The result is cached, so repeated attribute accesses return the same library each time. @@ -1578,7 +1579,7 @@ object is available: An instance of :class:`PyDLL` that exposes Python C API functions as attributes. Note that all these functions are assumed to return C :c:expr:`int`, which is of course not always the truth, so you have to assign - the correct :attr:`restype` attribute to use these functions. + the correct :attr:`!restype` attribute to use these functions. .. audit-event:: ctypes.dlopen name ctypes.LibraryLoader @@ -1630,7 +1631,7 @@ They are instances of a private class: the callable will be called with this integer, allowing further processing or error checking. Using this is deprecated, for more flexible post processing or error checking use a ctypes data type as - :attr:`restype` and assign a callable to the :attr:`errcheck` attribute. + :attr:`!restype` and assign a callable to the :attr:`errcheck` attribute. .. attribute:: argtypes @@ -1662,7 +1663,7 @@ They are instances of a private class: :module: *result* is what the foreign function returns, as specified by the - :attr:`restype` attribute. + :attr:`!restype` attribute. *func* is the foreign function object itself, this allows reusing the same callable object to check or post process the results of several @@ -1772,7 +1773,7 @@ different ways, depending on the type and number of the parameters in the call: COM methods use a special calling convention: They require a pointer to the COM interface as first argument, in addition to those parameters that - are specified in the :attr:`~_FuncPtr.argtypes` tuple. + are specified in the :attr:`!argtypes` tuple. The optional *paramflags* parameter creates foreign function wrappers with much more functionality than the features described above. @@ -1847,7 +1848,7 @@ value if there is a single one, or a tuple containing the output parameter values when there are more than one, so the GetWindowRect function now returns a RECT instance, when called. -Output parameters can be combined with the :attr:`errcheck` protocol to do +Output parameters can be combined with the :attr:`~_FuncPtr.errcheck` protocol to do further output processing and error checking. The win32 ``GetWindowRect`` api function returns a ``BOOL`` to signal success or failure, so this function could do the error checking, and raises an exception when the api call failed:: @@ -1860,7 +1861,7 @@ do the error checking, and raises an exception when the api call failed:: >>> GetWindowRect.errcheck = errcheck >>> -If the :attr:`errcheck` function returns the argument tuple it receives +If the :attr:`~_FuncPtr.errcheck` function returns the argument tuple it receives unchanged, :mod:`ctypes` continues the normal processing it does on the output parameters. If you want to return a tuple of window coordinates instead of a ``RECT`` instance, you can retrieve the fields in the function and return them @@ -2010,7 +2011,7 @@ Utility functions .. function:: get_last_error() Windows only: returns the current value of the ctypes-private copy of the system - :data:`LastError` variable in the calling thread. + :data:`!LastError` variable in the calling thread. .. audit-event:: ctypes.get_last_error "" ctypes.get_last_error @@ -2063,7 +2064,7 @@ Utility functions .. function:: set_last_error(value) Windows only: set the current value of the ctypes-private copy of the system - :data:`LastError` variable in the calling thread to *value* and return the + :data:`!LastError` variable in the calling thread to *value* and return the previous value. .. audit-event:: ctypes.set_last_error error ctypes.set_last_error @@ -2225,13 +2226,13 @@ Fundamental data types Fundamental data types, when returned as foreign function call results, or, for example, by retrieving structure field members or array items, are transparently converted to native Python types. In other words, if a foreign function has a -:attr:`restype` of :class:`c_char_p`, you will always receive a Python bytes +:attr:`~_FuncPtr.restype` of :class:`c_char_p`, you will always receive a Python bytes object, *not* a :class:`c_char_p` instance. .. XXX above is false, it actually returns a Unicode string Subclasses of fundamental data types do *not* inherit this behavior. So, if a -foreign functions :attr:`restype` is a subclass of :class:`c_void_p`, you will +foreign functions :attr:`!restype` is a subclass of :class:`c_void_p`, you will receive an instance of this subclass from the function call. Of course, you can get the value of the pointer by accessing the ``value`` attribute. @@ -2430,7 +2431,7 @@ These are the fundamental ctypes data types: .. class:: HRESULT - Windows only: Represents a :c:type:`HRESULT` value, which contains success or + Windows only: Represents a :c:type:`!HRESULT` value, which contains success or error information for a function or method call. @@ -2439,9 +2440,9 @@ These are the fundamental ctypes data types: Represents the C :c:expr:`PyObject *` datatype. Calling this without an argument creates a ``NULL`` :c:expr:`PyObject *` pointer. -The :mod:`ctypes.wintypes` module provides quite some other Windows specific -data types, for example :c:type:`HWND`, :c:type:`WPARAM`, or :c:type:`DWORD`. Some -useful structures like :c:type:`MSG` or :c:type:`RECT` are also defined. +The :mod:`!ctypes.wintypes` module provides quite some other Windows specific +data types, for example :c:type:`!HWND`, :c:type:`!WPARAM`, or :c:type:`!DWORD`. +Some useful structures like :c:type:`!MSG` or :c:type:`!RECT` are also defined. .. _ctypes-structured-data-types: diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index df7dcd607489c..21b350c4134fd 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -59,7 +59,6 @@ Doc/library/configparser.rst Doc/library/contextlib.rst Doc/library/copy.rst Doc/library/csv.rst -Doc/library/ctypes.rst Doc/library/datetime.rst Doc/library/dbm.rst Doc/library/decimal.rst From webhook-mailer at python.org Mon Aug 7 09:26:53 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Mon, 07 Aug 2023 13:26:53 -0000 Subject: [Python-checkins] gh-107713: Reduce usage of mocks in `test_clinic.py` (#107714) Message-ID: https://github.com/python/cpython/commit/c399b5e1a56f7c659fc92edc20bc4bf522bdeffd commit: c399b5e1a56f7c659fc92edc20bc4bf522bdeffd branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-07T14:26:49+01:00 summary: gh-107713: Reduce usage of mocks in `test_clinic.py` (#107714) files: M Lib/test/test_clinic.py M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index f594e39a90546..d13d8623f8093 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -7,7 +7,6 @@ from test.support.os_helper import TESTFN, unlink from textwrap import dedent from unittest import TestCase -import collections import contextlib import inspect import os.path @@ -21,6 +20,13 @@ from clinic import DSLParser +def _make_clinic(*, filename='clinic_tests'): + clang = clinic.CLanguage(None) + c = clinic.Clinic(clang, filename=filename) + c.block_parser = clinic.BlockParser('', clang) + return c + + def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None): """Helper for the parser tests. @@ -41,81 +47,6 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None): tc.assertEqual(cm.exception.lineno, lineno) -class FakeConverter: - def __init__(self, name, args): - self.name = name - self.args = args - - -class FakeConverterFactory: - def __init__(self, name): - self.name = name - - def __call__(self, name, default, **kwargs): - return FakeConverter(self.name, kwargs) - - -class FakeConvertersDict: - def __init__(self): - self.used_converters = {} - - def get(self, name, default): - return self.used_converters.setdefault(name, FakeConverterFactory(name)) - -c = clinic.Clinic(language='C', filename = "file") - -class FakeClinic: - def __init__(self): - self.converters = FakeConvertersDict() - self.legacy_converters = FakeConvertersDict() - self.language = clinic.CLanguage(None) - self.filename = "clinic_tests" - self.destination_buffers = {} - self.block_parser = clinic.BlockParser('', self.language) - self.modules = collections.OrderedDict() - self.classes = collections.OrderedDict() - clinic.clinic = self - self.name = "FakeClinic" - self.line_prefix = self.line_suffix = '' - self.destinations = {} - self.add_destination("block", "buffer") - self.add_destination("file", "buffer") - self.add_destination("suppress", "suppress") - d = self.destinations.get - self.field_destinations = collections.OrderedDict(( - ('docstring_prototype', d('suppress')), - ('docstring_definition', d('block')), - ('methoddef_define', d('block')), - ('impl_prototype', d('block')), - ('parser_prototype', d('suppress')), - ('parser_definition', d('block')), - ('impl_definition', d('block')), - )) - self.functions = [] - - def get_destination(self, name): - d = self.destinations.get(name) - if not d: - sys.exit("Destination does not exist: " + repr(name)) - return d - - def add_destination(self, name, type, *args): - if name in self.destinations: - sys.exit("Destination already exists: " + repr(name)) - self.destinations[name] = clinic.Destination(name, type, self, *args) - - def is_directive(self, name): - return name == "module" - - def directive(self, name, args): - self.called_directives[name] = args - - _module_and_class = clinic.Clinic._module_and_class - - def __repr__(self): - return "" - - class ClinicWholeFileTest(TestCase): maxDiff = None @@ -124,7 +55,7 @@ def expect_failure(self, raw, errmsg, *, filename=None, lineno=None): filename=filename, lineno=lineno) def setUp(self): - self.clinic = clinic.Clinic(clinic.CLanguage(None), filename="test.c") + self.clinic = _make_clinic(filename="test.c") def test_eol(self): # regression test: @@ -848,7 +779,7 @@ def test_clinic_1(self): class ClinicParserTest(TestCase): def parse(self, text): - c = FakeClinic() + c = _make_clinic() parser = DSLParser(c) block = clinic.Block(text) parser.parse(block) @@ -872,7 +803,7 @@ def checkDocstring(self, fn, expected): dedent(expected).strip()) def test_trivial(self): - parser = DSLParser(FakeClinic()) + parser = DSLParser(_make_clinic()) block = clinic.Block(""" module os os.access @@ -1119,7 +1050,7 @@ def test_cloning_nonexistent_function_correctly_fails(self): with support.captured_stderr() as stderr: self.expect_failure(block, err, lineno=0) expected_debug_print = dedent("""\ - cls=None, module=, existing='fooooooooooooooooo' + cls=None, module=, existing='fooooooooooooooooo' (cls or module).functions=[] """) stderr = stderr.getvalue() @@ -1740,8 +1671,7 @@ def test_indent_stack_illegal_outdent(self): self.expect_failure(block, err) def test_directive(self): - c = FakeClinic() - parser = DSLParser(c) + parser = DSLParser(_make_clinic()) parser.flag = False parser.directives['setflag'] = lambda : setattr(parser, 'flag', True) block = clinic.Block("setflag") @@ -3147,22 +3077,24 @@ def test_Block_repr(self): self.assertEqual(repr(block3), expected_repr_3) def test_Destination_repr(self): + c = _make_clinic() + destination = clinic.Destination( - "foo", type="file", clinic=FakeClinic(), args=("eggs",) + "foo", type="file", clinic=c, args=("eggs",) ) self.assertEqual( repr(destination), "" ) - destination2 = clinic.Destination("bar", type="buffer", clinic=FakeClinic()) + destination2 = clinic.Destination("bar", type="buffer", clinic=c) self.assertEqual(repr(destination2), "") def test_Module_repr(self): - module = clinic.Module("foo", FakeClinic()) + module = clinic.Module("foo", _make_clinic()) self.assertRegex(repr(module), r"") def test_Class_repr(self): - cls = clinic.Class("foo", FakeClinic(), None, 'some_typedef', 'some_type_object') + cls = clinic.Class("foo", _make_clinic(), None, 'some_typedef', 'some_type_object') self.assertRegex(repr(cls), r"") def test_FunctionKind_repr(self): @@ -3176,7 +3108,7 @@ def test_FunctionKind_repr(self): def test_Function_and_Parameter_reprs(self): function = clinic.Function( name='foo', - module=FakeClinic(), + module=_make_clinic(), cls=None, c_basename=None, full_name='foofoo', diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 4dfe90b314f54..2bed98c23674e 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2424,6 +2424,9 @@ def _module_and_class( return module, cls + def __repr__(self) -> str: + return "" + def parse_file( filename: str, From webhook-mailer at python.org Mon Aug 7 09:52:40 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Mon, 07 Aug 2023 13:52:40 -0000 Subject: [Python-checkins] gh-107710: Speed up `logging.getHandlerNames` function (#107711) Message-ID: https://github.com/python/cpython/commit/8fcee6b2795a504f1bd39118759e5ce44183f689 commit: 8fcee6b2795a504f1bd39118759e5ce44183f689 branch: main author: Nikita Sobolev committer: AlexWaygood date: 2023-08-07T14:52:36+01:00 summary: gh-107710: Speed up `logging.getHandlerNames` function (#107711) files: A Misc/NEWS.d/next/Library/2023-08-07-14-24-42.gh-issue-107710.xfOCfj.rst M Lib/logging/__init__.py diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 46e86cb87ecfc..527fc5c631730 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -916,8 +916,7 @@ def getHandlerNames(): """ Return all known handler names as an immutable set. """ - result = set(_handlers.keys()) - return frozenset(result) + return frozenset(_handlers) class Handler(Filterer): diff --git a/Misc/NEWS.d/next/Library/2023-08-07-14-24-42.gh-issue-107710.xfOCfj.rst b/Misc/NEWS.d/next/Library/2023-08-07-14-24-42.gh-issue-107710.xfOCfj.rst new file mode 100644 index 0000000000000..70f8b58e7ff5f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-07-14-24-42.gh-issue-107710.xfOCfj.rst @@ -0,0 +1 @@ +Speed up :func:`logging.getHandlerNames`. From webhook-mailer at python.org Mon Aug 7 10:33:57 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Mon, 07 Aug 2023 14:33:57 -0000 Subject: [Python-checkins] gh-95065: Argument Clinic: Pretty-print long C strings in generated code (#107712) Message-ID: https://github.com/python/cpython/commit/835e38891535649321230f94121410244c583966 commit: 835e38891535649321230f94121410244c583966 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-07T14:33:52Z summary: gh-95065: Argument Clinic: Pretty-print long C strings in generated code (#107712) Co-authored-by: Serhiy Storchaka files: M Lib/test/clinic.test.c M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 321ac69273189..c5c37b7e9e216 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5521,17 +5521,31 @@ test_deprecate_positional_pos1_len1_optional(PyObject *module, PyObject *const * PyObject *b = Py_None; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1_optional' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ + " 'test_deprecate_positional_pos1_len1_optional' to be " \ + "keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1_optional' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ + " 'test_deprecate_positional_pos1_len1_optional' to be " \ + "keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1_optional' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ + " 'test_deprecate_positional_pos1_len1_optional' to be " \ + "keyword-only." # endif #endif if (nargs == 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing 2 positional arguments to test_deprecate_positional_pos1_len1_optional() is deprecated. Parameter 'b' will become a keyword-only parameter in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 2 positional arguments to " + "test_deprecate_positional_pos1_len1_optional() is deprecated. " + "Parameter 'b' will become a keyword-only parameter in Python " + "3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); @@ -5553,7 +5567,7 @@ test_deprecate_positional_pos1_len1_optional(PyObject *module, PyObject *const * static PyObject * test_deprecate_positional_pos1_len1_optional_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=20bdea6a2960ddf3 input=89099f3dacd757da]*/ +/*[clinic end generated code: output=09a6edec1ddcd469 input=89099f3dacd757da]*/ /*[clinic input] @@ -5609,17 +5623,27 @@ test_deprecate_positional_pos1_len1(PyObject *module, PyObject *const *args, Py_ PyObject *b; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ + " 'test_deprecate_positional_pos1_len1' to be keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ + " 'test_deprecate_positional_pos1_len1' to be keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'b' in the clinic input of 'test_deprecate_positional_pos1_len1' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ + " 'test_deprecate_positional_pos1_len1' to be keyword-only." # endif #endif if (nargs == 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing 2 positional arguments to test_deprecate_positional_pos1_len1() is deprecated. Parameter 'b' will become a keyword-only parameter in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 2 positional arguments to " + "test_deprecate_positional_pos1_len1() is deprecated. Parameter " + "'b' will become a keyword-only parameter in Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -5637,7 +5661,7 @@ test_deprecate_positional_pos1_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=22c70f8b36085758 input=1702bbab1e9b3b99]*/ +/*[clinic end generated code: output=52a2618293df747d input=1702bbab1e9b3b99]*/ /*[clinic input] @@ -5699,17 +5723,31 @@ test_deprecate_positional_pos1_len2_with_kwd(PyObject *module, PyObject *const * PyObject *d; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos1_len2_with_kwd' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic " \ + "input of 'test_deprecate_positional_pos1_len2_with_kwd' to be " \ + "keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos1_len2_with_kwd' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic " \ + "input of 'test_deprecate_positional_pos1_len2_with_kwd' to be " \ + "keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos1_len2_with_kwd' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic " \ + "input of 'test_deprecate_positional_pos1_len2_with_kwd' to be " \ + "keyword-only." # endif #endif if (nargs > 1 && nargs <= 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 1 positional argument to test_deprecate_positional_pos1_len2_with_kwd() is deprecated. Parameters 'b' and 'c' will become keyword-only parameters in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to " + "test_deprecate_positional_pos1_len2_with_kwd() is deprecated. " + "Parameters 'b' and 'c' will become keyword-only parameters in " + "Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); @@ -5730,7 +5768,7 @@ static PyObject * test_deprecate_positional_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=79c5f04220a1f3aa input=28cdb885f6c34eab]*/ +/*[clinic end generated code: output=550aabea548589b4 input=28cdb885f6c34eab]*/ /*[clinic input] @@ -5783,17 +5821,27 @@ test_deprecate_positional_pos0_len1(PyObject *module, PyObject *const *args, Py_ PyObject *a; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'a' in the clinic input of 'test_deprecate_positional_pos0_len1' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'test_deprecate_positional_pos0_len1' to be keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'a' in the clinic input of 'test_deprecate_positional_pos0_len1' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'test_deprecate_positional_pos0_len1' to be keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'a' in the clinic input of 'test_deprecate_positional_pos0_len1' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'test_deprecate_positional_pos0_len1' to be keyword-only." # endif #endif if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to test_deprecate_positional_pos0_len1() is deprecated. Parameter 'a' will become a keyword-only parameter in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to " + "test_deprecate_positional_pos0_len1() is deprecated. Parameter " + "'a' will become a keyword-only parameter in Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -5809,7 +5857,7 @@ test_deprecate_positional_pos0_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos0_len1_impl(PyObject *module, PyObject *a) -/*[clinic end generated code: output=1b7f23b9ffca431b input=678206db25c0652c]*/ +/*[clinic end generated code: output=66c63ec8d6903bde input=678206db25c0652c]*/ /*[clinic input] @@ -5865,17 +5913,30 @@ test_deprecate_positional_pos0_len2(PyObject *module, PyObject *const *args, Py_ PyObject *b; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic input of 'test_deprecate_positional_pos0_len2' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic " \ + "input of 'test_deprecate_positional_pos0_len2' to be " \ + "keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic input of 'test_deprecate_positional_pos0_len2' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic " \ + "input of 'test_deprecate_positional_pos0_len2' to be " \ + "keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic input of 'test_deprecate_positional_pos0_len2' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic " \ + "input of 'test_deprecate_positional_pos0_len2' to be " \ + "keyword-only." # endif #endif if (nargs > 0 && nargs <= 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to test_deprecate_positional_pos0_len2() is deprecated. Parameters 'a' and 'b' will become keyword-only parameters in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to " + "test_deprecate_positional_pos0_len2() is deprecated. Parameters " + "'a' and 'b' will become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -5893,7 +5954,7 @@ test_deprecate_positional_pos0_len2(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=31b494f2dcc016af input=fae0d0b1d480c939]*/ +/*[clinic end generated code: output=6b6df40aaf751b2e input=fae0d0b1d480c939]*/ /*[clinic input] @@ -5958,17 +6019,34 @@ test_deprecate_positional_pos0_len3_with_kwdonly(PyObject *module, PyObject *con PyObject *e; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos0_len3_with_kwdonly' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the " \ + "clinic input of " \ + "'test_deprecate_positional_pos0_len3_with_kwdonly' to be " \ + "keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos0_len3_with_kwdonly' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the " \ + "clinic input of " \ + "'test_deprecate_positional_pos0_len3_with_kwdonly' to be " \ + "keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the clinic input of 'test_deprecate_positional_pos0_len3_with_kwdonly' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the " \ + "clinic input of " \ + "'test_deprecate_positional_pos0_len3_with_kwdonly' to be " \ + "keyword-only." # endif #endif if (nargs > 0 && nargs <= 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing positional arguments to test_deprecate_positional_pos0_len3_with_kwdonly() is deprecated. Parameters 'a', 'b' and 'c' will become keyword-only parameters in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to " + "test_deprecate_positional_pos0_len3_with_kwdonly() is " + "deprecated. Parameters 'a', 'b' and 'c' will become keyword-only" + " parameters in Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); @@ -5991,7 +6069,7 @@ test_deprecate_positional_pos0_len3_with_kwdonly_impl(PyObject *module, PyObject *b, PyObject *c, PyObject *e) -/*[clinic end generated code: output=96978e786acfbc7b input=1b0121770c0c52e0]*/ +/*[clinic end generated code: output=5c936993846d01a3 input=1b0121770c0c52e0]*/ /*[clinic input] @@ -6049,17 +6127,27 @@ test_deprecate_positional_pos2_len1(PyObject *module, PyObject *const *args, Py_ PyObject *c; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'c' in the clinic input of 'test_deprecate_positional_pos2_len1' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'c' in the clinic input of" \ + " 'test_deprecate_positional_pos2_len1' to be keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'c' in the clinic input of 'test_deprecate_positional_pos2_len1' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'c' in the clinic input of" \ + " 'test_deprecate_positional_pos2_len1' to be keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'c' in the clinic input of 'test_deprecate_positional_pos2_len1' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'c' in the clinic input of" \ + " 'test_deprecate_positional_pos2_len1' to be keyword-only." # endif #endif if (nargs == 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing 3 positional arguments to test_deprecate_positional_pos2_len1() is deprecated. Parameter 'c' will become a keyword-only parameter in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 3 positional arguments to " + "test_deprecate_positional_pos2_len1() is deprecated. Parameter " + "'c' will become a keyword-only parameter in Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); @@ -6078,7 +6166,7 @@ test_deprecate_positional_pos2_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=ceadd05f11f7f491 input=e1d129689e69ec7c]*/ +/*[clinic end generated code: output=2641e037296e3b61 input=e1d129689e69ec7c]*/ /*[clinic input] @@ -6139,17 +6227,30 @@ test_deprecate_positional_pos2_len2(PyObject *module, PyObject *const *args, Py_ PyObject *d; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len2' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'test_deprecate_positional_pos2_len2' to be " \ + "keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len2' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'test_deprecate_positional_pos2_len2' to be " \ + "keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len2' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'test_deprecate_positional_pos2_len2' to be " \ + "keyword-only." # endif #endif if (nargs > 2 && nargs <= 4) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 2 positional arguments to test_deprecate_positional_pos2_len2() is deprecated. Parameters 'c' and 'd' will become keyword-only parameters in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 2 positional arguments to " + "test_deprecate_positional_pos2_len2() is deprecated. Parameters " + "'c' and 'd' will become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); @@ -6170,7 +6271,7 @@ static PyObject * test_deprecate_positional_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=5693682e3fa1188b input=0d53533463a12792]*/ +/*[clinic end generated code: output=4a9068ef8fee61f6 input=0d53533463a12792]*/ /*[clinic input] @@ -6238,17 +6339,31 @@ test_deprecate_positional_pos2_len3_with_kwdonly(PyObject *module, PyObject *con PyObject *e; #if PY_VERSION_HEX >= 0x030e00C0 - # error "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to be keyword-only." + # error \ + "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to " \ + "be keyword-only." #elif PY_VERSION_HEX >= 0x030e00A0 # ifdef _MSC_VER - # pragma message ("In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to be keyword-only.") + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to " \ + "be keyword-only.") # else - # warning "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to be keyword-only." + # warning \ + "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to " \ + "be keyword-only." # endif #endif if (nargs > 2 && nargs <= 4) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 2 positional arguments to test_deprecate_positional_pos2_len3_with_kwdonly() is deprecated. Parameters 'c' and 'd' will become keyword-only parameters in Python 3.14.", 1)) { - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 2 positional arguments to " + "test_deprecate_positional_pos2_len3_with_kwdonly() is " + "deprecated. Parameters 'c' and 'd' will become keyword-only " + "parameters in Python 3.14.", 1)) + { + goto exit; } } args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 1, argsbuf); @@ -6273,4 +6388,4 @@ test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=00d436de747a00f3 input=154fd450448d8935]*/ +/*[clinic end generated code: output=1154c2e3e798948c input=154fd450448d8935]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 2bed98c23674e..3a8431cb67efe 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -221,6 +221,20 @@ def c_repr(s: str) -> str: return '"' + s + '"' +def wrapped_c_string_literal( + text: str, + *, + width: int = 72, + suffix: str = '', + initial_indent: int = 0, + subsequent_indent: int = 4 +) -> str: + wrapped = textwrap.wrap(text, width=width, replace_whitespace=False, + drop_whitespace=False, break_on_hyphens=False) + separator = '"' + suffix + '\n' + subsequent_indent * ' ' + '"' + return initial_indent * ' ' + '"' + separator.join(wrapped) + '"' + + is_legal_c_identifier = re.compile('^[A-Za-z_][A-Za-z0-9_]*$').match def is_legal_py_identifier(s: str) -> bool: @@ -837,17 +851,22 @@ class CLanguage(Language): """) DEPRECATED_POSITIONAL_PROTOTYPE: Final[str] = r""" #if PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00C0 - # error "{cpp_message}" + # error \ + {cpp_message} #elif PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00A0 # ifdef _MSC_VER - # pragma message ("{cpp_message}") + # pragma message ( \ + {cpp_message}) # else - # warning "{cpp_message}" + # warning \ + {cpp_message} # endif #endif if ({condition}) {{{{ - if (PyErr_WarnEx(PyExc_DeprecationWarning, "{depr_message}", 1)) {{{{ - goto exit; + if (PyErr_WarnEx(PyExc_DeprecationWarning, + {depr_message}, 1)) + {{{{ + goto exit; }}}} }}}} """ @@ -899,6 +918,7 @@ def deprecate_positional_use( f"In {source}, update parameter(s) {pstr} in the clinic " f"input of {func.full_name!r} to be keyword-only." ) + # Format the deprecation message. if first_pos == 0: preamble = "Passing positional arguments to " @@ -926,8 +946,11 @@ def deprecate_positional_use( condition=condition, major=major, minor=minor, - cpp_message=cpp_message, - depr_message=depr_message, + cpp_message=wrapped_c_string_literal(cpp_message, suffix=" \\", + width=64, + subsequent_indent=16), + depr_message=wrapped_c_string_literal(depr_message, width=64, + subsequent_indent=20), ) return normalize_snippet(code, indent=4) From webhook-mailer at python.org Mon Aug 7 10:38:59 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 07 Aug 2023 14:38:59 -0000 Subject: [Python-checkins] gh-100814: Fix exception for invalid callable value of Tkinter image option (GH-107692) Message-ID: https://github.com/python/cpython/commit/50e3cc9748eb2103eb7ed6cc5a74d177df3cfb13 commit: 50e3cc9748eb2103eb7ed6cc5a74d177df3cfb13 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-07T17:38:55+03:00 summary: gh-100814: Fix exception for invalid callable value of Tkinter image option (GH-107692) Passing a callable object as an option value to a Tkinter image now raises the expected TclError instead of an AttributeError. files: A Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst M Lib/test/test_tkinter/test_images.py M Lib/tkinter/__init__.py diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index b6ba1c22dcbac..317b0a5c8f4a3 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -144,6 +144,14 @@ def test_configure_foreground(self): self.assertEqual(image['foreground'], '-foreground {} {} #000000 yellow') + def test_bug_100814(self): + # gh-100814: Passing a callable option value causes AttributeError. + with self.assertRaises(tkinter.TclError): + tkinter.BitmapImage('::img::test', master=self.root, spam=print) + image = tkinter.BitmapImage('::img::test', master=self.root) + with self.assertRaises(tkinter.TclError): + image.configure(spam=print) + class PhotoImageTest(AbstractTkTest, unittest.TestCase): @@ -274,6 +282,14 @@ def test_configure_palette(self): image.configure(palette='3/4/2') self.assertEqual(image['palette'], '3/4/2') + def test_bug_100814(self): + # gh-100814: Passing a callable option value causes AttributeError. + with self.assertRaises(tkinter.TclError): + tkinter.PhotoImage('::img::test', master=self.root, spam=print) + image = tkinter.PhotoImage('::img::test', master=self.root) + with self.assertRaises(tkinter.TclError): + image.configure(spam=print) + def test_blank(self): image = self.create() image.blank() diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index c675c511e0453..c59f8d11e8a9d 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -4069,8 +4069,6 @@ def __init__(self, imgtype, name=None, cnf={}, master=None, **kw): elif kw: cnf = kw options = () for k, v in cnf.items(): - if callable(v): - v = self._register(v) options = options + ('-'+k, v) self.tk.call(('image', 'create', imgtype, name,) + options) self.name = name @@ -4097,8 +4095,6 @@ def configure(self, **kw): for k, v in _cnfmerge(kw).items(): if v is not None: if k[-1] == '_': k = k[:-1] - if callable(v): - v = self._register(v) res = res + ('-'+k, v) self.tk.call((self.name, 'config') + res) diff --git a/Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst b/Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst new file mode 100644 index 0000000000000..86cb7bf79f307 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst @@ -0,0 +1,2 @@ +Passing a callable object as an option value to a Tkinter image now raises +the expected TclError instead of an AttributeError. From webhook-mailer at python.org Mon Aug 7 10:42:23 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Mon, 07 Aug 2023 14:42:23 -0000 Subject: [Python-checkins] [3.11] gh-107442: Document all valid types for ctypes `_as_parameter_` (GH-107443) (#107718) Message-ID: https://github.com/python/cpython/commit/880670a34f664b1c93b7da79520a58040b68cabf commit: 880670a34f664b1c93b7da79520a58040b68cabf branch: 3.11 author: Tomas R committer: erlend-aasland date: 2023-08-07T16:42:19+02:00 summary: [3.11] gh-107442: Document all valid types for ctypes `_as_parameter_` (GH-107443) (#107718) (cherry picked from commit 6925c578a0e3cbb00858e64da813a7ffe79623c4) Co-authored-by: Tomas R files: M Doc/library/ctypes.rst diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index b55ec32ef5af5..debc1c0adde84 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -399,9 +399,10 @@ Calling functions with your own custom data types ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You can also customize :mod:`ctypes` argument conversion to allow instances of -your own classes be used as function arguments. :mod:`ctypes` looks for an -:attr:`_as_parameter_` attribute and uses this as the function argument. Of -course, it must be one of integer, string, or bytes:: +your own classes be used as function arguments. :mod:`ctypes` looks for an +:attr:`!_as_parameter_` attribute and uses this as the function argument. The +attribute must be an integer, string, bytes, a :mod:`ctypes` instance, or an +object with an :attr:`!_as_parameter_` attribute:: >>> class Bottles: ... def __init__(self, number): From webhook-mailer at python.org Mon Aug 7 10:48:48 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 07 Aug 2023 14:48:48 -0000 Subject: [Python-checkins] [3.11] gh-104496: Use correct Tcl or Tk version in Tkinter tests (GH-107688) (GH-107719) Message-ID: https://github.com/python/cpython/commit/81c8f7d61982cf34506d6b21466f35f0184b472e commit: 81c8f7d61982cf34506d6b21466f35f0184b472e branch: 3.11 author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-07T14:48:43Z summary: [3.11] gh-104496: Use correct Tcl or Tk version in Tkinter tests (GH-107688) (GH-107719) In future Tcl and Tk versions can be desynchronized. (cherry picked from commit 3c8e8f3ceeae08fc43d885f5a4c65a3ee4b1a2c8) files: M Lib/test/test_tcl.py M Lib/tkinter/test/support.py M Lib/tkinter/test/test_tkinter/test_images.py M Lib/tkinter/test/test_tkinter/test_widgets.py M Lib/tkinter/test/test_ttk/test_style.py M Lib/tkinter/test/test_ttk/test_widgets.py M Lib/tkinter/test/widget_tests.py diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index cb3ab1d04435c..8537bd56961b1 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -23,14 +23,6 @@ tcl_version = tuple(map(int, _tkinter.TCL_VERSION.split('.'))) -_tk_patchlevel = None -def get_tk_patchlevel(): - global _tk_patchlevel - if _tk_patchlevel is None: - tcl = Tcl() - _tk_patchlevel = tcl.info_patchlevel() - return _tk_patchlevel - class TkinterTest(unittest.TestCase): @@ -574,7 +566,6 @@ def test_splitlist(self): (1, '2', (3.4,)) if self.wantobjects else ('1', '2', '3.4')), ] - tk_patchlevel = get_tk_patchlevel() if not self.wantobjects: expected = ('12', '\u20ac', '\xe2\x82\xac', '3.4') else: @@ -583,8 +574,8 @@ def test_splitlist(self): (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)), expected), ] - dbg_info = ('want objects? %s, Tcl version: %s, Tk patchlevel: %s' - % (self.wantobjects, tcl_version, tk_patchlevel)) + dbg_info = ('want objects? %s, Tcl version: %s, Tcl patchlevel: %s' + % (self.wantobjects, tcl_version, self.interp.info_patchlevel())) for arg, res in testcases: self.assertEqual(splitlist(arg), res, 'arg=%a, %s' % (arg, dbg_info)) diff --git a/Lib/tkinter/test/support.py b/Lib/tkinter/test/support.py index 9e26d04536f22..7f8e1e7078b07 100644 --- a/Lib/tkinter/test/support.py +++ b/Lib/tkinter/test/support.py @@ -80,28 +80,28 @@ def simulate_mouse_click(widget, x, y): import _tkinter tcl_version = tuple(map(int, _tkinter.TCL_VERSION.split('.'))) +tk_version = tuple(map(int, _tkinter.TK_VERSION.split('.'))) -def requires_tcl(*version): - if len(version) <= 2: - return unittest.skipUnless(tcl_version >= version, - 'requires Tcl version >= ' + '.'.join(map(str, version))) +def requires_tk(*version): + if len(version) <= 2 and tk_version >= version: + return lambda test: test def deco(test): @functools.wraps(test) def newtest(self): - if get_tk_patchlevel() < version: - self.skipTest('requires Tcl version >= ' + + root = getattr(self, 'root', None) + if get_tk_patchlevel(root) < version: + self.skipTest('requires Tk version >= ' + '.'.join(map(str, version))) test(self) return newtest return deco _tk_patchlevel = None -def get_tk_patchlevel(): +def get_tk_patchlevel(root): global _tk_patchlevel if _tk_patchlevel is None: - tcl = tkinter.Tcl() - _tk_patchlevel = tcl.info_patchlevel() + _tk_patchlevel = tkinter._parse_version(root.tk.globalgetvar('tk_patchLevel')) return _tk_patchlevel units = { diff --git a/Lib/tkinter/test/test_tkinter/test_images.py b/Lib/tkinter/test/test_tkinter/test_images.py index cc69ccac62d74..8b473cce5f994 100644 --- a/Lib/tkinter/test/test_tkinter/test_images.py +++ b/Lib/tkinter/test/test_tkinter/test_images.py @@ -2,7 +2,7 @@ import tkinter from test import support from test.support import os_helper -from tkinter.test.support import AbstractTkTest, AbstractDefaultRootTest, requires_tcl +from tkinter.test.support import AbstractTkTest, AbstractDefaultRootTest, requires_tk support.requires('gui') @@ -213,11 +213,11 @@ def test_create_from_gif_file(self): def test_create_from_gif_data(self): self.check_create_from_data('gif') - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_create_from_png_file(self): self.check_create_from_file('png') - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_create_from_png_data(self): self.check_create_from_data('png') diff --git a/Lib/tkinter/test/test_tkinter/test_widgets.py b/Lib/tkinter/test/test_tkinter/test_widgets.py index 2a5913521fbe1..24d00b714ad3d 100644 --- a/Lib/tkinter/test/test_tkinter/test_widgets.py +++ b/Lib/tkinter/test/test_tkinter/test_widgets.py @@ -4,7 +4,7 @@ import os from test.support import requires -from tkinter.test.support import (requires_tcl, +from tkinter.test.support import (requires_tk, get_tk_patchlevel, widget_eq, AbstractDefaultRootTest) from tkinter.test.widget_tests import ( @@ -614,7 +614,7 @@ def test_configure_inactiveselectbackground(self): widget = self.create() self.checkColorParam(widget, 'inactiveselectbackground') - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_configure_insertunfocussed(self): widget = self.create() self.checkEnumParam(widget, 'insertunfocussed', @@ -919,7 +919,7 @@ def test_coords(self): for i in range(4): self.assertIsInstance(coords[i], float) - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_moveto(self): widget = self.create() i1 = widget.create_rectangle(1, 1, 20, 20, tags='group') @@ -964,7 +964,7 @@ def test_configure_activestyle(self): self.checkEnumParam(widget, 'activestyle', 'dotbox', 'none', 'underline') - test_configure_justify = requires_tcl(8, 6, 5)(StandardOptionsTests.test_configure_justify) + test_configure_justify = requires_tk(8, 6, 5)(StandardOptionsTests.test_configure_justify) def test_configure_listvariable(self): widget = self.create() @@ -1103,7 +1103,7 @@ def test_configure_digits(self): def test_configure_from(self): widget = self.create() - conv = float if get_tk_patchlevel() >= (8, 6, 10) else float_round + conv = float if get_tk_patchlevel(self.root) >= (8, 6, 10) else float_round self.checkFloatParam(widget, 'from', 100, 14.9, 15.1, conv=conv) def test_configure_label(self): @@ -1230,19 +1230,19 @@ def test_configure_opaqueresize(self): widget = self.create() self.checkBooleanParam(widget, 'opaqueresize') - @requires_tcl(8, 6, 5) + @requires_tk(8, 6, 5) def test_configure_proxybackground(self): widget = self.create() self.checkColorParam(widget, 'proxybackground') - @requires_tcl(8, 6, 5) + @requires_tk(8, 6, 5) def test_configure_proxyborderwidth(self): widget = self.create() self.checkPixelsParam(widget, 'proxyborderwidth', 0, 1.3, 2.9, 6, -2, '10p', conv=False) - @requires_tcl(8, 6, 5) + @requires_tk(8, 6, 5) def test_configure_proxyrelief(self): widget = self.create() self.checkReliefParam(widget, 'proxyrelief') diff --git a/Lib/tkinter/test/test_ttk/test_style.py b/Lib/tkinter/test/test_ttk/test_style.py index 54ad3437168fe..f94adc41f4df8 100644 --- a/Lib/tkinter/test/test_ttk/test_style.py +++ b/Lib/tkinter/test/test_ttk/test_style.py @@ -170,7 +170,7 @@ def test_map_custom_copy(self): newname = f'C.{name}' self.assertEqual(style.map(newname), {}) style.map(newname, **default) - if theme == 'alt' and name == '.' and get_tk_patchlevel() < (8, 6, 1): + if theme == 'alt' and name == '.' and get_tk_patchlevel(self.root) < (8, 6, 1): default['embossed'] = [('disabled', '1')] self.assertEqual(style.map(newname), default) for key, value in default.items(): diff --git a/Lib/tkinter/test/test_ttk/test_widgets.py b/Lib/tkinter/test/test_ttk/test_widgets.py index 96d2afcf90ea8..1b9499e018f78 100644 --- a/Lib/tkinter/test/test_ttk/test_widgets.py +++ b/Lib/tkinter/test/test_ttk/test_widgets.py @@ -5,7 +5,7 @@ import sys from test.test_ttk_textonly import MockTclObj -from tkinter.test.support import (AbstractTkTest, tcl_version, get_tk_patchlevel, +from tkinter.test.support import (AbstractTkTest, tk_version, get_tk_patchlevel, simulate_mouse_click, AbstractDefaultRootTest) from tkinter.test.widget_tests import (add_standard_options, AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests, @@ -20,7 +20,7 @@ def test_configure_class(self): widget = self.create() self.assertEqual(widget['class'], '') errmsg='attempt to change read-only option' - if get_tk_patchlevel() < (8, 6, 0, 'beta', 3): + if get_tk_patchlevel(self.root) < (8, 6, 0, 'beta', 3): errmsg='Attempt to change read-only option' self.checkInvalidParam(widget, 'class', 'Foo', errmsg=errmsg) widget2 = self.create(class_='Foo') @@ -562,7 +562,7 @@ def test_configure_orient(self): widget = self.create() self.assertEqual(str(widget['orient']), 'vertical') errmsg='attempt to change read-only option' - if get_tk_patchlevel() < (8, 6, 0, 'beta', 3): + if get_tk_patchlevel(self.root) < (8, 6, 0, 'beta', 3): errmsg='Attempt to change read-only option' self.checkInvalidParam(widget, 'orient', 'horizontal', errmsg=errmsg) @@ -1528,7 +1528,7 @@ def test_heading(self): def test_heading_callback(self): def simulate_heading_click(x, y): - if tcl_version >= (8, 6): + if tk_version >= (8, 6): self.assertEqual(self.tv.identify_column(x), '#0') self.assertEqual(self.tv.identify_region(x, y), 'heading') simulate_mouse_click(self.tv, x, y) diff --git a/Lib/tkinter/test/widget_tests.py b/Lib/tkinter/test/widget_tests.py index a450544c3ee6b..1beb446cf27d0 100644 --- a/Lib/tkinter/test/widget_tests.py +++ b/Lib/tkinter/test/widget_tests.py @@ -2,7 +2,7 @@ import unittest import tkinter -from tkinter.test.support import (AbstractTkTest, tcl_version, +from tkinter.test.support import (AbstractTkTest, tk_version, pixels_conv, tcl_obj_eq) import test.support @@ -23,7 +23,7 @@ def scaling(self): return self._scaling def _str(self, value): - if not self._stringify and self.wantobjects and tcl_version >= (8, 6): + if not self._stringify and self.wantobjects and tk_version >= (8, 6): return value if isinstance(value, tuple): return ' '.join(map(self._str, value)) @@ -157,7 +157,7 @@ def checkReliefParam(self, widget, name): 'flat', 'groove', 'raised', 'ridge', 'solid', 'sunken') errmsg='bad relief "spam": must be '\ 'flat, groove, raised, ridge, solid, or sunken' - if tcl_version < (8, 6): + if tk_version < (8, 6): errmsg = None self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) From webhook-mailer at python.org Mon Aug 7 11:10:00 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 07 Aug 2023 15:10:00 -0000 Subject: [Python-checkins] gh-106566: Optimize (?!) in regular expressions (GH-106567) Message-ID: https://github.com/python/cpython/commit/ed64204716035db58c7e11b78182596aa2d97176 commit: ed64204716035db58c7e11b78182596aa2d97176 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-07T18:09:56+03:00 summary: gh-106566: Optimize (?!) in regular expressions (GH-106567) files: A Misc/NEWS.d/next/Library/2023-07-09-13-10-54.gh-issue-106566.NN35-U.rst M Lib/re/_parser.py M Lib/test/test_re.py diff --git a/Lib/re/_parser.py b/Lib/re/_parser.py index 22d10ab6e31d3..d00b7e67d5595 100644 --- a/Lib/re/_parser.py +++ b/Lib/re/_parser.py @@ -773,8 +773,10 @@ def _parse(source, state, verbose, nested, first=False): source.tell() - start) if char == "=": subpatternappend((ASSERT, (dir, p))) - else: + elif p: subpatternappend((ASSERT_NOT, (dir, p))) + else: + subpatternappend((FAILURE, ())) continue elif char == "(": diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index a6f5af17d7d51..a565cbe1a8d81 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2362,6 +2362,9 @@ def test_regression_gh94675(self): p.terminate() p.join() + def test_fail(self): + self.assertEqual(re.search(r'12(?!)|3', '123')[0], '3') + def get_debug_out(pat): with captured_stdout() as out: diff --git a/Misc/NEWS.d/next/Library/2023-07-09-13-10-54.gh-issue-106566.NN35-U.rst b/Misc/NEWS.d/next/Library/2023-07-09-13-10-54.gh-issue-106566.NN35-U.rst new file mode 100644 index 0000000000000..3b88dc7918387 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-09-13-10-54.gh-issue-106566.NN35-U.rst @@ -0,0 +1 @@ +Optimize ``(?!)`` (pattern which alwais fails) in regular expressions. From webhook-mailer at python.org Mon Aug 7 11:24:05 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 07 Aug 2023 15:24:05 -0000 Subject: [Python-checkins] gh-107715: Escape class name in regular expression (GH-107716) Message-ID: https://github.com/python/cpython/commit/85793278793708ad6b7132a54ac9fb4b2c5bcac1 commit: 85793278793708ad6b7132a54ac9fb4b2c5bcac1 branch: main author: Gertjan van Zwieten committer: serhiy-storchaka date: 2023-08-07T18:24:02+03:00 summary: gh-107715: Escape class name in regular expression (GH-107716) This patch escapes the class name before embedding it in the regular expression for `pat` in `doctest.DocTestFinder._find_lineno`. While class names do not ordinarily contain special characters, it is possible to encounter these when a class is created dynamically. Escaping the name will correctly return `None` in this scenario, rather than potentially matching a different class or raising `re.error` depending on the symbols used. files: A Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst M Lib/doctest.py diff --git a/Lib/doctest.py b/Lib/doctest.py index 2776d74bf9b58..a63df46a112e6 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1110,7 +1110,7 @@ def _find_lineno(self, obj, source_lines): if source_lines is None: return None pat = re.compile(r'^\s*class\s*%s\b' % - getattr(obj, '__name__', '-')) + re.escape(getattr(obj, '__name__', '-'))) for i, line in enumerate(source_lines): if pat.match(line): lineno = i diff --git a/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst b/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst new file mode 100644 index 0000000000000..cd2a5d0d5324a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst @@ -0,0 +1 @@ +Fix `doctest.DocTestFinder.find` in presence of class names with special characters. Patch by Gertjan van Zwieten. From webhook-mailer at python.org Mon Aug 7 11:51:47 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 07 Aug 2023 15:51:47 -0000 Subject: [Python-checkins] gh-107178: Add the C API tests for the Abstract Objects Layer (GH-107179) Message-ID: https://github.com/python/cpython/commit/16c9415fba4972743f1944ebc44946e475e68bc4 commit: 16c9415fba4972743f1944ebc44946e475e68bc4 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-07T18:51:43+03:00 summary: gh-107178: Add the C API tests for the Abstract Objects Layer (GH-107179) Cover all the Mapping Protocol, almost all the Sequence Protocol (except PySequence_Fast) and a part of the Object Protocol. Move existing tests to Lib/test/test_capi/test_abstract.py and Modules/_testcapi/abstract.c. Add also tests for PyDict C API. files: A Lib/test/test_capi/test_abstract.py A Lib/test/test_capi/test_dict.py A Misc/NEWS.d/next/Tests/2023-07-24-16-56-59.gh-issue-107178.Gq1usE.rst A Modules/_testcapi/abstract.c A Modules/_testcapi/dict.c M Lib/test/test_bytes.py M Lib/test/test_capi/test_misc.py M Lib/test/test_class.py M Modules/Setup.stdlib.in M Modules/_testcapi/parts.h M Modules/_testcapimodule.c M PCbuild/_testcapi.vcxproj M PCbuild/_testcapi.vcxproj.filters diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 7c62b722059d1..afd506f07520d 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1354,7 +1354,7 @@ def do_tests(setitem): except ValueError: pass try: - setitem(b, 0, None) + setitem(b, 0, object()) self.fail("Didn't raise TypeError") except TypeError: pass diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py new file mode 100644 index 0000000000000..3f51e5b28104d --- /dev/null +++ b/Lib/test/test_capi/test_abstract.py @@ -0,0 +1,722 @@ +import unittest +import sys +from collections import OrderedDict +from test import support +from test.support import import_helper +import _testcapi + + +NULL = None + +class TestObject: + @property + def evil(self): + raise RuntimeError('do not get evil') + @evil.setter + def evil(self, value): + raise RuntimeError('do not set evil') + @evil.deleter + def evil(self): + raise RuntimeError('do not del evil') + +class ProxyGetItem: + def __init__(self, obj): + self.obj = obj + def __getitem__(self, key): + return self.obj[key] + +class ProxySetItem: + def __init__(self, obj): + self.obj = obj + def __setitem__(self, key, value): + self.obj[key] = value + +class ProxyDelItem: + def __init__(self, obj): + self.obj = obj + def __delitem__(self, key): + del self.obj[key] + +def gen(): + yield 'a' + yield 'b' + yield 'c' + + +class CAPITest(unittest.TestCase): + + def test_object_getattr(self): + xgetattr = _testcapi.object_getattr + obj = TestObject() + obj.a = 11 + setattr(obj, '\U0001f40d', 22) + self.assertEqual(xgetattr(obj, 'a'), 11) + self.assertRaises(AttributeError, xgetattr, obj, 'b') + self.assertEqual(xgetattr(obj, '\U0001f40d'), 22) + + self.assertRaises(RuntimeError, xgetattr, obj, 'evil') + self.assertRaises(TypeError, xgetattr, obj, 1) + # CRASHES xgetattr(obj, NULL) + # CRASHES xgetattr(NULL, 'a') + + def test_object_getattrstring(self): + getattrstring = _testcapi.object_getattrstring + obj = TestObject() + obj.a = 11 + setattr(obj, '\U0001f40d', 22) + self.assertEqual(getattrstring(obj, b'a'), 11) + self.assertRaises(AttributeError, getattrstring, obj, b'b') + self.assertEqual(getattrstring(obj, '\U0001f40d'.encode()), 22) + + self.assertRaises(RuntimeError, getattrstring, obj, b'evil') + self.assertRaises(UnicodeDecodeError, getattrstring, obj, b'\xff') + # CRASHES getattrstring(obj, NULL) + # CRASHES getattrstring(NULL, b'a') + + def test_object_getoptionalattr(self): + getoptionalattr = _testcapi.object_getoptionalattr + obj = TestObject() + obj.a = 11 + setattr(obj, '\U0001f40d', 22) + self.assertEqual(getoptionalattr(obj, 'a'), 11) + self.assertIs(getoptionalattr(obj, 'b'), AttributeError) + self.assertEqual(getoptionalattr(obj, '\U0001f40d'), 22) + + self.assertRaises(RuntimeError, getoptionalattr, obj, 'evil') + self.assertRaises(TypeError, getoptionalattr, obj, 1) + # CRASHES getoptionalattr(obj, NULL) + # CRASHES getoptionalattr(NULL, 'a') + + def test_object_getoptionalattrstring(self): + getoptionalattrstring = _testcapi.object_getoptionalattrstring + obj = TestObject() + obj.a = 11 + setattr(obj, '\U0001f40d', 22) + self.assertEqual(getoptionalattrstring(obj, b'a'), 11) + self.assertIs(getoptionalattrstring(obj, b'b'), AttributeError) + self.assertEqual(getoptionalattrstring(obj, '\U0001f40d'.encode()), 22) + + self.assertRaises(RuntimeError, getoptionalattrstring, obj, b'evil') + self.assertRaises(UnicodeDecodeError, getoptionalattrstring, obj, b'\xff') + # CRASHES getoptionalattrstring(obj, NULL) + # CRASHES getoptionalattrstring(NULL, b'a') + + def test_object_hasattr(self): + xhasattr = _testcapi.object_hasattr + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + self.assertTrue(xhasattr(obj, 'a')) + self.assertFalse(xhasattr(obj, 'b')) + self.assertTrue(xhasattr(obj, '\U0001f40d')) + + self.assertFalse(xhasattr(obj, 'evil')) + self.assertFalse(xhasattr(obj, 1)) + # CRASHES xhasattr(obj, NULL) + # CRASHES xhasattr(NULL, 'a') + + def test_object_hasattrstring(self): + hasattrstring = _testcapi.object_hasattrstring + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + self.assertTrue(hasattrstring(obj, b'a')) + self.assertFalse(hasattrstring(obj, b'b')) + self.assertTrue(hasattrstring(obj, '\U0001f40d'.encode())) + + self.assertFalse(hasattrstring(obj, b'evil')) + self.assertFalse(hasattrstring(obj, b'\xff')) + # CRASHES hasattrstring(obj, NULL) + # CRASHES hasattrstring(NULL, b'a') + + def test_object_setattr(self): + xsetattr = _testcapi.object_setattr + obj = TestObject() + xsetattr(obj, 'a', 5) + self.assertEqual(obj.a, 5) + xsetattr(obj, '\U0001f40d', 8) + self.assertEqual(getattr(obj, '\U0001f40d'), 8) + + # PyObject_SetAttr(obj, attr_name, NULL) removes the attribute + xsetattr(obj, 'a', NULL) + self.assertFalse(hasattr(obj, 'a')) + self.assertRaises(AttributeError, xsetattr, obj, 'b', NULL) + self.assertRaises(RuntimeError, xsetattr, obj, 'evil', NULL) + + self.assertRaises(RuntimeError, xsetattr, obj, 'evil', 'good') + self.assertRaises(AttributeError, xsetattr, 42, 'a', 5) + self.assertRaises(TypeError, xsetattr, obj, 1, 5) + # CRASHES xsetattr(obj, NULL, 5) + # CRASHES xsetattr(NULL, 'a', 5) + + def test_object_setattrstring(self): + setattrstring = _testcapi.object_setattrstring + obj = TestObject() + setattrstring(obj, b'a', 5) + self.assertEqual(obj.a, 5) + setattrstring(obj, '\U0001f40d'.encode(), 8) + self.assertEqual(getattr(obj, '\U0001f40d'), 8) + + # PyObject_SetAttrString(obj, attr_name, NULL) removes the attribute + setattrstring(obj, b'a', NULL) + self.assertFalse(hasattr(obj, 'a')) + self.assertRaises(AttributeError, setattrstring, obj, b'b', NULL) + self.assertRaises(RuntimeError, setattrstring, obj, b'evil', NULL) + + self.assertRaises(RuntimeError, setattrstring, obj, b'evil', 'good') + self.assertRaises(AttributeError, setattrstring, 42, b'a', 5) + self.assertRaises(TypeError, setattrstring, obj, 1, 5) + self.assertRaises(UnicodeDecodeError, setattrstring, obj, b'\xff', 5) + # CRASHES setattrstring(obj, NULL, 5) + # CRASHES setattrstring(NULL, b'a', 5) + + def test_object_delattr(self): + xdelattr = _testcapi.object_delattr + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + xdelattr(obj, 'a') + self.assertFalse(hasattr(obj, 'a')) + self.assertRaises(AttributeError, xdelattr, obj, 'b') + xdelattr(obj, '\U0001f40d') + self.assertFalse(hasattr(obj, '\U0001f40d')) + + self.assertRaises(AttributeError, xdelattr, 42, 'numerator') + self.assertRaises(RuntimeError, xdelattr, obj, 'evil') + self.assertRaises(TypeError, xdelattr, obj, 1) + # CRASHES xdelattr(obj, NULL) + # CRASHES xdelattr(NULL, 'a') + + def test_object_delattrstring(self): + delattrstring = _testcapi.object_delattrstring + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + delattrstring(obj, b'a') + self.assertFalse(hasattr(obj, 'a')) + self.assertRaises(AttributeError, delattrstring, obj, b'b') + delattrstring(obj, '\U0001f40d'.encode()) + self.assertFalse(hasattr(obj, '\U0001f40d')) + + self.assertRaises(AttributeError, delattrstring, 42, b'numerator') + self.assertRaises(RuntimeError, delattrstring, obj, b'evil') + self.assertRaises(UnicodeDecodeError, delattrstring, obj, b'\xff') + # CRASHES delattrstring(obj, NULL) + # CRASHES delattrstring(NULL, b'a') + + + def test_mapping_check(self): + check = _testcapi.mapping_check + self.assertTrue(check({1: 2})) + self.assertTrue(check([1, 2])) + self.assertTrue(check((1, 2))) + self.assertTrue(check('abc')) + self.assertTrue(check(b'abc')) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + self.assertFalse(check(NULL)) + + def test_mapping_size(self): + for size in _testcapi.mapping_size, _testcapi.mapping_length: + self.assertEqual(size({1: 2}), 1) + self.assertEqual(size([1, 2]), 2) + self.assertEqual(size((1, 2)), 2) + self.assertEqual(size('abc'), 3) + self.assertEqual(size(b'abc'), 3) + + self.assertRaises(TypeError, size, 42) + self.assertRaises(TypeError, size, object()) + self.assertRaises(SystemError, size, NULL) + + def test_object_getitem(self): + getitem = _testcapi.object_getitem + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertRaises(KeyError, getitem, dct, 'b') + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = ProxyGetItem(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertRaises(KeyError, getitem, dct2, 'b') + + self.assertEqual(getitem(['a', 'b', 'c'], 1), 'b') + + self.assertRaises(TypeError, getitem, 42, 'a') + self.assertRaises(TypeError, getitem, {}, []) # unhashable + self.assertRaises(SystemError, getitem, {}, NULL) + self.assertRaises(IndexError, getitem, [], 1) + self.assertRaises(TypeError, getitem, [], 'a') + self.assertRaises(SystemError, getitem, NULL, 'a') + + def test_mapping_getitemstring(self): + getitemstring = _testcapi.mapping_getitemstring + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitemstring(dct, b'a'), 1) + self.assertRaises(KeyError, getitemstring, dct, b'b') + self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) + + dct2 = ProxyGetItem(dct) + self.assertEqual(getitemstring(dct2, b'a'), 1) + self.assertRaises(KeyError, getitemstring, dct2, b'b') + + self.assertRaises(TypeError, getitemstring, 42, b'a') + self.assertRaises(UnicodeDecodeError, getitemstring, {}, b'\xff') + self.assertRaises(SystemError, getitemstring, {}, NULL) + self.assertRaises(TypeError, getitemstring, [], b'a') + self.assertRaises(SystemError, getitemstring, NULL, b'a') + + def test_mapping_haskey(self): + haskey = _testcapi.mapping_haskey + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(haskey(dct, 'a')) + self.assertFalse(haskey(dct, 'b')) + self.assertTrue(haskey(dct, '\U0001f40d')) + + dct2 = ProxyGetItem(dct) + self.assertTrue(haskey(dct2, 'a')) + self.assertFalse(haskey(dct2, 'b')) + + self.assertTrue(haskey(['a', 'b', 'c'], 1)) + + self.assertFalse(haskey(42, 'a')) + self.assertFalse(haskey({}, [])) # unhashable + self.assertFalse(haskey({}, NULL)) + self.assertFalse(haskey([], 1)) + self.assertFalse(haskey([], 'a')) + self.assertFalse(haskey(NULL, 'a')) + + def test_mapping_haskeystring(self): + haskeystring = _testcapi.mapping_haskeystring + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(haskeystring(dct, b'a')) + self.assertFalse(haskeystring(dct, b'b')) + self.assertTrue(haskeystring(dct, '\U0001f40d'.encode())) + + dct2 = ProxyGetItem(dct) + self.assertTrue(haskeystring(dct2, b'a')) + self.assertFalse(haskeystring(dct2, b'b')) + + self.assertFalse(haskeystring(42, b'a')) + self.assertFalse(haskeystring({}, b'\xff')) + self.assertFalse(haskeystring({}, NULL)) + self.assertFalse(haskeystring([], b'a')) + self.assertFalse(haskeystring(NULL, b'a')) + + def test_object_setitem(self): + setitem = _testcapi.object_setitem + dct = {} + setitem(dct, 'a', 5) + self.assertEqual(dct, {'a': 5}) + setitem(dct, '\U0001f40d', 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct = {} + dct2 = ProxySetItem(dct) + setitem(dct2, 'a', 5) + self.assertEqual(dct, {'a': 5}) + + lst = ['a', 'b', 'c'] + setitem(lst, 1, 'x') + self.assertEqual(lst, ['a', 'x', 'c']) + + self.assertRaises(TypeError, setitem, 42, 'a', 5) + self.assertRaises(TypeError, setitem, {}, [], 5) # unhashable + self.assertRaises(SystemError, setitem, {}, NULL, 5) + self.assertRaises(SystemError, setitem, {}, 'a', NULL) + self.assertRaises(IndexError, setitem, [], 1, 5) + self.assertRaises(TypeError, setitem, [], 'a', 5) + self.assertRaises(TypeError, setitem, (), 1, 5) + self.assertRaises(SystemError, setitem, NULL, 'a', 5) + + def test_mapping_setitemstring(self): + setitemstring = _testcapi.mapping_setitemstring + dct = {} + setitemstring(dct, b'a', 5) + self.assertEqual(dct, {'a': 5}) + setitemstring(dct, '\U0001f40d'.encode(), 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct = {} + dct2 = ProxySetItem(dct) + setitemstring(dct2, b'a', 5) + self.assertEqual(dct, {'a': 5}) + + self.assertRaises(TypeError, setitemstring, 42, b'a', 5) + self.assertRaises(UnicodeDecodeError, setitemstring, {}, b'\xff', 5) + self.assertRaises(SystemError, setitemstring, {}, NULL, 5) + self.assertRaises(SystemError, setitemstring, {}, b'a', NULL) + self.assertRaises(TypeError, setitemstring, [], b'a', 5) + self.assertRaises(SystemError, setitemstring, NULL, b'a', 5) + + def test_object_delitem(self): + for delitem in _testcapi.object_delitem, _testcapi.mapping_delitem: + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitem(dct, 'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitem, dct, 'b') + delitem(dct, '\U0001f40d') + self.assertEqual(dct, {'c': 2}) + + dct = {'a': 1, 'c': 2} + dct2 = ProxyDelItem(dct) + delitem(dct2, 'a') + self.assertEqual(dct, {'c': 2}) + self.assertRaises(KeyError, delitem, dct2, 'b') + + lst = ['a', 'b', 'c'] + delitem(lst, 1) + self.assertEqual(lst, ['a', 'c']) + + self.assertRaises(TypeError, delitem, 42, 'a') + self.assertRaises(TypeError, delitem, {}, []) # unhashable + self.assertRaises(SystemError, delitem, {}, NULL) + self.assertRaises(IndexError, delitem, [], 1) + self.assertRaises(TypeError, delitem, [], 'a') + self.assertRaises(SystemError, delitem, NULL, 'a') + + def test_mapping_delitemstring(self): + delitemstring = _testcapi.mapping_delitemstring + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitemstring(dct, b'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitemstring, dct, b'b') + delitemstring(dct, '\U0001f40d'.encode()) + self.assertEqual(dct, {'c': 2}) + + dct = {'a': 1, 'c': 2} + dct2 = ProxyDelItem(dct) + delitemstring(dct2, b'a') + self.assertEqual(dct, {'c': 2}) + self.assertRaises(KeyError, delitemstring, dct2, b'b') + + self.assertRaises(TypeError, delitemstring, 42, b'a') + self.assertRaises(UnicodeDecodeError, delitemstring, {}, b'\xff') + self.assertRaises(SystemError, delitemstring, {}, NULL) + self.assertRaises(TypeError, delitemstring, [], b'a') + self.assertRaises(SystemError, delitemstring, NULL, b'a') + + def test_mapping_keys_valuesitems(self): + class Mapping1(dict): + def keys(self): + return list(super().keys()) + def values(self): + return list(super().values()) + def items(self): + return list(super().items()) + class Mapping2(dict): + def keys(self): + return tuple(super().keys()) + def values(self): + return tuple(super().values()) + def items(self): + return tuple(super().items()) + dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} + + for mapping in [{}, OrderedDict(), Mapping1(), Mapping2(), + dict_obj, OrderedDict(dict_obj), + Mapping1(dict_obj), Mapping2(dict_obj)]: + self.assertListEqual(_testcapi.mapping_keys(mapping), + list(mapping.keys())) + self.assertListEqual(_testcapi.mapping_values(mapping), + list(mapping.values())) + self.assertListEqual(_testcapi.mapping_items(mapping), + list(mapping.items())) + + def test_mapping_keys_valuesitems_bad_arg(self): + self.assertRaises(AttributeError, _testcapi.mapping_keys, object()) + self.assertRaises(AttributeError, _testcapi.mapping_values, object()) + self.assertRaises(AttributeError, _testcapi.mapping_items, object()) + self.assertRaises(AttributeError, _testcapi.mapping_keys, []) + self.assertRaises(AttributeError, _testcapi.mapping_values, []) + self.assertRaises(AttributeError, _testcapi.mapping_items, []) + self.assertRaises(SystemError, _testcapi.mapping_keys, NULL) + self.assertRaises(SystemError, _testcapi.mapping_values, NULL) + self.assertRaises(SystemError, _testcapi.mapping_items, NULL) + + class BadMapping: + def keys(self): + return None + def values(self): + return None + def items(self): + return None + bad_mapping = BadMapping() + self.assertRaises(TypeError, _testcapi.mapping_keys, bad_mapping) + self.assertRaises(TypeError, _testcapi.mapping_values, bad_mapping) + self.assertRaises(TypeError, _testcapi.mapping_items, bad_mapping) + + def test_sequence_check(self): + check = _testcapi.sequence_check + self.assertFalse(check({1: 2})) + self.assertTrue(check([1, 2])) + self.assertTrue(check((1, 2))) + self.assertTrue(check('abc')) + self.assertTrue(check(b'abc')) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + # CRASHES check(NULL) + + def test_sequence_size(self): + for size in _testcapi.sequence_size, _testcapi.sequence_length: + self.assertEqual(size([1, 2]), 2) + self.assertEqual(size((1, 2)), 2) + self.assertEqual(size('abc'), 3) + self.assertEqual(size(b'abc'), 3) + + self.assertRaises(TypeError, size, {}) + self.assertRaises(TypeError, size, 42) + self.assertRaises(TypeError, size, object()) + self.assertRaises(SystemError, size, NULL) + + def test_sequence_getitem(self): + getitem = _testcapi.sequence_getitem + lst = ['a', 'b', 'c'] + self.assertEqual(getitem(lst, 1), 'b') + self.assertEqual(getitem(lst, -1), 'c') + self.assertRaises(IndexError, getitem, lst, 3) + + self.assertRaises(TypeError, getitem, 42, 1) + self.assertRaises(TypeError, getitem, {}, 1) + self.assertRaises(SystemError, getitem, NULL, 1) + + def test_sequence_concat(self): + concat = _testcapi.sequence_concat + self.assertEqual(concat(['a', 'b'], [1, 2]), ['a', 'b', 1, 2]) + self.assertEqual(concat(('a', 'b'), (1, 2)), ('a', 'b', 1, 2)) + + self.assertRaises(TypeError, concat, [], ()) + self.assertRaises(TypeError, concat, (), []) + self.assertRaises(TypeError, concat, [], 42) + self.assertRaises(TypeError, concat, 42, []) + self.assertRaises(TypeError, concat, 42, 43) + self.assertRaises(SystemError, concat, [], NULL) + self.assertRaises(SystemError, concat, NULL, []) + + def test_sequence_repeat(self): + repeat = _testcapi.sequence_repeat + self.assertEqual(repeat(['a', 'b'], 2), ['a', 'b', 'a', 'b']) + self.assertEqual(repeat(('a', 'b'), 2), ('a', 'b', 'a', 'b')) + self.assertEqual(repeat(['a', 'b'], 0), []) + self.assertEqual(repeat(['a', 'b'], -1), []) + + self.assertRaises(TypeError, repeat, set(), 2) + self.assertRaises(TypeError, repeat, 42, 2) + self.assertRaises(SystemError, repeat, NULL, 2) + + def test_sequence_inplaceconcat(self): + inplaceconcat = _testcapi.sequence_inplaceconcat + lst = ['a', 'b'] + res = inplaceconcat(lst, [1, 2]) + self.assertEqual(res, ['a', 'b', 1, 2]) + self.assertIs(res, lst) + lst = ['a', 'b'] + res = inplaceconcat(lst, (1, 2)) + self.assertEqual(res, ['a', 'b', 1, 2]) + self.assertIs(res, lst) + self.assertEqual(inplaceconcat(('a', 'b'), (1, 2)), ('a', 'b', 1, 2)) + + self.assertRaises(TypeError, inplaceconcat, (), []) + self.assertRaises(TypeError, inplaceconcat, [], 42) + self.assertRaises(TypeError, inplaceconcat, 42, []) + self.assertRaises(TypeError, inplaceconcat, 42, 43) + self.assertRaises(SystemError, inplaceconcat, [], NULL) + self.assertRaises(SystemError, inplaceconcat, NULL, []) + + def test_sequence_inplacerepeat(self): + inplacerepeat = _testcapi.sequence_inplacerepeat + lst = ['a', 'b'] + res = inplacerepeat(lst, 2) + self.assertEqual(res, ['a', 'b', 'a', 'b']) + self.assertIs(res, lst) + self.assertEqual(inplacerepeat(('a', 'b'), 2), ('a', 'b', 'a', 'b')) + self.assertEqual(inplacerepeat(['a', 'b'], 0), []) + self.assertEqual(inplacerepeat(['a', 'b'], -1), []) + + self.assertRaises(TypeError, inplacerepeat, set(), 2) + self.assertRaises(TypeError, inplacerepeat, 42, 2) + self.assertRaises(SystemError, inplacerepeat, NULL, 2) + + def test_sequence_setitem(self): + setitem = _testcapi.sequence_setitem + lst = ['a', 'b', 'c'] + setitem(lst, 1, 'x') + self.assertEqual(lst, ['a', 'x', 'c']) + setitem(lst, -1, 'y') + self.assertEqual(lst, ['a', 'x', 'y']) + + setitem(lst, 0, NULL) + self.assertEqual(lst, ['x', 'y']) + self.assertRaises(IndexError, setitem, lst, 3, 'x') + + self.assertRaises(TypeError, setitem, 42, 1, 'x') + self.assertRaises(TypeError, setitem, {}, 1, 'x') + self.assertRaises(SystemError, setitem, NULL, 1, 'x') + + def test_sequence_delitem(self): + delitem = _testcapi.sequence_delitem + lst = ['a', 'b', 'c'] + delitem(lst, 1) + self.assertEqual(lst, ['a', 'c']) + delitem(lst, -1) + self.assertEqual(lst, ['a']) + self.assertRaises(IndexError, delitem, lst, 3) + + self.assertRaises(TypeError, delitem, 42, 1) + self.assertRaises(TypeError, delitem, {}, 1) + self.assertRaises(SystemError, delitem, NULL, 1) + + def test_sequence_setslice(self): + setslice = _testcapi.sequence_setslice + + # Correct case: + data = [1, 2, 3, 4, 5] + data_copy = data.copy() + + setslice(data, 1, 3, [8, 9]) + data_copy[1:3] = [8, 9] + self.assertEqual(data, data_copy) + self.assertEqual(data, [1, 8, 9, 4, 5]) + + # Custom class: + class Custom: + def __setitem__(self, index, value): + self.index = index + self.value = value + + c = Custom() + setslice(c, 0, 5, 'abc') + self.assertEqual(c.index, slice(0, 5)) + self.assertEqual(c.value, 'abc') + + # Immutable sequences must raise: + bad_seq1 = (1, 2, 3, 4) + self.assertRaises(TypeError, setslice, bad_seq1, 1, 3, (8, 9)) + self.assertEqual(bad_seq1, (1, 2, 3, 4)) + + bad_seq2 = 'abcd' + self.assertRaises(TypeError, setslice, bad_seq2, 1, 3, 'xy') + self.assertEqual(bad_seq2, 'abcd') + + # Not a sequence: + self.assertRaises(TypeError, setslice, object(), 1, 3, 'xy') + self.assertRaises(SystemError, setslice, NULL, 1, 3, 'xy') + + data_copy = data.copy() + setslice(data_copy, 1, 3, NULL) + self.assertEqual(data_copy, [1, 4, 5]) + + def test_sequence_delslice(self): + delslice = _testcapi.sequence_delslice + + # Correct case: + data = [1, 2, 3, 4, 5] + data_copy = data.copy() + + delslice(data, 1, 3) + del data_copy[1:3] + self.assertEqual(data, data_copy) + self.assertEqual(data, [1, 4, 5]) + + # Custom class: + class Custom: + def __delitem__(self, index): + self.index = index + + c = Custom() + delslice(c, 0, 5) + self.assertEqual(c.index, slice(0, 5)) + + # Immutable sequences must raise: + bad_seq1 = (1, 2, 3, 4) + self.assertRaises(TypeError, delslice, bad_seq1, 1, 3) + self.assertEqual(bad_seq1, (1, 2, 3, 4)) + + bad_seq2 = 'abcd' + self.assertRaises(TypeError, delslice, bad_seq2, 1, 3) + self.assertEqual(bad_seq2, 'abcd') + + # Not a sequence: + self.assertRaises(TypeError, delslice, object(), 1, 3) + self.assertRaises(SystemError, delslice, NULL, 1, 3) + + mapping = {1: 'a', 2: 'b', 3: 'c'} + self.assertRaises(KeyError, delslice, mapping, 1, 3) + self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) + + def test_sequence_count(self): + count = _testcapi.sequence_count + + lst = ['a', 'b', 'a'] + self.assertEqual(count(lst, 'a'), 2) + self.assertEqual(count(lst, 'c'), 0) + self.assertEqual(count(iter(lst), 'a'), 2) + self.assertEqual(count(iter(lst), 'c'), 0) + self.assertEqual(count({'a': 2}, 'a'), 1) + + self.assertRaises(TypeError, count, 42, 'a') + self.assertRaises(SystemError, count, [], NULL) + self.assertRaises(SystemError, count, [1], NULL) + self.assertRaises(SystemError, count, NULL, 'a') + + def test_sequence_contains(self): + contains = _testcapi.sequence_contains + + lst = ['a', 'b', 'a'] + self.assertEqual(contains(lst, 'a'), 1) + self.assertEqual(contains(lst, 'c'), 0) + self.assertEqual(contains(iter(lst), 'a'), 1) + self.assertEqual(contains(iter(lst), 'c'), 0) + self.assertEqual(contains({'a': 2}, 'a'), 1) + + # XXX Only for empty sequences. Should be SystemError? + self.assertEqual(contains([], NULL), 0) + + self.assertRaises(TypeError, contains, 42, 'a') + self.assertRaises(SystemError, contains, [1], NULL) + # CRASHES contains({}, NULL) + # CRASHES contains(set(), NULL) + # CRASHES contains(NULL, 'a') + + def test_sequence_index(self): + index = _testcapi.sequence_index + + lst = ['a', 'b', 'a'] + self.assertEqual(index(lst, 'a'), 0) + self.assertEqual(index(lst, 'b'), 1) + self.assertRaises(ValueError, index, lst, 'c') + self.assertEqual(index(iter(lst), 'a'), 0) + self.assertEqual(index(iter(lst), 'b'), 1) + self.assertRaises(ValueError, index, iter(lst), 'c') + dct = {'a': 2, 'b': 3} + self.assertEqual(index(dct, 'a'), 0) + self.assertEqual(index(dct, 'b'), 1) + self.assertRaises(ValueError, index, dct, 'c') + + self.assertRaises(TypeError, index, 42, 'a') + self.assertRaises(SystemError, index, [], NULL) + self.assertRaises(SystemError, index, [1], NULL) + self.assertRaises(SystemError, index, NULL, 'a') + + def test_sequence_list(self): + xlist = _testcapi.sequence_list + self.assertEqual(xlist(['a', 'b', 'c']), ['a', 'b', 'c']) + self.assertEqual(xlist(('a', 'b', 'c')), ['a', 'b', 'c']) + self.assertEqual(xlist(iter(['a', 'b', 'c'])), ['a', 'b', 'c']) + self.assertEqual(xlist(gen()), ['a', 'b', 'c']) + + self.assertRaises(TypeError, xlist, 42) + self.assertRaises(SystemError, xlist, NULL) + + def test_sequence_tuple(self): + xtuple = _testcapi.sequence_tuple + self.assertEqual(xtuple(['a', 'b', 'c']), ('a', 'b', 'c')) + self.assertEqual(xtuple(('a', 'b', 'c')), ('a', 'b', 'c')) + self.assertEqual(xtuple(iter(['a', 'b', 'c'])), ('a', 'b', 'c')) + self.assertEqual(xtuple(gen()), ('a', 'b', 'c')) + + self.assertRaises(TypeError, xtuple, 42) + self.assertRaises(SystemError, xtuple, NULL) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py new file mode 100644 index 0000000000000..9da6efd695eca --- /dev/null +++ b/Lib/test/test_capi/test_dict.py @@ -0,0 +1,413 @@ +import unittest +import sys +from collections import OrderedDict, UserDict +from types import MappingProxyType +from test import support +from test.support import import_helper +import _testcapi + + +NULL = None + +class DictSubclass(dict): + def __getitem__(self, key): + raise RuntimeError('do not get evil') + def __setitem__(self, key, value): + raise RuntimeError('do not set evil') + def __delitem__(self, key): + raise RuntimeError('do not del evil') + +def gen(): + yield 'a' + yield 'b' + yield 'c' + + +class CAPITest(unittest.TestCase): + + def test_dict_check(self): + check = _testcapi.dict_check + self.assertTrue(check({1: 2})) + self.assertTrue(check(OrderedDict({1: 2}))) + self.assertFalse(check(UserDict({1: 2}))) + self.assertFalse(check([1, 2])) + self.assertFalse(check(object())) + #self.assertFalse(check(NULL)) + + def test_dict_checkexact(self): + check = _testcapi.dict_checkexact + self.assertTrue(check({1: 2})) + self.assertFalse(check(OrderedDict({1: 2}))) + self.assertFalse(check(UserDict({1: 2}))) + self.assertFalse(check([1, 2])) + self.assertFalse(check(object())) + #self.assertFalse(check(NULL)) + + def test_dict_new(self): + dict_new = _testcapi.dict_new + dct = dict_new() + self.assertEqual(dct, {}) + self.assertIs(type(dct), dict) + dct2 = dict_new() + self.assertIsNot(dct2, dct) + + def test_dictproxy_new(self): + dictproxy_new = _testcapi.dictproxy_new + for dct in {1: 2}, OrderedDict({1: 2}), UserDict({1: 2}): + proxy = dictproxy_new(dct) + self.assertIs(type(proxy), MappingProxyType) + self.assertEqual(proxy, dct) + with self.assertRaises(TypeError): + proxy[1] = 3 + self.assertEqual(proxy[1], 2) + dct[1] = 4 + self.assertEqual(proxy[1], 4) + + self.assertRaises(TypeError, dictproxy_new, []) + self.assertRaises(TypeError, dictproxy_new, 42) + # CRASHES dictproxy_new(NULL) + + def test_dict_copy(self): + copy = _testcapi.dict_copy + for dct in {1: 2}, OrderedDict({1: 2}): + dct_copy = copy(dct) + self.assertIs(type(dct_copy), dict) + self.assertEqual(dct_copy, dct) + + self.assertRaises(SystemError, copy, UserDict()) + self.assertRaises(SystemError, copy, []) + self.assertRaises(SystemError, copy, 42) + self.assertRaises(SystemError, copy, NULL) + + def test_dict_clear(self): + clear = _testcapi.dict_clear + dct = {1: 2} + clear(dct) + self.assertEqual(dct, {}) + + # NOTE: It is not safe to call it with OrderedDict. + + # Has no effect for non-dicts. + dct = UserDict({1: 2}) + clear(dct) + self.assertEqual(dct, {1: 2}) + lst = [1, 2] + clear(lst) + self.assertEqual(lst, [1, 2]) + clear(object()) + + # CRASHES? clear(NULL) + + def test_dict_size(self): + size = _testcapi.dict_size + self.assertEqual(size({1: 2}), 1) + self.assertEqual(size(OrderedDict({1: 2})), 1) + + self.assertRaises(SystemError, size, UserDict()) + self.assertRaises(SystemError, size, []) + self.assertRaises(SystemError, size, 42) + self.assertRaises(SystemError, size, object()) + self.assertRaises(SystemError, size, NULL) + + def test_dict_getitem(self): + getitem = _testcapi.dict_getitem + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertIs(getitem(dct, 'b'), KeyError) + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertIs(getitem(dct2, 'b'), KeyError) + + self.assertIs(getitem({}, []), KeyError) # unhashable + self.assertIs(getitem(42, 'a'), KeyError) + self.assertIs(getitem([1], 0), KeyError) + # CRASHES getitem({}, NULL) + # CRASHES getitem(NULL, 'a') + + def test_dict_getitemstring(self): + getitemstring = _testcapi.dict_getitemstring + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitemstring(dct, b'a'), 1) + self.assertIs(getitemstring(dct, b'b'), KeyError) + self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitemstring(dct2, b'a'), 1) + self.assertIs(getitemstring(dct2, b'b'), KeyError) + + self.assertIs(getitemstring({}, b'\xff'), KeyError) + self.assertIs(getitemstring(42, b'a'), KeyError) + self.assertIs(getitemstring([], b'a'), KeyError) + # CRASHES getitemstring({}, NULL) + # CRASHES getitemstring(NULL, b'a') + + def test_dict_getitemref(self): + getitem = _testcapi.dict_getitemref + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertIs(getitem(dct, 'b'), KeyError) + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertIs(getitem(dct2, 'b'), KeyError) + + self.assertRaises(SystemError, getitem, 42, 'a') + self.assertRaises(TypeError, getitem, {}, []) # unhashable + self.assertRaises(SystemError, getitem, [], 1) + self.assertRaises(SystemError, getitem, [], 'a') + # CRASHES getitem({}, NULL) + # CRASHES getitem(NULL, 'a') + + def test_dict_getitemstringref(self): + getitemstring = _testcapi.dict_getitemstringref + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitemstring(dct, b'a'), 1) + self.assertIs(getitemstring(dct, b'b'), KeyError) + self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitemstring(dct2, b'a'), 1) + self.assertIs(getitemstring(dct2, b'b'), KeyError) + + self.assertRaises(SystemError, getitemstring, 42, b'a') + self.assertRaises(UnicodeDecodeError, getitemstring, {}, b'\xff') + self.assertRaises(SystemError, getitemstring, [], b'a') + # CRASHES getitemstring({}, NULL) + # CRASHES getitemstring(NULL, b'a') + + def test_dict_getitemwitherror(self): + getitem = _testcapi.dict_getitemwitherror + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertIs(getitem(dct, 'b'), KeyError) + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertIs(getitem(dct2, 'b'), KeyError) + + self.assertRaises(SystemError, getitem, 42, 'a') + self.assertRaises(TypeError, getitem, {}, []) # unhashable + self.assertRaises(SystemError, getitem, [], 1) + self.assertRaises(SystemError, getitem, [], 'a') + # CRASHES getitem({}, NULL) + # CRASHES getitem(NULL, 'a') + + def test_dict_contains(self): + contains = _testcapi.dict_contains + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(contains(dct, 'a')) + self.assertFalse(contains(dct, 'b')) + self.assertTrue(contains(dct, '\U0001f40d')) + + dct2 = DictSubclass(dct) + self.assertTrue(contains(dct2, 'a')) + self.assertFalse(contains(dct2, 'b')) + + self.assertRaises(TypeError, contains, {}, []) # unhashable + # CRASHES contains({}, NULL) + # CRASHES contains(UserDict(), 'a') + # CRASHES contains(42, 'a') + # CRASHES contains(NULL, 'a') + + def test_dict_setitem(self): + setitem = _testcapi.dict_setitem + dct = {} + setitem(dct, 'a', 5) + self.assertEqual(dct, {'a': 5}) + setitem(dct, '\U0001f40d', 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct2 = DictSubclass() + setitem(dct2, 'a', 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(TypeError, setitem, {}, [], 5) # unhashable + self.assertRaises(SystemError, setitem, UserDict(), 'a', 5) + self.assertRaises(SystemError, setitem, [1], 0, 5) + self.assertRaises(SystemError, setitem, 42, 'a', 5) + # CRASHES setitem({}, NULL, 5) + # CRASHES setitem({}, 'a', NULL) + # CRASHES setitem(NULL, 'a', 5) + + def test_dict_setitemstring(self): + setitemstring = _testcapi.dict_setitemstring + dct = {} + setitemstring(dct, b'a', 5) + self.assertEqual(dct, {'a': 5}) + setitemstring(dct, '\U0001f40d'.encode(), 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct2 = DictSubclass() + setitemstring(dct2, b'a', 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(UnicodeDecodeError, setitemstring, {}, b'\xff', 5) + self.assertRaises(SystemError, setitemstring, UserDict(), b'a', 5) + self.assertRaises(SystemError, setitemstring, 42, b'a', 5) + # CRASHES setitemstring({}, NULL, 5) + # CRASHES setitemstring({}, b'a', NULL) + # CRASHES setitemstring(NULL, b'a', 5) + + def test_dict_delitem(self): + delitem = _testcapi.dict_delitem + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitem(dct, 'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitem, dct, 'b') + delitem(dct, '\U0001f40d') + self.assertEqual(dct, {'c': 2}) + + dct2 = DictSubclass({'a': 1, 'c': 2}) + delitem(dct2, 'a') + self.assertEqual(dct2, {'c': 2}) + self.assertRaises(KeyError, delitem, dct2, 'b') + + self.assertRaises(TypeError, delitem, {}, []) # unhashable + self.assertRaises(SystemError, delitem, UserDict({'a': 1}), 'a') + self.assertRaises(SystemError, delitem, [1], 0) + self.assertRaises(SystemError, delitem, 42, 'a') + # CRASHES delitem({}, NULL) + # CRASHES delitem(NULL, 'a') + + def test_dict_delitemstring(self): + delitemstring = _testcapi.dict_delitemstring + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitemstring(dct, b'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitemstring, dct, b'b') + delitemstring(dct, '\U0001f40d'.encode()) + self.assertEqual(dct, {'c': 2}) + + dct2 = DictSubclass({'a': 1, 'c': 2}) + delitemstring(dct2, b'a') + self.assertEqual(dct2, {'c': 2}) + self.assertRaises(KeyError, delitemstring, dct2, b'b') + + self.assertRaises(UnicodeDecodeError, delitemstring, {}, b'\xff') + self.assertRaises(SystemError, delitemstring, UserDict({'a': 1}), b'a') + self.assertRaises(SystemError, delitemstring, 42, b'a') + # CRASHES delitemstring({}, NULL) + # CRASHES delitemstring(NULL, b'a') + + def test_dict_setdefault(self): + setdefault = _testcapi.dict_setdefault + dct = {} + self.assertEqual(setdefault(dct, 'a', 5), 5) + self.assertEqual(dct, {'a': 5}) + self.assertEqual(setdefault(dct, 'a', 8), 5) + self.assertEqual(dct, {'a': 5}) + + dct2 = DictSubclass() + self.assertEqual(setdefault(dct2, 'a', 5), 5) + self.assertEqual(dct2, {'a': 5}) + self.assertEqual(setdefault(dct2, 'a', 8), 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(TypeError, setdefault, {}, [], 5) # unhashable + self.assertRaises(SystemError, setdefault, UserDict(), 'a', 5) + self.assertRaises(SystemError, setdefault, [1], 0, 5) + self.assertRaises(SystemError, setdefault, 42, 'a', 5) + # CRASHES setdefault({}, NULL, 5) + # CRASHES setdefault({}, 'a', NULL) + # CRASHES setdefault(NULL, 'a', 5) + + def test_mapping_keys_valuesitems(self): + class BadMapping(dict): + def keys(self): + return None + def values(self): + return None + def items(self): + return None + dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} + for mapping in [dict_obj, DictSubclass(dict_obj), BadMapping(dict_obj)]: + self.assertListEqual(_testcapi.dict_keys(mapping), + list(dict_obj.keys())) + self.assertListEqual(_testcapi.dict_values(mapping), + list(dict_obj.values())) + self.assertListEqual(_testcapi.dict_items(mapping), + list(dict_obj.items())) + + def test_dict_keys_valuesitems_bad_arg(self): + for mapping in UserDict(), [], object(): + self.assertRaises(SystemError, _testcapi.dict_keys, mapping) + self.assertRaises(SystemError, _testcapi.dict_values, mapping) + self.assertRaises(SystemError, _testcapi.dict_items, mapping) + + def test_dict_next(self): + dict_next = _testcapi.dict_next + self.assertIsNone(dict_next({}, 0)) + dct = {'a': 1, 'b': 2, 'c': 3} + pos = 0 + pairs = [] + while True: + res = dict_next(dct, pos) + if res is None: + break + rc, pos, key, value = res + self.assertEqual(rc, 1) + pairs.append((key, value)) + self.assertEqual(pairs, list(dct.items())) + + # CRASHES dict_next(NULL, 0) + + def test_dict_update(self): + update = _testcapi.dict_update + for cls1 in dict, DictSubclass: + for cls2 in dict, DictSubclass, UserDict: + dct = cls1({'a': 1, 'b': 2}) + update(dct, cls2({'b': 3, 'c': 4})) + self.assertEqual(dct, {'a': 1, 'b': 3, 'c': 4}) + + self.assertRaises(AttributeError, update, {}, []) + self.assertRaises(AttributeError, update, {}, 42) + self.assertRaises(SystemError, update, UserDict(), {}) + self.assertRaises(SystemError, update, 42, {}) + self.assertRaises(SystemError, update, {}, NULL) + self.assertRaises(SystemError, update, NULL, {}) + + def test_dict_merge(self): + merge = _testcapi.dict_merge + for cls1 in dict, DictSubclass: + for cls2 in dict, DictSubclass, UserDict: + dct = cls1({'a': 1, 'b': 2}) + merge(dct, cls2({'b': 3, 'c': 4}), 0) + self.assertEqual(dct, {'a': 1, 'b': 2, 'c': 4}) + dct = cls1({'a': 1, 'b': 2}) + merge(dct, cls2({'b': 3, 'c': 4}), 1) + self.assertEqual(dct, {'a': 1, 'b': 3, 'c': 4}) + + self.assertRaises(AttributeError, merge, {}, [], 0) + self.assertRaises(AttributeError, merge, {}, 42, 0) + self.assertRaises(SystemError, merge, UserDict(), {}, 0) + self.assertRaises(SystemError, merge, 42, {}, 0) + self.assertRaises(SystemError, merge, {}, NULL, 0) + self.assertRaises(SystemError, merge, NULL, {}, 0) + + def test_dict_mergefromseq2(self): + mergefromseq2 = _testcapi.dict_mergefromseq2 + for cls1 in dict, DictSubclass: + for cls2 in list, iter: + dct = cls1({'a': 1, 'b': 2}) + mergefromseq2(dct, cls2([('b', 3), ('c', 4)]), 0) + self.assertEqual(dct, {'a': 1, 'b': 2, 'c': 4}) + dct = cls1({'a': 1, 'b': 2}) + mergefromseq2(dct, cls2([('b', 3), ('c', 4)]), 1) + self.assertEqual(dct, {'a': 1, 'b': 3, 'c': 4}) + + self.assertRaises(ValueError, mergefromseq2, {}, [(1,)], 0) + self.assertRaises(ValueError, mergefromseq2, {}, [(1, 2, 3)], 0) + self.assertRaises(TypeError, mergefromseq2, {}, [1], 0) + self.assertRaises(TypeError, mergefromseq2, {}, 42, 0) + # CRASHES mergefromseq2(UserDict(), [], 0) + # CRASHES mergefromseq2(42, [], 0) + # CRASHES mergefromseq2({}, NULL, 0) + # CRASHES mergefromseq2(NULL, {}, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index e7cdd4be002a1..4ab1692447804 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -299,137 +299,6 @@ def test_getitem_with_error(self): def test_buildvalue_N(self): _testcapi.test_buildvalue_N() - def test_mapping_keys_values_items(self): - class Mapping1(dict): - def keys(self): - return list(super().keys()) - def values(self): - return list(super().values()) - def items(self): - return list(super().items()) - class Mapping2(dict): - def keys(self): - return tuple(super().keys()) - def values(self): - return tuple(super().values()) - def items(self): - return tuple(super().items()) - dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} - - for mapping in [{}, OrderedDict(), Mapping1(), Mapping2(), - dict_obj, OrderedDict(dict_obj), - Mapping1(dict_obj), Mapping2(dict_obj)]: - self.assertListEqual(_testcapi.get_mapping_keys(mapping), - list(mapping.keys())) - self.assertListEqual(_testcapi.get_mapping_values(mapping), - list(mapping.values())) - self.assertListEqual(_testcapi.get_mapping_items(mapping), - list(mapping.items())) - - def test_mapping_keys_values_items_bad_arg(self): - self.assertRaises(AttributeError, _testcapi.get_mapping_keys, None) - self.assertRaises(AttributeError, _testcapi.get_mapping_values, None) - self.assertRaises(AttributeError, _testcapi.get_mapping_items, None) - - class BadMapping: - def keys(self): - return None - def values(self): - return None - def items(self): - return None - bad_mapping = BadMapping() - self.assertRaises(TypeError, _testcapi.get_mapping_keys, bad_mapping) - self.assertRaises(TypeError, _testcapi.get_mapping_values, bad_mapping) - self.assertRaises(TypeError, _testcapi.get_mapping_items, bad_mapping) - - def test_mapping_has_key(self): - dct = {'a': 1} - self.assertTrue(_testcapi.mapping_has_key(dct, 'a')) - self.assertFalse(_testcapi.mapping_has_key(dct, 'b')) - - class SubDict(dict): - pass - - dct2 = SubDict({'a': 1}) - self.assertTrue(_testcapi.mapping_has_key(dct2, 'a')) - self.assertFalse(_testcapi.mapping_has_key(dct2, 'b')) - - def test_sequence_set_slice(self): - # Correct case: - data = [1, 2, 3, 4, 5] - data_copy = data.copy() - - _testcapi.sequence_set_slice(data, 1, 3, [8, 9]) - data_copy[1:3] = [8, 9] - self.assertEqual(data, data_copy) - self.assertEqual(data, [1, 8, 9, 4, 5]) - - # Custom class: - class Custom: - def __setitem__(self, index, value): - self.index = index - self.value = value - - c = Custom() - _testcapi.sequence_set_slice(c, 0, 5, 'abc') - self.assertEqual(c.index, slice(0, 5)) - self.assertEqual(c.value, 'abc') - - # Immutable sequences must raise: - bad_seq1 = (1, 2, 3, 4) - with self.assertRaises(TypeError): - _testcapi.sequence_set_slice(bad_seq1, 1, 3, (8, 9)) - self.assertEqual(bad_seq1, (1, 2, 3, 4)) - - bad_seq2 = 'abcd' - with self.assertRaises(TypeError): - _testcapi.sequence_set_slice(bad_seq2, 1, 3, 'xy') - self.assertEqual(bad_seq2, 'abcd') - - # Not a sequence: - with self.assertRaises(TypeError): - _testcapi.sequence_set_slice(None, 1, 3, 'xy') - - def test_sequence_del_slice(self): - # Correct case: - data = [1, 2, 3, 4, 5] - data_copy = data.copy() - - _testcapi.sequence_del_slice(data, 1, 3) - del data_copy[1:3] - self.assertEqual(data, data_copy) - self.assertEqual(data, [1, 4, 5]) - - # Custom class: - class Custom: - def __delitem__(self, index): - self.index = index - - c = Custom() - _testcapi.sequence_del_slice(c, 0, 5) - self.assertEqual(c.index, slice(0, 5)) - - # Immutable sequences must raise: - bad_seq1 = (1, 2, 3, 4) - with self.assertRaises(TypeError): - _testcapi.sequence_del_slice(bad_seq1, 1, 3) - self.assertEqual(bad_seq1, (1, 2, 3, 4)) - - bad_seq2 = 'abcd' - with self.assertRaises(TypeError): - _testcapi.sequence_del_slice(bad_seq2, 1, 3) - self.assertEqual(bad_seq2, 'abcd') - - # Not a sequence: - with self.assertRaises(TypeError): - _testcapi.sequence_del_slice(None, 1, 3) - - mapping = {1: 'a', 2: 'b', 3: 'c'} - with self.assertRaises(KeyError): - _testcapi.sequence_del_slice(mapping, 1, 3) - self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) - @unittest.skipUnless(hasattr(_testcapi, 'negative_refcount'), 'need _testcapi.negative_refcount') def test_negative_refcount(self): diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index bb35baea508c7..1531aad4f1f77 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -455,8 +455,8 @@ def __init__(self): self.attr = 1 a = A() - self.assertEqual(_testcapi.hasattr_string(a, "attr"), True) - self.assertEqual(_testcapi.hasattr_string(a, "noattr"), False) + self.assertEqual(_testcapi.object_hasattrstring(a, b"attr"), 1) + self.assertEqual(_testcapi.object_hasattrstring(a, b"noattr"), 0) self.assertIsNone(sys.exception()) def testDel(self): diff --git a/Misc/NEWS.d/next/Tests/2023-07-24-16-56-59.gh-issue-107178.Gq1usE.rst b/Misc/NEWS.d/next/Tests/2023-07-24-16-56-59.gh-issue-107178.Gq1usE.rst new file mode 100644 index 0000000000000..dd6becf6b0013 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-07-24-16-56-59.gh-issue-107178.Gq1usE.rst @@ -0,0 +1,2 @@ +Add the C API test for functions in the Mapping Protocol, the Sequence +Protocol and some functions in the Object Protocol. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 11a022e3d2044..689f1d42ef0ee 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -159,7 +159,7 @@ @MODULE__XXTESTFUZZ_TRUE at _xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE at _testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE at _testinternalcapi _testinternalcapi.c - at MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c + at MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c @MODULE__TESTCLINIC_TRUE at _testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/abstract.c b/Modules/_testcapi/abstract.c new file mode 100644 index 0000000000000..10d7ff8d4a7bc --- /dev/null +++ b/Modules/_testcapi/abstract.c @@ -0,0 +1,640 @@ +#include // ptrdiff_t + +#include "parts.h" + +#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromLong(_ret); \ + } while (0) + +#define RETURN_SIZE(value) do { \ + Py_ssize_t _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromSsize_t(_ret); \ + } while (0) + + +static PyObject * +object_getattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + return PyObject_GetAttr(obj, attr_name); +} + +static PyObject * +object_getattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + return PyObject_GetAttrString(obj, attr_name); +} + +static PyObject * +object_getoptionalattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name, *value; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + + switch (PyObject_GetOptionalAttr(obj, attr_name, &value)) { + case -1: + assert(value == NULL); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_AttributeError); + case 1: + return value; + default: + Py_FatalError("PyObject_GetOptionalAttr() returned invalid code"); + Py_UNREACHABLE(); + } +} + +static PyObject * +object_getoptionalattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj, *value; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + + switch (PyObject_GetOptionalAttrString(obj, attr_name, &value)) { + case -1: + assert(value == NULL); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_AttributeError); + case 1: + return value; + default: + Py_FatalError("PyObject_GetOptionalAttrString() returned invalid code"); + Py_UNREACHABLE(); + } +} + +static PyObject * +object_hasattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + return PyLong_FromLong(PyObject_HasAttr(obj, attr_name)); +} + +static PyObject * +object_hasattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + return PyLong_FromLong(PyObject_HasAttrString(obj, attr_name)); +} + +static PyObject * +object_setattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name, *value; + if (!PyArg_ParseTuple(args, "OOO", &obj, &attr_name, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + NULLABLE(value); + RETURN_INT(PyObject_SetAttr(obj, attr_name, value)); +} + +static PyObject * +object_setattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj, *value; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &obj, &attr_name, &size, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyObject_SetAttrString(obj, attr_name, value)); +} + +static PyObject * +object_delattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; +if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + RETURN_INT(PyObject_DelAttr(obj, attr_name)); +} + +static PyObject * +object_delattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + RETURN_INT(PyObject_DelAttrString(obj, attr_name)); +} + + +static PyObject * +mapping_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyMapping_Check(obj)); +} + +static PyObject * +mapping_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyMapping_Size(obj)); +} + +static PyObject * +mapping_length(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyMapping_Length(obj)); +} + +static PyObject * +object_getitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + return PyObject_GetItem(mapping, key); +} + +static PyObject * +mapping_getitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + return PyMapping_GetItemString(mapping, key); +} + +static PyObject * +mapping_getoptionalitem(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name, *value; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + + switch (PyMapping_GetOptionalItem(obj, attr_name, &value)) { + case -1: + assert(value == NULL); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_KeyError); + case 1: + return value; + default: + Py_FatalError("PyMapping_GetOptionalItem() returned invalid code"); + Py_UNREACHABLE(); + } +} + +static PyObject * +mapping_getoptionalitemstring(PyObject *self, PyObject *args) +{ + PyObject *obj, *value; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + + switch (PyMapping_GetOptionalItemString(obj, attr_name, &value)) { + case -1: + assert(value == NULL); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_KeyError); + case 1: + return value; + default: + Py_FatalError("PyMapping_GetOptionalItemString() returned invalid code"); + Py_UNREACHABLE(); + } +} + +static PyObject * +mapping_haskey(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + return PyLong_FromLong(PyMapping_HasKey(mapping, key)); +} + +static PyObject * +mapping_haskeystring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + return PyLong_FromLong(PyMapping_HasKeyString(mapping, key)); +} + +static PyObject * +object_setitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(value); + RETURN_INT(PyObject_SetItem(mapping, key, value)); +} + +static PyObject * +mapping_setitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping, *value; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(value); + RETURN_INT(PyMapping_SetItemString(mapping, key, value)); +} + +static PyObject * +object_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyObject_DelItem(mapping, key)); +} + +static PyObject * +mapping_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyMapping_DelItem(mapping, key)); +} + +static PyObject * +mapping_delitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyMapping_DelItemString(mapping, key)); +} + +static PyObject * +mapping_keys(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Keys(obj); +} + +static PyObject * +mapping_values(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Values(obj); +} + +static PyObject * +mapping_items(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Items(obj); +} + + +static PyObject * +sequence_check(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PySequence_Check(obj)); +} + +static PyObject * +sequence_size(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySequence_Size(obj)); +} + +static PyObject * +sequence_length(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySequence_Length(obj)); +} + +static PyObject * +sequence_concat(PyObject *self, PyObject *args) +{ + PyObject *seq1, *seq2; + if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + return NULL; + } + NULLABLE(seq1); + NULLABLE(seq2); + + return PySequence_Concat(seq1, seq2); +} + +static PyObject * +sequence_repeat(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t count; + if (!PyArg_ParseTuple(args, "On", &seq, &count)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_Repeat(seq, count); +} + +static PyObject * +sequence_inplaceconcat(PyObject *self, PyObject *args) +{ + PyObject *seq1, *seq2; + if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + return NULL; + } + NULLABLE(seq1); + NULLABLE(seq2); + + return PySequence_InPlaceConcat(seq1, seq2); +} + +static PyObject * +sequence_inplacerepeat(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t count; + if (!PyArg_ParseTuple(args, "On", &seq, &count)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_InPlaceRepeat(seq, count); +} + +static PyObject * +sequence_getitem(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &seq, &i)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_GetItem(seq, i); +} + +static PyObject * +sequence_setitem(PyObject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *seq, *val; + if (!PyArg_ParseTuple(args, "OnO", &seq, &i, &val)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(val); + + RETURN_INT(PySequence_SetItem(seq, i, val)); +} + + +static PyObject * +sequence_delitem(PyObject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *seq; + if (!PyArg_ParseTuple(args, "On", &seq, &i)) { + return NULL; + } + NULLABLE(seq); + + RETURN_INT(PySequence_DelItem(seq, i)); +} + +static PyObject * +sequence_setslice(PyObject* self, PyObject *args) +{ + PyObject *sequence, *obj; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "OnnO", &sequence, &i1, &i2, &obj)) { + return NULL; + } + NULLABLE(sequence); + NULLABLE(obj); + + RETURN_INT(PySequence_SetSlice(sequence, i1, i2, obj)); +} + +static PyObject * +sequence_delslice(PyObject *self, PyObject *args) +{ + PyObject *sequence; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "Onn", &sequence, &i1, &i2)) { + return NULL; + } + NULLABLE(sequence); + + RETURN_INT(PySequence_DelSlice(sequence, i1, i2)); +} + +static PyObject * +sequence_count(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_SIZE(PySequence_Count(seq, value)); +} + +static PyObject * +sequence_contains(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_INT(PySequence_Contains(seq, value)); +} + +static PyObject * +sequence_index(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_SIZE(PySequence_Index(seq, value)); +} + +static PyObject * +sequence_list(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySequence_List(obj); +} + +static PyObject * +sequence_tuple(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySequence_Tuple(obj); +} + + +static PyMethodDef test_methods[] = { + {"object_getattr", object_getattr, METH_VARARGS}, + {"object_getattrstring", object_getattrstring, METH_VARARGS}, + {"object_getoptionalattr", object_getoptionalattr, METH_VARARGS}, + {"object_getoptionalattrstring", object_getoptionalattrstring, METH_VARARGS}, + {"object_hasattr", object_hasattr, METH_VARARGS}, + {"object_hasattrstring", object_hasattrstring, METH_VARARGS}, + {"object_setattr", object_setattr, METH_VARARGS}, + {"object_setattrstring", object_setattrstring, METH_VARARGS}, + {"object_delattr", object_delattr, METH_VARARGS}, + {"object_delattrstring", object_delattrstring, METH_VARARGS}, + + {"mapping_check", mapping_check, METH_O}, + {"mapping_size", mapping_size, METH_O}, + {"mapping_length", mapping_length, METH_O}, + {"object_getitem", object_getitem, METH_VARARGS}, + {"mapping_getitemstring", mapping_getitemstring, METH_VARARGS}, + {"mapping_getoptionalitem", mapping_getoptionalitem, METH_VARARGS}, + {"mapping_getoptionalitemstring", mapping_getoptionalitemstring, METH_VARARGS}, + {"mapping_haskey", mapping_haskey, METH_VARARGS}, + {"mapping_haskeystring", mapping_haskeystring, METH_VARARGS}, + {"object_setitem", object_setitem, METH_VARARGS}, + {"mapping_setitemstring", mapping_setitemstring, METH_VARARGS}, + {"object_delitem", object_delitem, METH_VARARGS}, + {"mapping_delitem", mapping_delitem, METH_VARARGS}, + {"mapping_delitemstring", mapping_delitemstring, METH_VARARGS}, + {"mapping_keys", mapping_keys, METH_O}, + {"mapping_values", mapping_values, METH_O}, + {"mapping_items", mapping_items, METH_O}, + + {"sequence_check", sequence_check, METH_O}, + {"sequence_size", sequence_size, METH_O}, + {"sequence_length", sequence_length, METH_O}, + {"sequence_concat", sequence_concat, METH_VARARGS}, + {"sequence_repeat", sequence_repeat, METH_VARARGS}, + {"sequence_inplaceconcat", sequence_inplaceconcat, METH_VARARGS}, + {"sequence_inplacerepeat", sequence_inplacerepeat, METH_VARARGS}, + {"sequence_getitem", sequence_getitem, METH_VARARGS}, + {"sequence_setitem", sequence_setitem, METH_VARARGS}, + {"sequence_delitem", sequence_delitem, METH_VARARGS}, + {"sequence_setslice", sequence_setslice, METH_VARARGS}, + {"sequence_delslice", sequence_delslice, METH_VARARGS}, + {"sequence_count", sequence_count, METH_VARARGS}, + {"sequence_contains", sequence_contains, METH_VARARGS}, + {"sequence_index", sequence_index, METH_VARARGS}, + {"sequence_list", sequence_list, METH_O}, + {"sequence_tuple", sequence_tuple, METH_O}, + + {NULL}, +}; + +int +_PyTestCapi_Init_Abstract(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c new file mode 100644 index 0000000000000..b1dfcf4c707da --- /dev/null +++ b/Modules/_testcapi/dict.c @@ -0,0 +1,376 @@ +#include // ptrdiff_t + +#include "parts.h" + +#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromLong(_ret); \ + } while (0) + +#define RETURN_SIZE(value) do { \ + Py_ssize_t _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromSsize_t(_ret); \ + } while (0) + + +static PyObject * +dict_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyDict_Check(obj)); +} + +static PyObject * +dict_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyDict_CheckExact(obj)); +} + +static PyObject * +dict_new(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyDict_New(); +} + +static PyObject * +dictproxy_new(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDictProxy_New(obj); +} + +static PyObject * +dict_clear(PyObject *self, PyObject *obj) +{ + PyDict_Clear(obj); + Py_RETURN_NONE; +} + +static PyObject * +dict_copy(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Copy(obj); +} + +static PyObject * +dict_contains(PyObject *self, PyObject *args) +{ + PyObject *obj, *key; + if (!PyArg_ParseTuple(args, "OO", &obj, &key)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(key); + RETURN_INT(PyDict_Contains(obj, key)); +} + +static PyObject * +dict_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyDict_Size(obj)); +} + +static PyObject * +dict_getitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + PyObject *value = PyDict_GetItem(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + +static PyObject * +dict_getitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + PyObject *value = PyDict_GetItemString(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + +static PyObject * +dict_getitemwitherror(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + PyObject *value = PyDict_GetItemWithError(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + + +static PyObject * +dict_getitemref(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name, *value; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + + switch (PyDict_GetItemRef(obj, attr_name, &value)) { + case -1: + assert(value == NULL); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_KeyError); + case 1: + return value; + default: + Py_FatalError("PyMapping_GetItemRef() returned invalid code"); + Py_UNREACHABLE(); + } +} + +static PyObject * +dict_getitemstringref(PyObject *self, PyObject *args) +{ + PyObject *obj, *value; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + + switch (PyDict_GetItemStringRef(obj, attr_name, &value)) { + case -1: + assert(value == NULL); + return NULL; + case 0: + assert(value == NULL); + return Py_NewRef(PyExc_KeyError); + case 1: + return value; + default: + Py_FatalError("PyDict_GetItemStringRef() returned invalid code"); + Py_UNREACHABLE(); + } +} + +static PyObject * +dict_setitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(value); + RETURN_INT(PyDict_SetItem(mapping, key, value)); +} + +static PyObject * +dict_setitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping, *value; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(value); + RETURN_INT(PyDict_SetItemString(mapping, key, value)); +} + +static PyObject * +dict_setdefault(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *defaultobj; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &defaultobj)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(defaultobj); + return PyDict_SetDefault(mapping, key, defaultobj); +} + +static PyObject * +dict_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyDict_DelItem(mapping, key)); +} + +static PyObject * +dict_delitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyDict_DelItemString(mapping, key)); +} + +static PyObject * +dict_keys(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Keys(obj); +} + +static PyObject * +dict_values(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Values(obj); +} + +static PyObject * +dict_items(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Items(obj); +} + +static PyObject * +dict_next(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + Py_ssize_t pos; + if (!PyArg_ParseTuple(args, "On", &mapping, &pos)) { + return NULL; + } + NULLABLE(mapping); + int rc = PyDict_Next(mapping, &pos, &key, &value); + if (rc != 0) { + return Py_BuildValue("inOO", rc, pos, key, value); + } + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +dict_merge(PyObject *self, PyObject *args) +{ + PyObject *mapping, *mapping2; + int override; + if (!PyArg_ParseTuple(args, "OOi", &mapping, &mapping2, &override)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(mapping2); + RETURN_INT(PyDict_Merge(mapping, mapping2, override)); +} + +static PyObject * +dict_update(PyObject *self, PyObject *args) +{ + PyObject *mapping, *mapping2; + if (!PyArg_ParseTuple(args, "OO", &mapping, &mapping2)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(mapping2); + RETURN_INT(PyDict_Update(mapping, mapping2)); +} + +static PyObject * +dict_mergefromseq2(PyObject *self, PyObject *args) +{ + PyObject *mapping, *seq; + int override; + if (!PyArg_ParseTuple(args, "OOi", &mapping, &seq, &override)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(seq); + RETURN_INT(PyDict_MergeFromSeq2(mapping, seq, override)); +} + + +static PyMethodDef test_methods[] = { + {"dict_check", dict_check, METH_O}, + {"dict_checkexact", dict_checkexact, METH_O}, + {"dict_new", dict_new, METH_NOARGS}, + {"dictproxy_new", dictproxy_new, METH_O}, + {"dict_clear", dict_clear, METH_O}, + {"dict_copy", dict_copy, METH_O}, + {"dict_size", dict_size, METH_O}, + {"dict_getitem", dict_getitem, METH_VARARGS}, + {"dict_getitemwitherror", dict_getitemwitherror, METH_VARARGS}, + {"dict_getitemstring", dict_getitemstring, METH_VARARGS}, + {"dict_getitemref", dict_getitemref, METH_VARARGS}, + {"dict_getitemstringref", dict_getitemstringref, METH_VARARGS}, + {"dict_contains", dict_contains, METH_VARARGS}, + {"dict_setitem", dict_setitem, METH_VARARGS}, + {"dict_setitemstring", dict_setitemstring, METH_VARARGS}, + {"dict_delitem", dict_delitem, METH_VARARGS}, + {"dict_delitemstring", dict_delitemstring, METH_VARARGS}, + {"dict_setdefault", dict_setdefault, METH_VARARGS}, + {"dict_keys", dict_keys, METH_O}, + {"dict_values", dict_values, METH_O}, + {"dict_items", dict_items, METH_O}, + {"dict_next", dict_next, METH_VARARGS}, + {"dict_merge", dict_merge, METH_VARARGS}, + {"dict_update", dict_update, METH_VARARGS}, + {"dict_mergefromseq2", dict_mergefromseq2, METH_VARARGS}, + + {NULL}, +}; + +int +_PyTestCapi_Init_Dict(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index aaec0a6191694..65ebf80bcd1e9 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -26,6 +26,7 @@ int _PyTestCapi_Init_Vectorcall(PyObject *module); int _PyTestCapi_Init_Heaptype(PyObject *module); +int _PyTestCapi_Init_Abstract(PyObject *module); int _PyTestCapi_Init_Unicode(PyObject *module); int _PyTestCapi_Init_GetArgs(PyObject *module); int _PyTestCapi_Init_DateTime(PyObject *module); @@ -34,6 +35,7 @@ int _PyTestCapi_Init_Mem(PyObject *module); int _PyTestCapi_Init_Watchers(PyObject *module); int _PyTestCapi_Init_Long(PyObject *module); int _PyTestCapi_Init_Float(PyObject *module); +int _PyTestCapi_Init_Dict(PyObject *module); int _PyTestCapi_Init_Structmember(PyObject *module); int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 2286a925e36e2..23dd2cc9ad40f 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1983,7 +1983,7 @@ return_result_with_error(PyObject *self, PyObject *args) Py_RETURN_NONE; } -static PyObject* +static PyObject * getitem_with_error(PyObject *self, PyObject *args) { PyObject *map, *key; @@ -2060,90 +2060,6 @@ py_w_stopcode(PyObject *self, PyObject *args) #endif -static PyObject * -get_mapping_keys(PyObject* self, PyObject *obj) -{ - return PyMapping_Keys(obj); -} - -static PyObject * -get_mapping_values(PyObject* self, PyObject *obj) -{ - return PyMapping_Values(obj); -} - -static PyObject * -get_mapping_items(PyObject* self, PyObject *obj) -{ - return PyMapping_Items(obj); -} - -static PyObject * -test_mapping_has_key_string(PyObject *self, PyObject *Py_UNUSED(args)) -{ - PyObject *context = PyDict_New(); - PyObject *val = PyLong_FromLong(1); - - // Since this uses `const char*` it is easier to test this in C: - PyDict_SetItemString(context, "a", val); - if (!PyMapping_HasKeyString(context, "a")) { - PyErr_SetString(PyExc_RuntimeError, - "Existing mapping key does not exist"); - return NULL; - } - if (PyMapping_HasKeyString(context, "b")) { - PyErr_SetString(PyExc_RuntimeError, - "Missing mapping key exists"); - return NULL; - } - - Py_DECREF(val); - Py_DECREF(context); - Py_RETURN_NONE; -} - -static PyObject * -mapping_has_key(PyObject* self, PyObject *args) -{ - PyObject *context, *key; - if (!PyArg_ParseTuple(args, "OO", &context, &key)) { - return NULL; - } - return PyLong_FromLong(PyMapping_HasKey(context, key)); -} - -static PyObject * -sequence_set_slice(PyObject* self, PyObject *args) -{ - PyObject *sequence, *obj; - Py_ssize_t i1, i2; - if (!PyArg_ParseTuple(args, "OnnO", &sequence, &i1, &i2, &obj)) { - return NULL; - } - - int res = PySequence_SetSlice(sequence, i1, i2, obj); - if (res == -1) { - return NULL; - } - Py_RETURN_NONE; -} - -static PyObject * -sequence_del_slice(PyObject* self, PyObject *args) -{ - PyObject *sequence; - Py_ssize_t i1, i2; - if (!PyArg_ParseTuple(args, "Onn", &sequence, &i1, &i2)) { - return NULL; - } - - int res = PySequence_DelSlice(sequence, i1, i2); - if (res == -1) { - return NULL; - } - Py_RETURN_NONE; -} - static PyObject * test_pythread_tss_key_state(PyObject *self, PyObject *args) { @@ -2247,72 +2163,6 @@ negative_refcount(PyObject *self, PyObject *Py_UNUSED(args)) #endif -static PyObject * -sequence_getitem(PyObject *self, PyObject *args) -{ - PyObject *seq; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "On", &seq, &i)) { - return NULL; - } - return PySequence_GetItem(seq, i); -} - - -static PyObject * -sequence_setitem(PyObject *self, PyObject *args) -{ - Py_ssize_t i; - PyObject *seq, *val; - if (!PyArg_ParseTuple(args, "OnO", &seq, &i, &val)) { - return NULL; - } - if (PySequence_SetItem(seq, i, val)) { - return NULL; - } - Py_RETURN_NONE; -} - - -static PyObject * -sequence_delitem(PyObject *self, PyObject *args) -{ - Py_ssize_t i; - PyObject *seq; - if (!PyArg_ParseTuple(args, "On", &seq, &i)) { - return NULL; - } - if (PySequence_DelItem(seq, i)) { - return NULL; - } - Py_RETURN_NONE; -} - -static PyObject * -hasattr_string(PyObject *self, PyObject* args) -{ - PyObject* obj; - PyObject* attr_name; - - if (!PyArg_UnpackTuple(args, "hasattr_string", 2, 2, &obj, &attr_name)) { - return NULL; - } - - if (!PyUnicode_Check(attr_name)) { - PyErr_SetString(PyExc_TypeError, "attribute name must a be string"); - return PyErr_Occurred(); - } - - const char *name_str = PyUnicode_AsUTF8(attr_name); - if (PyObject_HasAttrString(obj, name_str)) { - Py_RETURN_TRUE; - } - else { - Py_RETURN_FALSE; - } -} - - /* Functions for testing C calling conventions (METH_*) are named meth_*, * e.g. "meth_varargs" for METH_VARARGS. * @@ -3736,23 +3586,12 @@ static PyMethodDef TestMethods[] = { #ifdef W_STOPCODE {"W_STOPCODE", py_w_stopcode, METH_VARARGS}, #endif - {"get_mapping_keys", get_mapping_keys, METH_O}, - {"get_mapping_values", get_mapping_values, METH_O}, - {"get_mapping_items", get_mapping_items, METH_O}, - {"test_mapping_has_key_string", test_mapping_has_key_string, METH_NOARGS}, - {"mapping_has_key", mapping_has_key, METH_VARARGS}, - {"sequence_set_slice", sequence_set_slice, METH_VARARGS}, - {"sequence_del_slice", sequence_del_slice, METH_VARARGS}, {"test_pythread_tss_key_state", test_pythread_tss_key_state, METH_VARARGS}, {"hamt", new_hamt, METH_NOARGS}, {"bad_get", _PyCFunction_CAST(bad_get), METH_FASTCALL}, #ifdef Py_REF_DEBUG {"negative_refcount", negative_refcount, METH_NOARGS}, #endif - {"sequence_getitem", sequence_getitem, METH_VARARGS}, - {"sequence_setitem", sequence_setitem, METH_VARARGS}, - {"sequence_delitem", sequence_delitem, METH_VARARGS}, - {"hasattr_string", hasattr_string, METH_VARARGS}, {"meth_varargs", meth_varargs, METH_VARARGS}, {"meth_varargs_keywords", _PyCFunction_CAST(meth_varargs_keywords), METH_VARARGS|METH_KEYWORDS}, {"meth_o", meth_o, METH_O}, @@ -4394,6 +4233,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Heaptype(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Abstract(m) < 0) { + return NULL; + } if (_PyTestCapi_Init_Unicode(m) < 0) { return NULL; } @@ -4418,6 +4260,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Float(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Dict(m) < 0) { + return NULL; + } if (_PyTestCapi_Init_Structmember(m) < 0) { return NULL; } diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index de17d74c52e56..8c0fd0cf052b0 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -99,7 +99,9 @@ + + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 637f7178d39d0..87d33ebe28e47 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -27,9 +27,15 @@ Source Files + + Source Files + Source Files + + Source Files + Source Files From webhook-mailer at python.org Mon Aug 7 15:15:00 2023 From: webhook-mailer at python.org (ericsnowcurrently) Date: Mon, 07 Aug 2023 19:15:00 -0000 Subject: [Python-checkins] gh-107630: Initialize Each Interpreter's refchain Properly (gh-107733) Message-ID: https://github.com/python/cpython/commit/430632d6f710c99879c5d1736f3b40ea09b11c4d commit: 430632d6f710c99879c5d1736f3b40ea09b11c4d branch: main author: Eric Snow committer: ericsnowcurrently date: 2023-08-07T13:14:56-06:00 summary: gh-107630: Initialize Each Interpreter's refchain Properly (gh-107733) This finishes fixing the crashes in Py_TRACE_REFS builds. We missed this part in gh-107567. files: M Include/internal/pycore_object.h M Objects/object.c M Python/pylifecycle.c M Python/pystate.c diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 76b3cd69cbf5a..7cdf64bcdacb9 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -173,6 +173,7 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { extern void _PyType_InitCache(PyInterpreterState *interp); +extern void _PyObject_InitState(PyInterpreterState *interp); /* Inline functions trading binary compatibility for speed: _PyObject_Init() is the fast version of PyObject_Init(), and diff --git a/Objects/object.c b/Objects/object.c index c4413a00ede80..d1154eb344fab 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -162,6 +162,14 @@ _PyDebug_PrintTotalRefs(void) { #define REFCHAIN(interp) &interp->object_state.refchain +static inline void +init_refchain(PyInterpreterState *interp) +{ + PyObject *refchain = REFCHAIN(interp); + refchain->_ob_prev = refchain; + refchain->_ob_next = refchain; +} + /* Insert op at the front of the list of all objects. If force is true, * op is added even if _ob_prev and _ob_next are non-NULL already. If * force is false amd _ob_prev or _ob_next are non-NULL, do nothing. @@ -2019,6 +2027,18 @@ PyObject _Py_NotImplementedStruct = { &_PyNotImplemented_Type }; + +void +_PyObject_InitState(PyInterpreterState *interp) +{ +#ifdef Py_TRACE_REFS + if (!_Py_IsMainInterpreter(interp)) { + init_refchain(interp); + } +#endif +} + + extern PyTypeObject _Py_GenericAliasIterType; extern PyTypeObject _PyMemoryIter_Type; extern PyTypeObject _PyLineIterator; @@ -2326,7 +2346,7 @@ _Py_GetObjects(PyObject *self, PyObject *args) #undef REFCHAIN -#endif +#endif /* Py_TRACE_REFS */ /* Hack to force loading of abstract.o */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 7a17f92b550c0..0de3abf940789 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2075,6 +2075,8 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) } has_gil = 1; + /* No objects have been created yet. */ + status = pycore_interp_init(tstate); if (_PyStatus_EXCEPTION(status)) { goto error; diff --git a/Python/pystate.c b/Python/pystate.c index a1864cc3249e2..3a05cb0fa7988 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -674,6 +674,7 @@ init_interpreter(PyInterpreterState *interp, _obmalloc_pools_INIT(interp->obmalloc.pools); memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp)); } + _PyObject_InitState(interp); _PyEval_InitState(interp, pending_lock); _PyGC_InitState(&interp->gc); From webhook-mailer at python.org Mon Aug 7 15:29:05 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 07 Aug 2023 19:29:05 -0000 Subject: [Python-checkins] gh-107735: Add C API tests for PySys_GetObject() and PySys_SetObject() (GH-107736) Message-ID: https://github.com/python/cpython/commit/bea5f93196d213d6fbf4ba8984caf4c3cd1da882 commit: bea5f93196d213d6fbf4ba8984caf4c3cd1da882 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-07T22:29:01+03:00 summary: gh-107735: Add C API tests for PySys_GetObject() and PySys_SetObject() (GH-107736) files: M Lib/test/test_capi/test_misc.py M Modules/_testcapimodule.c diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 4ab1692447804..844dd2a102096 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -51,6 +51,8 @@ import _testinternalcapi +NULL = None + def decode_stderr(err): return err.decode('utf-8', 'replace').replace('\r', '') @@ -2576,5 +2578,46 @@ def testfunc(it): with self.assertRaises(StopIteration): next(it) + def test_sys_getobject(self): + getobject = _testcapi.sys_getobject + + self.assertIs(getobject(b'stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(getobject('\U0001f40d'.encode()), 42) + + self.assertIs(getobject(b'nonexisting'), AttributeError) + self.assertIs(getobject(b'\xff'), AttributeError) + # CRASHES getobject(NULL) + + def test_sys_setobject(self): + setobject = _testcapi.sys_setobject + + value = ['value'] + value2 = ['value2'] + try: + self.assertEqual(setobject(b'newattr', value), 0) + self.assertIs(sys.newattr, value) + self.assertEqual(setobject(b'newattr', value2), 0) + self.assertIs(sys.newattr, value2) + self.assertEqual(setobject(b'newattr', NULL), 0) + self.assertFalse(hasattr(sys, 'newattr')) + self.assertEqual(setobject(b'newattr', NULL), 0) + finally: + with contextlib.suppress(AttributeError): + del sys.newattr + try: + self.assertEqual(setobject('\U0001f40d'.encode(), value), 0) + self.assertIs(getattr(sys, '\U0001f40d'), value) + self.assertEqual(setobject('\U0001f40d'.encode(), NULL), 0) + self.assertFalse(hasattr(sys, '\U0001f40d')) + finally: + with contextlib.suppress(AttributeError): + delattr(sys, '\U0001f40d') + + with self.assertRaises(UnicodeDecodeError): + setobject(b'\xff', value) + # CRASHES setobject(NULL, value) + + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 23dd2cc9ad40f..35599f8baa204 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -44,6 +44,16 @@ // Include definitions from there. #include "_testcapi/parts.h" +#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromLong(_ret); \ + } while (0) + // Forward declarations static struct PyModuleDef _testcapimodule; static PyObject *TestError; /* set to exception object in init */ @@ -3505,6 +3515,35 @@ test_dict_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) } +static PyObject * +sys_getobject(PyObject *Py_UNUSED(module), PyObject *arg) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_Parse(arg, "z#", &name, &size)) { + return NULL; + } + PyObject *result = PySys_GetObject(name); + if (result == NULL) { + result = PyExc_AttributeError; + } + return Py_NewRef(result); +} + +static PyObject * +sys_setobject(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *value; + if (!PyArg_ParseTuple(args, "z#O", &name, &size, &value)) { + return NULL; + } + NULLABLE(value); + RETURN_INT(PySys_SetObject(name, value)); +} + + static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, @@ -3640,6 +3679,8 @@ static PyMethodDef TestMethods[] = { {"check_pyimport_addmodule", check_pyimport_addmodule, METH_VARARGS}, {"test_weakref_capi", test_weakref_capi, METH_NOARGS}, {"test_dict_capi", test_dict_capi, METH_NOARGS}, + {"sys_getobject", sys_getobject, METH_O}, + {"sys_setobject", sys_setobject, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; From webhook-mailer at python.org Mon Aug 7 16:34:58 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 07 Aug 2023 20:34:58 -0000 Subject: [Python-checkins] gh-86457: Fix signature for code.replace() (GH-23199) Message-ID: https://github.com/python/cpython/commit/0e6e32fb84b2f7cb668e0b9927637587081e38cd commit: 0e6e32fb84b2f7cb668e0b9927637587081e38cd branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-07T23:34:53+03:00 summary: gh-86457: Fix signature for code.replace() (GH-23199) Also add support of @text_signature in Argument Clinic. files: A Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst M Objects/clinic/codeobject.c.h M Objects/codeobject.c M Tools/clinic/clinic.py diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst new file mode 100644 index 0000000000000..41cfd72cb3676 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst @@ -0,0 +1,2 @@ +Argument Clinic now supports overriding automatically generated signature by +using directive `@text_signature`. diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index 1f2ab56775a1e..511a8e4aaffea 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -154,12 +154,7 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(code_replace__doc__, -"replace($self, /, *, co_argcount=-1, co_posonlyargcount=-1,\n" -" co_kwonlyargcount=-1, co_nlocals=-1, co_stacksize=-1,\n" -" co_flags=-1, co_firstlineno=-1, co_code=None, co_consts=None,\n" -" co_names=None, co_varnames=None, co_freevars=None,\n" -" co_cellvars=None, co_filename=None, co_name=None,\n" -" co_qualname=None, co_linetable=None, co_exceptiontable=None)\n" +"replace($self, /, **changes)\n" "--\n" "\n" "Return a copy of the code object with new values for the specified fields."); @@ -171,13 +166,12 @@ static PyObject * code_replace_impl(PyCodeObject *self, int co_argcount, int co_posonlyargcount, int co_kwonlyargcount, int co_nlocals, int co_stacksize, int co_flags, - int co_firstlineno, PyBytesObject *co_code, - PyObject *co_consts, PyObject *co_names, - PyObject *co_varnames, PyObject *co_freevars, - PyObject *co_cellvars, PyObject *co_filename, - PyObject *co_name, PyObject *co_qualname, - PyBytesObject *co_linetable, - PyBytesObject *co_exceptiontable); + int co_firstlineno, PyObject *co_code, PyObject *co_consts, + PyObject *co_names, PyObject *co_varnames, + PyObject *co_freevars, PyObject *co_cellvars, + PyObject *co_filename, PyObject *co_name, + PyObject *co_qualname, PyObject *co_linetable, + PyObject *co_exceptiontable); static PyObject * code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -217,7 +211,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje int co_stacksize = self->co_stacksize; int co_flags = self->co_flags; int co_firstlineno = self->co_firstlineno; - PyBytesObject *co_code = NULL; + PyObject *co_code = NULL; PyObject *co_consts = self->co_consts; PyObject *co_names = self->co_names; PyObject *co_varnames = NULL; @@ -226,8 +220,8 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *co_filename = self->co_filename; PyObject *co_name = self->co_name; PyObject *co_qualname = self->co_qualname; - PyBytesObject *co_linetable = (PyBytesObject *)self->co_linetable; - PyBytesObject *co_exceptiontable = (PyBytesObject *)self->co_exceptiontable; + PyObject *co_linetable = self->co_linetable; + PyObject *co_exceptiontable = self->co_exceptiontable; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); if (!args) { @@ -304,7 +298,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("replace", "argument 'co_code'", "bytes", args[7]); goto exit; } - co_code = (PyBytesObject *)args[7]; + co_code = args[7]; if (!--noptargs) { goto skip_optional_kwonly; } @@ -394,7 +388,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("replace", "argument 'co_linetable'", "bytes", args[16]); goto exit; } - co_linetable = (PyBytesObject *)args[16]; + co_linetable = args[16]; if (!--noptargs) { goto skip_optional_kwonly; } @@ -403,7 +397,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("replace", "argument 'co_exceptiontable'", "bytes", args[17]); goto exit; } - co_exceptiontable = (PyBytesObject *)args[17]; + co_exceptiontable = args[17]; skip_optional_kwonly: return_value = code_replace_impl(self, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, co_stacksize, co_flags, co_firstlineno, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_qualname, co_linetable, co_exceptiontable); @@ -470,4 +464,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=4ca4c0c403dbfa71 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=16c95266bbc4bc03 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 81fa33962e010..6987a2382d81c 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1968,27 +1968,28 @@ code_linesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args)) } /*[clinic input] + at text_signature "($self, /, **changes)" code.replace * - co_argcount: int(c_default="self->co_argcount") = -1 - co_posonlyargcount: int(c_default="self->co_posonlyargcount") = -1 - co_kwonlyargcount: int(c_default="self->co_kwonlyargcount") = -1 - co_nlocals: int(c_default="self->co_nlocals") = -1 - co_stacksize: int(c_default="self->co_stacksize") = -1 - co_flags: int(c_default="self->co_flags") = -1 - co_firstlineno: int(c_default="self->co_firstlineno") = -1 - co_code: PyBytesObject(c_default="NULL") = None - co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = None - co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = None - co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = None - co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None - co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None - co_filename: unicode(c_default="self->co_filename") = None - co_name: unicode(c_default="self->co_name") = None - co_qualname: unicode(c_default="self->co_qualname") = None - co_linetable: PyBytesObject(c_default="(PyBytesObject *)self->co_linetable") = None - co_exceptiontable: PyBytesObject(c_default="(PyBytesObject *)self->co_exceptiontable") = None + co_argcount: int(c_default="self->co_argcount") = unchanged + co_posonlyargcount: int(c_default="self->co_posonlyargcount") = unchanged + co_kwonlyargcount: int(c_default="self->co_kwonlyargcount") = unchanged + co_nlocals: int(c_default="self->co_nlocals") = unchanged + co_stacksize: int(c_default="self->co_stacksize") = unchanged + co_flags: int(c_default="self->co_flags") = unchanged + co_firstlineno: int(c_default="self->co_firstlineno") = unchanged + co_code: object(subclass_of="&PyBytes_Type", c_default="NULL") = unchanged + co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = unchanged + co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = unchanged + co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged + co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged + co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged + co_filename: unicode(c_default="self->co_filename") = unchanged + co_name: unicode(c_default="self->co_name") = unchanged + co_qualname: unicode(c_default="self->co_qualname") = unchanged + co_linetable: object(subclass_of="&PyBytes_Type", c_default="self->co_linetable") = unchanged + co_exceptiontable: object(subclass_of="&PyBytes_Type", c_default="self->co_exceptiontable") = unchanged Return a copy of the code object with new values for the specified fields. [clinic start generated code]*/ @@ -1997,14 +1998,13 @@ static PyObject * code_replace_impl(PyCodeObject *self, int co_argcount, int co_posonlyargcount, int co_kwonlyargcount, int co_nlocals, int co_stacksize, int co_flags, - int co_firstlineno, PyBytesObject *co_code, - PyObject *co_consts, PyObject *co_names, - PyObject *co_varnames, PyObject *co_freevars, - PyObject *co_cellvars, PyObject *co_filename, - PyObject *co_name, PyObject *co_qualname, - PyBytesObject *co_linetable, - PyBytesObject *co_exceptiontable) -/*[clinic end generated code: output=b6cd9988391d5711 input=f6f68e03571f8d7c]*/ + int co_firstlineno, PyObject *co_code, PyObject *co_consts, + PyObject *co_names, PyObject *co_varnames, + PyObject *co_freevars, PyObject *co_cellvars, + PyObject *co_filename, PyObject *co_name, + PyObject *co_qualname, PyObject *co_linetable, + PyObject *co_exceptiontable) +/*[clinic end generated code: output=e75c48a15def18b9 input=18e280e07846c122]*/ { #define CHECK_INT_ARG(ARG) \ if (ARG < 0) { \ @@ -2029,7 +2029,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount, if (code == NULL) { return NULL; } - co_code = (PyBytesObject *)code; + co_code = code; } if (PySys_Audit("code.__new__", "OOOiiiiii", @@ -2068,10 +2068,10 @@ code_replace_impl(PyCodeObject *self, int co_argcount, co = PyCode_NewWithPosOnlyArgs( co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, - co_stacksize, co_flags, (PyObject*)co_code, co_consts, co_names, + co_stacksize, co_flags, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_qualname, co_firstlineno, - (PyObject*)co_linetable, (PyObject*)co_exceptiontable); + co_linetable, co_exceptiontable); error: Py_XDECREF(code); diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 3a8431cb67efe..57ad4e41ba24a 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4595,6 +4595,7 @@ def reset(self) -> None: self.indent = IndentStack() self.kind = CALLABLE self.coexist = False + self.forced_text_signature: str | None = None self.parameter_continuation = '' self.preserve_output = False @@ -4735,6 +4736,11 @@ def at_coexist(self) -> None: fail("Called @coexist twice!") self.coexist = True + def at_text_signature(self, text_signature: str) -> None: + if self.forced_text_signature: + fail("Called @text_signature twice!") + self.forced_text_signature = text_signature + def parse(self, block: Block) -> None: self.reset() self.block = block @@ -5515,142 +5521,145 @@ def format_docstring(self) -> str: add(f.cls.name) else: add(f.name) - add('(') - - # populate "right_bracket_count" field for every parameter - assert parameters, "We should always have a self parameter. " + repr(f) - assert isinstance(parameters[0].converter, self_converter) - # self is always positional-only. - assert parameters[0].is_positional_only() - assert parameters[0].right_bracket_count == 0 - positional_only = True - for p in parameters[1:]: - if not p.is_positional_only(): - positional_only = False - else: - assert positional_only - if positional_only: - p.right_bracket_count = abs(p.group) - else: - # don't put any right brackets around non-positional-only parameters, ever. - p.right_bracket_count = 0 - - right_bracket_count = 0 - - def fix_right_bracket_count(desired: int) -> str: - nonlocal right_bracket_count - s = '' - while right_bracket_count < desired: - s += '[' - right_bracket_count += 1 - while right_bracket_count > desired: - s += ']' - right_bracket_count -= 1 - return s - - need_slash = False - added_slash = False - need_a_trailing_slash = False - - # we only need a trailing slash: - # * if this is not a "docstring_only" signature - # * and if the last *shown* parameter is - # positional only - if not f.docstring_only: - for p in reversed(parameters): - if not p.converter.show_in_signature: - continue - if p.is_positional_only(): - need_a_trailing_slash = True - break + if self.forced_text_signature: + add(self.forced_text_signature) + else: + add('(') + + # populate "right_bracket_count" field for every parameter + assert parameters, "We should always have a self parameter. " + repr(f) + assert isinstance(parameters[0].converter, self_converter) + # self is always positional-only. + assert parameters[0].is_positional_only() + assert parameters[0].right_bracket_count == 0 + positional_only = True + for p in parameters[1:]: + if not p.is_positional_only(): + positional_only = False + else: + assert positional_only + if positional_only: + p.right_bracket_count = abs(p.group) + else: + # don't put any right brackets around non-positional-only parameters, ever. + p.right_bracket_count = 0 + + right_bracket_count = 0 + + def fix_right_bracket_count(desired: int) -> str: + nonlocal right_bracket_count + s = '' + while right_bracket_count < desired: + s += '[' + right_bracket_count += 1 + while right_bracket_count > desired: + s += ']' + right_bracket_count -= 1 + return s + + need_slash = False + added_slash = False + need_a_trailing_slash = False + + # we only need a trailing slash: + # * if this is not a "docstring_only" signature + # * and if the last *shown* parameter is + # positional only + if not f.docstring_only: + for p in reversed(parameters): + if not p.converter.show_in_signature: + continue + if p.is_positional_only(): + need_a_trailing_slash = True + break - added_star = False + added_star = False - first_parameter = True - last_p = parameters[-1] - line_length = len(''.join(text)) - indent = " " * line_length - def add_parameter(text: str) -> None: - nonlocal line_length - nonlocal first_parameter - if first_parameter: - s = text - first_parameter = False - else: - s = ' ' + text - if line_length + len(s) >= 72: - add('\n') - add(indent) - line_length = len(indent) + first_parameter = True + last_p = parameters[-1] + line_length = len(''.join(text)) + indent = " " * line_length + def add_parameter(text: str) -> None: + nonlocal line_length + nonlocal first_parameter + if first_parameter: s = text - line_length += len(s) - add(s) - - for p in parameters: - if not p.converter.show_in_signature: - continue - assert p.name + first_parameter = False + else: + s = ' ' + text + if line_length + len(s) >= 72: + add('\n') + add(indent) + line_length = len(indent) + s = text + line_length += len(s) + add(s) + + for p in parameters: + if not p.converter.show_in_signature: + continue + assert p.name - is_self = isinstance(p.converter, self_converter) - if is_self and f.docstring_only: - # this isn't a real machine-parsable signature, - # so let's not print the "self" parameter - continue + is_self = isinstance(p.converter, self_converter) + if is_self and f.docstring_only: + # this isn't a real machine-parsable signature, + # so let's not print the "self" parameter + continue - if p.is_positional_only(): - need_slash = not f.docstring_only - elif need_slash and not (added_slash or p.is_positional_only()): - added_slash = True - add_parameter('/,') - - if p.is_keyword_only() and not added_star: - added_star = True - add_parameter('*,') - - p_add, p_output = text_accumulator() - p_add(fix_right_bracket_count(p.right_bracket_count)) - - if isinstance(p.converter, self_converter): - # annotate first parameter as being a "self". - # - # if inspect.Signature gets this function, - # and it's already bound, the self parameter - # will be stripped off. - # - # if it's not bound, it should be marked - # as positional-only. - # - # note: we don't print "self" for __init__, - # because this isn't actually the signature - # for __init__. (it can't be, __init__ doesn't - # have a docstring.) if this is an __init__ - # (or __new__), then this signature is for - # calling the class to construct a new instance. - p_add('$') + if p.is_positional_only(): + need_slash = not f.docstring_only + elif need_slash and not (added_slash or p.is_positional_only()): + added_slash = True + add_parameter('/,') + + if p.is_keyword_only() and not added_star: + added_star = True + add_parameter('*,') + + p_add, p_output = text_accumulator() + p_add(fix_right_bracket_count(p.right_bracket_count)) + + if isinstance(p.converter, self_converter): + # annotate first parameter as being a "self". + # + # if inspect.Signature gets this function, + # and it's already bound, the self parameter + # will be stripped off. + # + # if it's not bound, it should be marked + # as positional-only. + # + # note: we don't print "self" for __init__, + # because this isn't actually the signature + # for __init__. (it can't be, __init__ doesn't + # have a docstring.) if this is an __init__ + # (or __new__), then this signature is for + # calling the class to construct a new instance. + p_add('$') - if p.is_vararg(): - p_add("*") + if p.is_vararg(): + p_add("*") - name = p.converter.signature_name or p.name - p_add(name) + name = p.converter.signature_name or p.name + p_add(name) - if not p.is_vararg() and p.converter.is_optional(): - p_add('=') - value = p.converter.py_default - if not value: - value = repr(p.converter.default) - p_add(value) + if not p.is_vararg() and p.converter.is_optional(): + p_add('=') + value = p.converter.py_default + if not value: + value = repr(p.converter.default) + p_add(value) - if (p != last_p) or need_a_trailing_slash: - p_add(',') + if (p != last_p) or need_a_trailing_slash: + p_add(',') - add_parameter(p_output()) + add_parameter(p_output()) - add(fix_right_bracket_count(0)) - if need_a_trailing_slash: - add_parameter('/') - add(')') + add(fix_right_bracket_count(0)) + if need_a_trailing_slash: + add_parameter('/') + add(')') # PEP 8 says: # From webhook-mailer at python.org Mon Aug 7 16:51:05 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 07 Aug 2023 20:51:05 -0000 Subject: [Python-checkins] [3.11] gh-107735: Add C API tests for PySys_GetObject() and PySys_SetObject() (GH-107736) (GH-107741) Message-ID: https://github.com/python/cpython/commit/22b39d13eccb965515e3a4b3fd358755f8db9d7f commit: 22b39d13eccb965515e3a4b3fd358755f8db9d7f branch: 3.11 author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-07T20:51:00Z summary: [3.11] gh-107735: Add C API tests for PySys_GetObject() and PySys_SetObject() (GH-107736) (GH-107741) (cherry picked from commit bea5f93196d213d6fbf4ba8984caf4c3cd1da882) files: M Lib/test/test_capi/test_misc.py M Modules/_testcapimodule.c diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 3e36fbde8c66d..341b3b79091ae 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -3,6 +3,7 @@ from collections import OrderedDict import _thread +import contextlib import importlib.machinery import importlib.util import os @@ -40,6 +41,8 @@ Py_DEBUG = hasattr(sys, 'gettotalrefcount') +NULL = None + def decode_stderr(err): return err.decode('utf-8', 'replace').replace('\r', '') @@ -910,6 +913,46 @@ def some(): with self.assertRaises(SystemError): _testcapi.function_get_module(None) # not a function + def test_sys_getobject(self): + getobject = _testcapi.sys_getobject + + self.assertIs(getobject(b'stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(getobject('\U0001f40d'.encode()), 42) + + self.assertIs(getobject(b'nonexisting'), AttributeError) + self.assertIs(getobject(b'\xff'), AttributeError) + # CRASHES getobject(NULL) + + def test_sys_setobject(self): + setobject = _testcapi.sys_setobject + + value = ['value'] + value2 = ['value2'] + try: + self.assertEqual(setobject(b'newattr', value), 0) + self.assertIs(sys.newattr, value) + self.assertEqual(setobject(b'newattr', value2), 0) + self.assertIs(sys.newattr, value2) + self.assertEqual(setobject(b'newattr', NULL), 0) + self.assertFalse(hasattr(sys, 'newattr')) + self.assertEqual(setobject(b'newattr', NULL), 0) + finally: + with contextlib.suppress(AttributeError): + del sys.newattr + try: + self.assertEqual(setobject('\U0001f40d'.encode(), value), 0) + self.assertIs(getattr(sys, '\U0001f40d'), value) + self.assertEqual(setobject('\U0001f40d'.encode(), NULL), 0) + self.assertFalse(hasattr(sys, '\U0001f40d')) + finally: + with contextlib.suppress(AttributeError): + delattr(sys, '\U0001f40d') + + with self.assertRaises(UnicodeDecodeError): + setobject(b'\xff', value) + # CRASHES setobject(NULL, value) + class TestPendingCalls(unittest.TestCase): diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b29a919a8325b..5c00b48001a91 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -44,6 +44,16 @@ # error "The public headers should not include , see bpo-46748" #endif +#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromLong(_ret); \ + } while (0) + // Forward declarations static struct PyModuleDef _testcapimodule; static PyType_Spec HeapTypeNameType_Spec; @@ -6449,6 +6459,35 @@ static PyObject *getargs_s_hash_int(PyObject *, PyObject *, PyObject*); static PyObject *getargs_s_hash_int2(PyObject *, PyObject *, PyObject*); static PyObject *gh_99240_clear_args(PyObject *, PyObject *); +static PyObject * +sys_getobject(PyObject *Py_UNUSED(module), PyObject *arg) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_Parse(arg, "z#", &name, &size)) { + return NULL; + } + PyObject *result = PySys_GetObject(name); + if (result == NULL) { + result = PyExc_AttributeError; + } + return Py_NewRef(result); +} + +static PyObject * +sys_setobject(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *value; + if (!PyArg_ParseTuple(args, "z#O", &name, &size, &value)) { + return NULL; + } + NULLABLE(value); + RETURN_INT(PySys_SetObject(name, value)); +} + + static PyMethodDef TestMethods[] = { {"exc_set_object", exc_set_object, METH_VARARGS}, {"raise_exception", raise_exception, METH_VARARGS}, @@ -6761,6 +6800,8 @@ static PyMethodDef TestMethods[] = { {"function_get_code", function_get_code, METH_O, NULL}, {"function_get_globals", function_get_globals, METH_O, NULL}, {"function_get_module", function_get_module, METH_O, NULL}, + {"sys_getobject", sys_getobject, METH_O}, + {"sys_setobject", sys_setobject, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; From webhook-mailer at python.org Mon Aug 7 17:04:15 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 07 Aug 2023 21:04:15 -0000 Subject: [Python-checkins] gh-107735: Move just added C API tests to better place (GH-107743) Message-ID: https://github.com/python/cpython/commit/0191af97a6bf3f720cd0ae69a0bdb14c97351679 commit: 0191af97a6bf3f720cd0ae69a0bdb14c97351679 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-07T21:04:11Z summary: gh-107735: Move just added C API tests to better place (GH-107743) files: M Lib/test/test_capi/test_misc.py diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 844dd2a102096..001d37de8e0eb 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -992,6 +992,46 @@ class Data(_testcapi.ObjExtraData): del d.extra self.assertIsNone(d.extra) + def test_sys_getobject(self): + getobject = _testcapi.sys_getobject + + self.assertIs(getobject(b'stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(getobject('\U0001f40d'.encode()), 42) + + self.assertIs(getobject(b'nonexisting'), AttributeError) + self.assertIs(getobject(b'\xff'), AttributeError) + # CRASHES getobject(NULL) + + def test_sys_setobject(self): + setobject = _testcapi.sys_setobject + + value = ['value'] + value2 = ['value2'] + try: + self.assertEqual(setobject(b'newattr', value), 0) + self.assertIs(sys.newattr, value) + self.assertEqual(setobject(b'newattr', value2), 0) + self.assertIs(sys.newattr, value2) + self.assertEqual(setobject(b'newattr', NULL), 0) + self.assertFalse(hasattr(sys, 'newattr')) + self.assertEqual(setobject(b'newattr', NULL), 0) + finally: + with contextlib.suppress(AttributeError): + del sys.newattr + try: + self.assertEqual(setobject('\U0001f40d'.encode(), value), 0) + self.assertIs(getattr(sys, '\U0001f40d'), value) + self.assertEqual(setobject('\U0001f40d'.encode(), NULL), 0) + self.assertFalse(hasattr(sys, '\U0001f40d')) + finally: + with contextlib.suppress(AttributeError): + delattr(sys, '\U0001f40d') + + with self.assertRaises(UnicodeDecodeError): + setobject(b'\xff', value) + # CRASHES setobject(NULL, value) + @requires_limited_api class TestHeapTypeRelative(unittest.TestCase): @@ -2578,46 +2618,6 @@ def testfunc(it): with self.assertRaises(StopIteration): next(it) - def test_sys_getobject(self): - getobject = _testcapi.sys_getobject - - self.assertIs(getobject(b'stdout'), sys.stdout) - with support.swap_attr(sys, '\U0001f40d', 42): - self.assertEqual(getobject('\U0001f40d'.encode()), 42) - - self.assertIs(getobject(b'nonexisting'), AttributeError) - self.assertIs(getobject(b'\xff'), AttributeError) - # CRASHES getobject(NULL) - - def test_sys_setobject(self): - setobject = _testcapi.sys_setobject - - value = ['value'] - value2 = ['value2'] - try: - self.assertEqual(setobject(b'newattr', value), 0) - self.assertIs(sys.newattr, value) - self.assertEqual(setobject(b'newattr', value2), 0) - self.assertIs(sys.newattr, value2) - self.assertEqual(setobject(b'newattr', NULL), 0) - self.assertFalse(hasattr(sys, 'newattr')) - self.assertEqual(setobject(b'newattr', NULL), 0) - finally: - with contextlib.suppress(AttributeError): - del sys.newattr - try: - self.assertEqual(setobject('\U0001f40d'.encode(), value), 0) - self.assertIs(getattr(sys, '\U0001f40d'), value) - self.assertEqual(setobject('\U0001f40d'.encode(), NULL), 0) - self.assertFalse(hasattr(sys, '\U0001f40d')) - finally: - with contextlib.suppress(AttributeError): - delattr(sys, '\U0001f40d') - - with self.assertRaises(UnicodeDecodeError): - setobject(b'\xff', value) - # CRASHES setobject(NULL, value) - if __name__ == "__main__": unittest.main() From webhook-mailer at python.org Mon Aug 7 17:41:03 2023 From: webhook-mailer at python.org (ericsnowcurrently) Date: Mon, 07 Aug 2023 21:41:03 -0000 Subject: [Python-checkins] gh-98154: Clarify Usage of "Reference Count" In the Docs (gh-107552) Message-ID: https://github.com/python/cpython/commit/5dc825d504ad08d64c9d1ce578f9deebbe012604 commit: 5dc825d504ad08d64c9d1ce578f9deebbe012604 branch: main author: Eric Snow committer: ericsnowcurrently date: 2023-08-07T15:40:59-06:00 summary: gh-98154: Clarify Usage of "Reference Count" In the Docs (gh-107552) PEP 683 (immortal objects) revealed some ways in which the Python documentation has been unnecessarily coupled to the implementation details of reference counts. In the end users should focus on reference ownership, including taking references and releasing them, rather than on how many reference counts an object has. This change updates the documentation to reflect that perspective. It also updates the docs relative to immortal objects in a handful of places. files: M Doc/c-api/allocation.rst M Doc/c-api/arg.rst M Doc/c-api/buffer.rst M Doc/c-api/bytes.rst M Doc/c-api/exceptions.rst M Doc/c-api/init_config.rst M Doc/c-api/intro.rst M Doc/c-api/module.rst M Doc/c-api/object.rst M Doc/c-api/refcounting.rst M Doc/c-api/sys.rst M Doc/c-api/typeobj.rst M Doc/c-api/unicode.rst M Doc/glossary.rst M Doc/library/sys.rst diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 44747e2964366..b3609c233156b 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -29,12 +29,13 @@ Allocating Objects on the Heap .. c:macro:: PyObject_New(TYPE, typeobj) - Allocate a new Python object using the C structure type *TYPE* and the - Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header - are not initialized; the object's reference count will be one. The size of - the memory allocation is determined from the :c:member:`~PyTypeObject.tp_basicsize` field of - the type object. + Allocate a new Python object using the C structure type *TYPE* + and the Python type object *typeobj* (``PyTypeObject*``). + Fields not defined by the Python object header are not initialized. + The caller will own the only reference to the object + (i.e. its reference count will be one). + The size of the memory allocation is determined from the + :c:member:`~PyTypeObject.tp_basicsize` field of the type object. .. c:macro:: PyObject_NewVar(TYPE, typeobj, size) diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index ae321ff918d13..c43dd0f4303cd 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -293,8 +293,10 @@ Other objects ``O`` (object) [PyObject \*] Store a Python object (without any conversion) in a C object pointer. The C - program thus receives the actual object that was passed. The object's reference - count is not increased. The pointer stored is not ``NULL``. + program thus receives the actual object that was passed. A new + :term:`strong reference` to the object is not created + (i.e. its reference count is not increased). + The pointer stored is not ``NULL``. ``O!`` (object) [*typeobject*, PyObject \*] Store a Python object in a C object pointer. This is similar to ``O``, but @@ -378,7 +380,8 @@ inside nested parentheses. They are: mutually exclude each other. Note that any Python object references which are provided to the caller are -*borrowed* references; do not decrement their reference count! +*borrowed* references; do not release them +(i.e. do not decrement their reference count)! Additional arguments passed to these functions must be addresses of variables whose type is determined by the format string; these are used to store values @@ -621,8 +624,10 @@ Building values Convert a C :c:type:`Py_complex` structure to a Python complex number. ``O`` (object) [PyObject \*] - Pass a Python object untouched (except for its reference count, which is - incremented by one). If the object passed in is a ``NULL`` pointer, it is assumed + Pass a Python object untouched but create a new + :term:`strong reference` to it + (i.e. its reference count is incremented by one). + If the object passed in is a ``NULL`` pointer, it is assumed that this was caused because the call producing the argument found an error and set an exception. Therefore, :c:func:`Py_BuildValue` will return ``NULL`` but won't raise an exception. If no exception has been raised yet, :exc:`SystemError` is @@ -632,7 +637,7 @@ Building values Same as ``O``. ``N`` (object) [PyObject \*] - Same as ``O``, except it doesn't increment the reference count on the object. + Same as ``O``, except it doesn't create a new :term:`strong reference`. Useful when the object is created by a call to an object constructor in the argument list. diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 02b53ec149c73..8ca1c190dab9a 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -102,7 +102,9 @@ a buffer, see :c:func:`PyObject_GetBuffer`. .. c:member:: PyObject *obj A new reference to the exporting object. The reference is owned by - the consumer and automatically decremented and set to ``NULL`` by + the consumer and automatically released + (i.e. reference count decremented) + and set to ``NULL`` by :c:func:`PyBuffer_Release`. The field is the equivalent of the return value of any standard C-API function. @@ -454,7 +456,8 @@ Buffer-related functions .. c:function:: void PyBuffer_Release(Py_buffer *view) - Release the buffer *view* and decrement the reference count for + Release the buffer *view* and release the :term:`strong reference` + (i.e. decrement the reference count) to the view's supporting object, ``view->obj``. This function MUST be called when the buffer is no longer being used, otherwise reference leaks may occur. diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 4e3ffc7e23e3f..61a68f5277388 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -184,8 +184,8 @@ called with a non-bytes parameter. .. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) Create a new bytes object in *\*bytes* containing the contents of *newpart* - appended to *bytes*. This version decrements the reference count of - *newpart*. + appended to *bytes*. This version releases the :term:`strong reference` + to *newpart* (i.e. decrements its reference count). .. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index a2126ffc559ab..f1d6c995188ab 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -110,7 +110,8 @@ For convenience, some of these functions will always return a This is the most common way to set the error indicator. The first argument specifies the exception type; it is normally one of the standard exceptions, - e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. + e.g. :c:data:`PyExc_RuntimeError`. You need not create a new + :term:`strong reference` to it (e.g. with :c:func:`Py_INCREF`). The second argument is an error message; it is decoded from ``'utf-8'``. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index ac7b357e08eeb..3ad2d435665f8 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1128,7 +1128,7 @@ PyConfig .. c:member:: int show_ref_count - Show total reference count at exit? + Show total reference count at exit (excluding immortal objects)? Set to ``1`` by :option:`-X showrefcount <-X>` command line option. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 1739eabd2a303..5cf9914be8c8b 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -287,52 +287,58 @@ true if (and only if) the object pointed to by *a* is a Python list. Reference Counts ---------------- -The reference count is important because today's computers have a finite (and -often severely limited) memory size; it counts how many different places there -are that have a reference to an object. Such a place could be another object, -or a global (or static) C variable, or a local variable in some C function. -When an object's reference count becomes zero, the object is deallocated. If -it contains references to other objects, their reference count is decremented. -Those other objects may be deallocated in turn, if this decrement makes their -reference count become zero, and so on. (There's an obvious problem with -objects that reference each other here; for now, the solution is "don't do -that.") +The reference count is important because today's computers have a finite +(and often severely limited) memory size; it counts how many different +places there are that have a :term:`strong reference` to an object. +Such a place could be another object, or a global (or static) C variable, +or a local variable in some C function. +When the last :term:`strong reference` to an object is released +(i.e. its reference count becomes zero), the object is deallocated. +If it contains references to other objects, those references are released. +Those other objects may be deallocated in turn, if there are no more +references to them, and so on. (There's an obvious problem with +objects that reference each other here; for now, the solution +is "don't do that.") .. index:: single: Py_INCREF() single: Py_DECREF() -Reference counts are always manipulated explicitly. The normal way is to use -the macro :c:func:`Py_INCREF` to increment an object's reference count by one, -and :c:func:`Py_DECREF` to decrement it by one. The :c:func:`Py_DECREF` macro +Reference counts are always manipulated explicitly. The normal way is +to use the macro :c:func:`Py_INCREF` to take a new reference to an +object (i.e. increment its reference count by one), +and :c:func:`Py_DECREF` to release that reference (i.e. decrement the +reference count by one). The :c:func:`Py_DECREF` macro is considerably more complex than the incref one, since it must check whether the reference count becomes zero and then cause the object's deallocator to be -called. The deallocator is a function pointer contained in the object's type -structure. The type-specific deallocator takes care of decrementing the -reference counts for other objects contained in the object if this is a compound +called. The deallocator is a function pointer contained in the object's type +structure. The type-specific deallocator takes care of releasing references +for other objects contained in the object if this is a compound object type, such as a list, as well as performing any additional finalization that's needed. There's no chance that the reference count can overflow; at least as many bits are used to hold the reference count as there are distinct memory locations in virtual memory (assuming ``sizeof(Py_ssize_t) >= sizeof(void*)``). Thus, the reference count increment is a simple operation. -It is not necessary to increment an object's reference count for every local -variable that contains a pointer to an object. In theory, the object's +It is not necessary to hold a :term:`strong reference` (i.e. increment +the reference count) for every local variable that contains a pointer +to an object. In theory, the object's reference count goes up by one when the variable is made to point to it and it goes down by one when the variable goes out of scope. However, these two cancel each other out, so at the end the reference count hasn't changed. The only real reason to use the reference count is to prevent the object from being deallocated as long as our variable is pointing to it. If we know that there is at least one other reference to the object that lives at least as long as -our variable, there is no need to increment the reference count temporarily. +our variable, there is no need to take a new :term:`strong reference` +(i.e. increment the reference count) temporarily. An important situation where this arises is in objects that are passed as arguments to C functions in an extension module that are called from Python; the call mechanism guarantees to hold a reference to every argument for the duration of the call. However, a common pitfall is to extract an object from a list and hold on to it -for a while without incrementing its reference count. Some other operation might -conceivably remove the object from the list, decrementing its reference count +for a while without taking a new reference. Some other operation might +conceivably remove the object from the list, releasing that reference, and possibly deallocating it. The real danger is that innocent-looking operations may invoke arbitrary Python code which could do this; there is a code path which allows control to flow back to the user from a :c:func:`Py_DECREF`, so @@ -340,7 +346,8 @@ almost any operation is potentially dangerous. A safe approach is to always use the generic operations (functions whose name begins with ``PyObject_``, ``PyNumber_``, ``PySequence_`` or ``PyMapping_``). -These operations always increment the reference count of the object they return. +These operations always create a new :term:`strong reference` +(i.e. increment the reference count) of the object they return. This leaves the caller with the responsibility to call :c:func:`Py_DECREF` when they are done with the result; this soon becomes second nature. @@ -356,7 +363,7 @@ to objects (objects are not owned: they are always shared). "Owning a reference" means being responsible for calling Py_DECREF on it when the reference is no longer needed. Ownership can also be transferred, meaning that the code that receives ownership of the reference then becomes responsible for -eventually decref'ing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF` +eventually releasing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF` when it's no longer needed---or passing on this responsibility (usually to its caller). When a function passes ownership of a reference on to its caller, the caller is said to receive a *new* reference. When no ownership is transferred, @@ -414,9 +421,9 @@ For example, the above two blocks of code could be replaced by the following It is much more common to use :c:func:`PyObject_SetItem` and friends with items whose references you are only borrowing, like arguments that were passed in to -the function you are writing. In that case, their behaviour regarding reference -counts is much saner, since you don't have to increment a reference count so you -can give a reference away ("have it be stolen"). For example, this function +the function you are writing. In that case, their behaviour regarding references +is much saner, since you don't have to take a new reference just so you +can give that reference away ("have it be stolen"). For example, this function sets all items of a list (actually, any mutable sequence) to a given item:: int diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index e37505f6450a3..187f8419d4ee4 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -546,7 +546,7 @@ state: .. note:: Unlike other functions that steal references, ``PyModule_AddObject()`` - only decrements the reference count of *value* **on success**. + only releases the reference to *value* **on success**. This means that its return value must be checked, and calling code must :c:func:`Py_XDECREF` *value* manually on error. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 1b7e05e7c53ed..6d9fc7f50604d 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -15,8 +15,8 @@ Object Protocol .. c:macro:: Py_RETURN_NOTIMPLEMENTED Properly handle returning :c:data:`Py_NotImplemented` from within a C - function (that is, increment the reference count of NotImplemented and - return it). + function (that is, create a new :term:`strong reference` + to NotImplemented and return it). .. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags) @@ -357,11 +357,12 @@ Object Protocol When *o* is non-``NULL``, returns a type object corresponding to the object type of object *o*. On failure, raises :exc:`SystemError` and returns ``NULL``. This - is equivalent to the Python expression ``type(o)``. This function increments the - reference count of the return value. There's really no reason to use this + is equivalent to the Python expression ``type(o)``. + This function creates a new :term:`strong reference` to the return value. + There's really no reason to use this function instead of the :c:func:`Py_TYPE()` function, which returns a - pointer of type :c:expr:`PyTypeObject*`, except when the incremented reference - count is needed. + pointer of type :c:expr:`PyTypeObject*`, except when a new + :term:`strong reference` is needed. .. c:function:: int PyObject_TypeCheck(PyObject *o, PyTypeObject *type) diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 640c5c610899f..118af7a1a8cf9 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -15,6 +15,12 @@ of Python objects. Get the reference count of the Python object *o*. + Note that the returned value may not actually reflect how many + references to the object are actually held. For example, some + objects are "immortal" and have a very high refcount that does not + reflect the actual number of references. Consequently, do not rely + on the returned value to be accurate, other than a value of 0 or 1. + Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count. .. versionchanged:: 3.11 @@ -28,36 +34,53 @@ of Python objects. Set the object *o* reference counter to *refcnt*. + Note that this function has no effect on + `immortal `_ + objects. + .. versionadded:: 3.9 + .. versionchanged:: 3.12 + Immortal objects are not modified. + .. c:function:: void Py_INCREF(PyObject *o) - Increment the reference count for object *o*. + Indicate taking a new :term:`strong reference` to object *o*, + indicating it is in use and should not be destroyed. This function is usually used to convert a :term:`borrowed reference` to a :term:`strong reference` in-place. The :c:func:`Py_NewRef` function can be used to create a new :term:`strong reference`. + When done using the object, release is by calling :c:func:`Py_DECREF`. + The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, use :c:func:`Py_XINCREF`. + Do not expect this function to actually modify *o* in any way. + For at least `some objects `_, + this function has no effect. + + .. versionchanged:: 3.12 + Immortal objects are not modified. + .. c:function:: void Py_XINCREF(PyObject *o) - Increment the reference count for object *o*. The object may be ``NULL``, in - which case the macro has no effect. + Similar to :c:func:`Py_INCREF`, but the object *o* can be ``NULL``, + in which case this has no effect. See also :c:func:`Py_XNewRef`. .. c:function:: PyObject* Py_NewRef(PyObject *o) - Create a new :term:`strong reference` to an object: increment the reference - count of the object *o* and return the object *o*. + Create a new :term:`strong reference` to an object: + call :c:func:`Py_INCREF` on *o* and return the object *o*. When the :term:`strong reference` is no longer needed, :c:func:`Py_DECREF` - should be called on it to decrement the object reference count. + should be called on it to release the reference. The object *o* must not be ``NULL``; use :c:func:`Py_XNewRef` if *o* can be ``NULL``. @@ -87,9 +110,12 @@ of Python objects. .. c:function:: void Py_DECREF(PyObject *o) - Decrement the reference count for object *o*. + Release a :term:`strong reference` to object *o*, indicating the + reference is no longer used. - If the reference count reaches zero, the object's type's deallocation + Once the last :term:`strong reference` is released + (i.e. the object's reference count reaches 0), + the object's type's deallocation function (which must not be ``NULL``) is invoked. This function is usually used to delete a :term:`strong reference` before @@ -98,6 +124,10 @@ of Python objects. The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, use :c:func:`Py_XDECREF`. + Do not expect this function to actually modify *o* in any way. + For at least `some objects `_, + this function has no effect. + .. warning:: The deallocation function can cause arbitrary Python code to be invoked (e.g. @@ -109,25 +139,29 @@ of Python objects. reference to the deleted object in a temporary variable, update the list data structure, and then call :c:func:`Py_DECREF` for the temporary variable. + .. versionchanged:: 3.12 + Immortal objects are not modified. + .. c:function:: void Py_XDECREF(PyObject *o) - Decrement the reference count for object *o*. The object may be ``NULL``, in - which case the macro has no effect; otherwise the effect is the same as for - :c:func:`Py_DECREF`, and the same warning applies. + Similar to :c:func:`Py_DECREF`, but the object *o* can be ``NULL``, + in which case this has no effect. + The same warning from :c:func:`Py_DECREF` applies here as well. .. c:function:: void Py_CLEAR(PyObject *o) - Decrement the reference count for object *o*. The object may be ``NULL``, in + Release a :term:`strong reference` for object *o*. + The object may be ``NULL``, in which case the macro has no effect; otherwise the effect is the same as for :c:func:`Py_DECREF`, except that the argument is also set to ``NULL``. The warning for :c:func:`Py_DECREF` does not apply with respect to the object passed because the macro carefully uses a temporary variable and sets the argument to ``NULL`` - before decrementing its reference count. + before releasing the reference. - It is a good idea to use this macro whenever decrementing the reference - count of an object that might be traversed during garbage collection. + It is a good idea to use this macro whenever releasing a reference + to an object that might be traversed during garbage collection. .. versionchanged:: 3.12 The macro argument is now only evaluated once. If the argument has side @@ -136,20 +170,22 @@ of Python objects. .. c:function:: void Py_IncRef(PyObject *o) - Increment the reference count for object *o*. A function version of :c:func:`Py_XINCREF`. + Indicate taking a new :term:`strong reference` to object *o*. + A function version of :c:func:`Py_XINCREF`. It can be used for runtime dynamic embedding of Python. .. c:function:: void Py_DecRef(PyObject *o) - Decrement the reference count for object *o*. A function version of :c:func:`Py_XDECREF`. + Release a :term:`strong reference` to object *o*. + A function version of :c:func:`Py_XDECREF`. It can be used for runtime dynamic embedding of Python. .. c:macro:: Py_SETREF(dst, src) - Macro safely decrementing the `dst` reference count and setting `dst` to - `src`. + Macro safely releasing a :term:`strong reference` to object *dst* + and setting *dst* to *src*. As in case of :c:func:`Py_CLEAR`, "the obvious" code can be deadly:: @@ -160,9 +196,10 @@ of Python objects. Py_SETREF(dst, src); - That arranges to set `dst` to `src` _before_ decrementing reference count of - *dst* old value, so that any code triggered as a side-effect of `dst` - getting torn down no longer believes `dst` points to a valid object. + That arranges to set *dst* to *src* _before_ releasing the reference + to the old value of *dst*, so that any code triggered as a side-effect + of *dst* getting torn down no longer believes *dst* points + to a valid object. .. versionadded:: 3.6 diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 5457d17159f8d..aed625c5f6cda 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -8,8 +8,9 @@ Operating System Utilities .. c:function:: PyObject* PyOS_FSPath(PyObject *path) Return the file system representation for *path*. If the object is a - :class:`str` or :class:`bytes` object, then its reference count is - incremented. If the object implements the :class:`os.PathLike` interface, + :class:`str` or :class:`bytes` object, then a new + :term:`strong reference` is returned. + If the object implements the :class:`os.PathLike` interface, then :meth:`~os.PathLike.__fspath__` is returned as long as it is a :class:`str` or :class:`bytes` object. Otherwise :exc:`TypeError` is raised and ``NULL`` is returned. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 26e6133aebaa8..221a05b192240 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -690,7 +690,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) } Finally, if the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the - deallocator should decrement the reference count for its type object after + deallocator should release the owned reference to its type object + (via :c:func:`Py_DECREF`) after calling the type deallocator. In order to avoid dangling pointers, the recommended way to achieve this is: @@ -1461,9 +1462,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) } The :c:func:`Py_CLEAR` macro should be used, because clearing references is - delicate: the reference to the contained object must not be decremented until + delicate: the reference to the contained object must not be released + (via :c:func:`Py_DECREF`) until after the pointer to the contained object is set to ``NULL``. This is because - decrementing the reference count may cause the contained object to become trash, + releasing the reference may cause the contained object to become trash, triggering a chain of reclamation activity that may include invoking arbitrary Python code (due to finalizers, or weakref callbacks, associated with the contained object). If it's possible for such code to reference *self* again, @@ -1541,7 +1543,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) they may be C ints or floats). The third argument specifies the requested operation, as for :c:func:`PyObject_RichCompare`. - The return value's reference count is properly incremented. + The returned value is a new :term:`strong reference`. On error, sets an exception and returns ``NULL`` from the function. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index e875cf6ded655..cf965fa167652 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -564,7 +564,7 @@ APIs: Copy an instance of a Unicode subtype to a new true Unicode object if necessary. If *obj* is already a true Unicode object (not a subtype), - return the reference with incremented refcount. + return a new :term:`strong reference` to the object. Objects other than Unicode or its subtypes will cause a :exc:`TypeError`. @@ -1438,11 +1438,11 @@ They all return ``NULL`` or ``-1`` if an exception occurs. Intern the argument *\*string* in place. The argument must be the address of a pointer variable pointing to a Python Unicode string object. If there is an existing interned string that is the same as *\*string*, it sets *\*string* to - it (decrementing the reference count of the old string object and incrementing - the reference count of the interned string object), otherwise it leaves - *\*string* alone and interns it (incrementing its reference count). - (Clarification: even though there is a lot of talk about reference counts, think - of this function as reference-count-neutral; you own the object after the call + it (releasing the reference to the old string object and creating a new + :term:`strong reference` to the interned string object), otherwise it leaves + *\*string* alone and interns it (creating a new :term:`strong reference`). + (Clarification: even though there is a lot of talk about references, think + of this function as reference-neutral; you own the object after the call if and only if you owned it before the call.) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index a4650a6c3efa2..a4cd05b0cf019 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -159,8 +159,9 @@ Glossary :class:`str` objects. borrowed reference - In Python's C API, a borrowed reference is a reference to an object. - It does not modify the object reference count. It becomes a dangling + In Python's C API, a borrowed reference is a reference to an object, + where the code using the object does not own the reference. + It becomes a dangling pointer if the object is destroyed. For example, a garbage collection can remove the last :term:`strong reference` to the object and so destroy it. @@ -1054,7 +1055,9 @@ Glossary reference count The number of references to an object. When the reference count of an - object drops to zero, it is deallocated. Reference counting is + object drops to zero, it is deallocated. Some objects are + "immortal" and have reference counts that are never modified, and + therefore the objects are never deallocated. Reference counting is generally not visible to Python code, but it is a key element of the :term:`CPython` implementation. Programmers can call the :func:`sys.getrefcount` function to return the @@ -1137,8 +1140,10 @@ Glossary strong reference In Python's C API, a strong reference is a reference to an object - which increments the object's reference count when it is created and - decrements the object's reference count when it is deleted. + which is owned by the code holding the reference. The strong + reference is taken by calling :c:func:`Py_INCREF` when the + reference is created and released with :c:func:`Py_DECREF` + when the reference is deleted. The :c:func:`Py_NewRef` function can be used to create a strong reference to an object. Usually, the :c:func:`Py_DECREF` function must be called on diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index a61737383e393..33391d11ab392 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -770,6 +770,15 @@ always available. higher than you might expect, because it includes the (temporary) reference as an argument to :func:`getrefcount`. + Note that the returned value may not actually reflect how many + references to the object are actually held. For example, some + objects are "immortal" and have a very high refcount that does not + reflect the actual number of references. Consequently, do not rely + on the returned value to be accurate, other than a value of 0 or 1. + + .. versionchanged:: 3.12 + Immortal objects have very large refcounts that do not match + the actual number of references to the object. .. function:: getrecursionlimit() From webhook-mailer at python.org Mon Aug 7 18:17:16 2023 From: webhook-mailer at python.org (ericsnowcurrently) Date: Mon, 07 Aug 2023 22:17:16 -0000 Subject: [Python-checkins] [3.11] gh-98154: Clarify Usage of "Reference Count" In the Docs (gh-107753) Message-ID: https://github.com/python/cpython/commit/951320e4d0f498857d0f78b3dbd0ee353bc4b93c commit: 951320e4d0f498857d0f78b3dbd0ee353bc4b93c branch: 3.11 author: Eric Snow committer: ericsnowcurrently date: 2023-08-07T16:17:12-06:00 summary: [3.11] gh-98154: Clarify Usage of "Reference Count" In the Docs (gh-107753) PEP 683 (immortal objects) revealed some ways in which the Python documentation has been unnecessarily coupled to the implementation details of reference counts. In the end users should focus on reference ownership, including taking references and releasing them, rather than on how many reference counts an object has. This change updates the documentation to reflect that perspective. files: M Doc/c-api/allocation.rst M Doc/c-api/arg.rst M Doc/c-api/buffer.rst M Doc/c-api/bytes.rst M Doc/c-api/exceptions.rst M Doc/c-api/intro.rst M Doc/c-api/module.rst M Doc/c-api/object.rst M Doc/c-api/refcounting.rst M Doc/c-api/sys.rst M Doc/c-api/typeobj.rst M Doc/c-api/unicode.rst M Doc/glossary.rst M Doc/library/sys.rst diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 44747e2964366..b3609c233156b 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -29,12 +29,13 @@ Allocating Objects on the Heap .. c:macro:: PyObject_New(TYPE, typeobj) - Allocate a new Python object using the C structure type *TYPE* and the - Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header - are not initialized; the object's reference count will be one. The size of - the memory allocation is determined from the :c:member:`~PyTypeObject.tp_basicsize` field of - the type object. + Allocate a new Python object using the C structure type *TYPE* + and the Python type object *typeobj* (``PyTypeObject*``). + Fields not defined by the Python object header are not initialized. + The caller will own the only reference to the object + (i.e. its reference count will be one). + The size of the memory allocation is determined from the + :c:member:`~PyTypeObject.tp_basicsize` field of the type object. .. c:macro:: PyObject_NewVar(TYPE, typeobj, size) diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index dfbec82c457bc..08d6cf788e299 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -330,8 +330,10 @@ Other objects ``O`` (object) [PyObject \*] Store a Python object (without any conversion) in a C object pointer. The C - program thus receives the actual object that was passed. The object's reference - count is not increased. The pointer stored is not ``NULL``. + program thus receives the actual object that was passed. A new + :term:`strong reference` to the object is not created + (i.e. its reference count is not increased). + The pointer stored is not ``NULL``. ``O!`` (object) [*typeobject*, PyObject \*] Store a Python object in a C object pointer. This is similar to ``O``, but @@ -415,7 +417,8 @@ inside nested parentheses. They are: mutually exclude each other. Note that any Python object references which are provided to the caller are -*borrowed* references; do not decrement their reference count! +*borrowed* references; do not release them +(i.e. do not decrement their reference count)! Additional arguments passed to these functions must be addresses of variables whose type is determined by the format string; these are used to store values @@ -650,8 +653,10 @@ Building values Convert a C :c:type:`Py_complex` structure to a Python complex number. ``O`` (object) [PyObject \*] - Pass a Python object untouched (except for its reference count, which is - incremented by one). If the object passed in is a ``NULL`` pointer, it is assumed + Pass a Python object untouched but create a new + :term:`strong reference` to it + (i.e. its reference count is incremented by one). + If the object passed in is a ``NULL`` pointer, it is assumed that this was caused because the call producing the argument found an error and set an exception. Therefore, :c:func:`Py_BuildValue` will return ``NULL`` but won't raise an exception. If no exception has been raised yet, :exc:`SystemError` is @@ -661,7 +666,7 @@ Building values Same as ``O``. ``N`` (object) [PyObject \*] - Same as ``O``, except it doesn't increment the reference count on the object. + Same as ``O``, except it doesn't create a new :term:`strong reference`. Useful when the object is created by a call to an object constructor in the argument list. diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 02b53ec149c73..8ca1c190dab9a 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -102,7 +102,9 @@ a buffer, see :c:func:`PyObject_GetBuffer`. .. c:member:: PyObject *obj A new reference to the exporting object. The reference is owned by - the consumer and automatically decremented and set to ``NULL`` by + the consumer and automatically released + (i.e. reference count decremented) + and set to ``NULL`` by :c:func:`PyBuffer_Release`. The field is the equivalent of the return value of any standard C-API function. @@ -454,7 +456,8 @@ Buffer-related functions .. c:function:: void PyBuffer_Release(Py_buffer *view) - Release the buffer *view* and decrement the reference count for + Release the buffer *view* and release the :term:`strong reference` + (i.e. decrement the reference count) to the view's supporting object, ``view->obj``. This function MUST be called when the buffer is no longer being used, otherwise reference leaks may occur. diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 110a98e19d823..33b7d23ff85a3 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -187,8 +187,8 @@ called with a non-bytes parameter. .. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) Create a new bytes object in *\*bytes* containing the contents of *newpart* - appended to *bytes*. This version decrements the reference count of - *newpart*. + appended to *bytes*. This version releases the :term:`strong reference` + to *newpart* (i.e. decrements its reference count). .. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 8d8b0ac7dd9a9..ae45aaef238fe 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -99,7 +99,8 @@ For convenience, some of these functions will always return a This is the most common way to set the error indicator. The first argument specifies the exception type; it is normally one of the standard exceptions, - e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. + e.g. :c:data:`PyExc_RuntimeError`. You need not create a new + :term:`strong reference` to it (e.g. with :c:func:`Py_INCREF`). The second argument is an error message; it is decoded from ``'utf-8'``. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index a3ccfa08baf70..ec2d8bc0ef02e 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -287,52 +287,58 @@ true if (and only if) the object pointed to by *a* is a Python list. Reference Counts ---------------- -The reference count is important because today's computers have a finite (and -often severely limited) memory size; it counts how many different places there -are that have a reference to an object. Such a place could be another object, -or a global (or static) C variable, or a local variable in some C function. -When an object's reference count becomes zero, the object is deallocated. If -it contains references to other objects, their reference count is decremented. -Those other objects may be deallocated in turn, if this decrement makes their -reference count become zero, and so on. (There's an obvious problem with -objects that reference each other here; for now, the solution is "don't do -that.") +The reference count is important because today's computers have a finite +(and often severely limited) memory size; it counts how many different +places there are that have a :term:`strong reference` to an object. +Such a place could be another object, or a global (or static) C variable, +or a local variable in some C function. +When the last :term:`strong reference` to an object is released +(i.e. its reference count becomes zero), the object is deallocated. +If it contains references to other objects, those references are released. +Those other objects may be deallocated in turn, if there are no more +references to them, and so on. (There's an obvious problem with +objects that reference each other here; for now, the solution +is "don't do that.") .. index:: single: Py_INCREF() single: Py_DECREF() -Reference counts are always manipulated explicitly. The normal way is to use -the macro :c:func:`Py_INCREF` to increment an object's reference count by one, -and :c:func:`Py_DECREF` to decrement it by one. The :c:func:`Py_DECREF` macro +Reference counts are always manipulated explicitly. The normal way is +to use the macro :c:func:`Py_INCREF` to take a new reference to an +object (i.e. increment its reference count by one), +and :c:func:`Py_DECREF` to release that reference (i.e. decrement the +reference count by one). The :c:func:`Py_DECREF` macro is considerably more complex than the incref one, since it must check whether the reference count becomes zero and then cause the object's deallocator to be -called. The deallocator is a function pointer contained in the object's type -structure. The type-specific deallocator takes care of decrementing the -reference counts for other objects contained in the object if this is a compound +called. The deallocator is a function pointer contained in the object's type +structure. The type-specific deallocator takes care of releasing references +for other objects contained in the object if this is a compound object type, such as a list, as well as performing any additional finalization that's needed. There's no chance that the reference count can overflow; at least as many bits are used to hold the reference count as there are distinct memory locations in virtual memory (assuming ``sizeof(Py_ssize_t) >= sizeof(void*)``). Thus, the reference count increment is a simple operation. -It is not necessary to increment an object's reference count for every local -variable that contains a pointer to an object. In theory, the object's +It is not necessary to hold a :term:`strong reference` (i.e. increment +the reference count) for every local variable that contains a pointer +to an object. In theory, the object's reference count goes up by one when the variable is made to point to it and it goes down by one when the variable goes out of scope. However, these two cancel each other out, so at the end the reference count hasn't changed. The only real reason to use the reference count is to prevent the object from being deallocated as long as our variable is pointing to it. If we know that there is at least one other reference to the object that lives at least as long as -our variable, there is no need to increment the reference count temporarily. +our variable, there is no need to take a new :term:`strong reference` +(i.e. increment the reference count) temporarily. An important situation where this arises is in objects that are passed as arguments to C functions in an extension module that are called from Python; the call mechanism guarantees to hold a reference to every argument for the duration of the call. However, a common pitfall is to extract an object from a list and hold on to it -for a while without incrementing its reference count. Some other operation might -conceivably remove the object from the list, decrementing its reference count +for a while without taking a new reference. Some other operation might +conceivably remove the object from the list, releasing that reference, and possibly deallocating it. The real danger is that innocent-looking operations may invoke arbitrary Python code which could do this; there is a code path which allows control to flow back to the user from a :c:func:`Py_DECREF`, so @@ -340,7 +346,8 @@ almost any operation is potentially dangerous. A safe approach is to always use the generic operations (functions whose name begins with ``PyObject_``, ``PyNumber_``, ``PySequence_`` or ``PyMapping_``). -These operations always increment the reference count of the object they return. +These operations always create a new :term:`strong reference` +(i.e. increment the reference count) of the object they return. This leaves the caller with the responsibility to call :c:func:`Py_DECREF` when they are done with the result; this soon becomes second nature. @@ -356,7 +363,7 @@ to objects (objects are not owned: they are always shared). "Owning a reference" means being responsible for calling Py_DECREF on it when the reference is no longer needed. Ownership can also be transferred, meaning that the code that receives ownership of the reference then becomes responsible for -eventually decref'ing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF` +eventually releasing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF` when it's no longer needed---or passing on this responsibility (usually to its caller). When a function passes ownership of a reference on to its caller, the caller is said to receive a *new* reference. When no ownership is transferred, @@ -414,9 +421,9 @@ For example, the above two blocks of code could be replaced by the following It is much more common to use :c:func:`PyObject_SetItem` and friends with items whose references you are only borrowing, like arguments that were passed in to -the function you are writing. In that case, their behaviour regarding reference -counts is much saner, since you don't have to increment a reference count so you -can give a reference away ("have it be stolen"). For example, this function +the function you are writing. In that case, their behaviour regarding references +is much saner, since you don't have to take a new reference just so you +can give that reference away ("have it be stolen"). For example, this function sets all items of a list (actually, any mutable sequence) to a given item:: int diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 8ca48c852d4e6..6ef4eea6a07f7 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -498,7 +498,7 @@ state: .. note:: Unlike other functions that steal references, ``PyModule_AddObject()`` - only decrements the reference count of *value* **on success**. + only releases the reference to *value* **on success**. This means that its return value must be checked, and calling code must :c:func:`Py_DECREF` *value* manually on error. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index e7d80dbf8c5c1..bc67fb60dc8be 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -15,8 +15,8 @@ Object Protocol .. c:macro:: Py_RETURN_NOTIMPLEMENTED Properly handle returning :c:data:`Py_NotImplemented` from within a C - function (that is, increment the reference count of NotImplemented and - return it). + function (that is, create a new :term:`strong reference` + to NotImplemented and return it). .. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags) @@ -320,11 +320,12 @@ Object Protocol When *o* is non-``NULL``, returns a type object corresponding to the object type of object *o*. On failure, raises :exc:`SystemError` and returns ``NULL``. This - is equivalent to the Python expression ``type(o)``. This function increments the - reference count of the return value. There's really no reason to use this + is equivalent to the Python expression ``type(o)``. + This function creates a new :term:`strong reference` to the return value. + There's really no reason to use this function instead of the :c:func:`Py_TYPE()` function, which returns a - pointer of type :c:expr:`PyTypeObject*`, except when the incremented reference - count is needed. + pointer of type :c:expr:`PyTypeObject*`, except when a new + :term:`strong reference` is needed. .. c:function:: int PyObject_TypeCheck(PyObject *o, PyTypeObject *type) diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index af25e8bb91368..e1782ab27f4f6 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -13,31 +13,36 @@ objects. .. c:function:: void Py_INCREF(PyObject *o) - Increment the reference count for object *o*. + Indicate taking a new :term:`strong reference` to object *o*, + indicating it is in use and should not be destroyed. This function is usually used to convert a :term:`borrowed reference` to a :term:`strong reference` in-place. The :c:func:`Py_NewRef` function can be used to create a new :term:`strong reference`. + When done using the object, release it by calling :c:func:`Py_DECREF`. + The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, use :c:func:`Py_XINCREF`. + Do not expect this function to actually modify *o* in any way. + .. c:function:: void Py_XINCREF(PyObject *o) - Increment the reference count for object *o*. The object may be ``NULL``, in - which case the macro has no effect. + Similar to :c:func:`Py_INCREF`, but the object *o* can be ``NULL``, + in which case this has no effect. See also :c:func:`Py_XNewRef`. .. c:function:: PyObject* Py_NewRef(PyObject *o) - Create a new :term:`strong reference` to an object: increment the reference - count of the object *o* and return the object *o*. + Create a new :term:`strong reference` to an object: + call :c:func:`Py_INCREF` on *o* and return the object *o*. When the :term:`strong reference` is no longer needed, :c:func:`Py_DECREF` - should be called on it to decrement the object reference count. + should be called on it to release the reference. The object *o* must not be ``NULL``; use :c:func:`Py_XNewRef` if *o* can be ``NULL``. @@ -67,9 +72,12 @@ objects. .. c:function:: void Py_DECREF(PyObject *o) - Decrement the reference count for object *o*. + Release a :term:`strong reference` to object *o*, indicating the + reference is no longer used. - If the reference count reaches zero, the object's type's deallocation + Once the last :term:`strong reference` is released + (i.e. the object's reference count reaches 0), + the object's type's deallocation function (which must not be ``NULL``) is invoked. This function is usually used to delete a :term:`strong reference` before @@ -78,6 +86,8 @@ objects. The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, use :c:func:`Py_XDECREF`. + Do not expect this function to actually modify *o* in any way. + .. warning:: The deallocation function can cause arbitrary Python code to be invoked (e.g. @@ -92,32 +102,35 @@ objects. .. c:function:: void Py_XDECREF(PyObject *o) - Decrement the reference count for object *o*. The object may be ``NULL``, in - which case the macro has no effect; otherwise the effect is the same as for - :c:func:`Py_DECREF`, and the same warning applies. + Similar to :c:func:`Py_DECREF`, but the object *o* can be ``NULL``, + in which case this has no effect. + The same warning from :c:func:`Py_DECREF` applies here as well. .. c:function:: void Py_CLEAR(PyObject *o) - Decrement the reference count for object *o*. The object may be ``NULL``, in + Release a :term:`strong reference` for object *o*. + The object may be ``NULL``, in which case the macro has no effect; otherwise the effect is the same as for :c:func:`Py_DECREF`, except that the argument is also set to ``NULL``. The warning for :c:func:`Py_DECREF` does not apply with respect to the object passed because the macro carefully uses a temporary variable and sets the argument to ``NULL`` - before decrementing its reference count. + before releasing the reference. - It is a good idea to use this macro whenever decrementing the reference - count of an object that might be traversed during garbage collection. + It is a good idea to use this macro whenever releasing a reference + to an object that might be traversed during garbage collection. .. c:function:: void Py_IncRef(PyObject *o) - Increment the reference count for object *o*. A function version of :c:func:`Py_XINCREF`. + Indicate taking a new :term:`strong reference` to object *o*. + A function version of :c:func:`Py_XINCREF`. It can be used for runtime dynamic embedding of Python. .. c:function:: void Py_DecRef(PyObject *o) - Decrement the reference count for object *o*. A function version of :c:func:`Py_XDECREF`. + Release a :term:`strong reference` to object *o*. + A function version of :c:func:`Py_XDECREF`. It can be used for runtime dynamic embedding of Python. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 34028d6c87e97..a811eb17b328e 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -8,8 +8,9 @@ Operating System Utilities .. c:function:: PyObject* PyOS_FSPath(PyObject *path) Return the file system representation for *path*. If the object is a - :class:`str` or :class:`bytes` object, then its reference count is - incremented. If the object implements the :class:`os.PathLike` interface, + :class:`str` or :class:`bytes` object, then a new + :term:`strong reference` is returned. + If the object implements the :class:`os.PathLike` interface, then :meth:`~os.PathLike.__fspath__` is returned as long as it is a :class:`str` or :class:`bytes` object. Otherwise :exc:`TypeError` is raised and ``NULL`` is returned. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 11f6034b9e330..40cbcba63d69a 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -688,7 +688,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) } Finally, if the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the - deallocator should decrement the reference count for its type object after + deallocator should release the owned reference to its type object + (via :c:func:`Py_DECREF`) after calling the type deallocator. In order to avoid dangling pointers, the recommended way to achieve this is: @@ -1397,9 +1398,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) } The :c:func:`Py_CLEAR` macro should be used, because clearing references is - delicate: the reference to the contained object must not be decremented until + delicate: the reference to the contained object must not be released + (via :c:func:`Py_DECREF`) until after the pointer to the contained object is set to ``NULL``. This is because - decrementing the reference count may cause the contained object to become trash, + releasing the reference may cause the contained object to become trash, triggering a chain of reclamation activity that may include invoking arbitrary Python code (due to finalizers, or weakref callbacks, associated with the contained object). If it's possible for such code to reference *self* again, @@ -1477,7 +1479,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) they may be C ints or floats). The third argument specifies the requested operation, as for :c:func:`PyObject_RichCompare`. - The return value's reference count is properly incremented. + The returned value is a new :term:`strong reference`. On error, sets an exception and returns ``NULL`` from the function. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 8b4252f08dd4c..3d6b4c49fa9c8 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -576,7 +576,7 @@ APIs: Copy an instance of a Unicode subtype to a new true Unicode object if necessary. If *obj* is already a true Unicode object (not a subtype), - return the reference with incremented refcount. + return a new :term:`strong reference` to the object. Objects other than Unicode or its subtypes will cause a :exc:`TypeError`. @@ -1537,11 +1537,11 @@ They all return ``NULL`` or ``-1`` if an exception occurs. Intern the argument *\*string* in place. The argument must be the address of a pointer variable pointing to a Python Unicode string object. If there is an existing interned string that is the same as *\*string*, it sets *\*string* to - it (decrementing the reference count of the old string object and incrementing - the reference count of the interned string object), otherwise it leaves - *\*string* alone and interns it (incrementing its reference count). - (Clarification: even though there is a lot of talk about reference counts, think - of this function as reference-count-neutral; you own the object after the call + it (releasing the reference to the old string object and creating a new + :term:`strong reference` to the interned string object), otherwise it leaves + *\*string* alone and interns it (creating a new :term:`strong reference`). + (Clarification: even though there is a lot of talk about references, think + of this function as reference-neutral; you own the object after the call if and only if you owned it before the call.) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 5c0f0f15217a0..81599477fc953 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -168,8 +168,9 @@ Glossary :class:`str` objects. borrowed reference - In Python's C API, a borrowed reference is a reference to an object. - It does not modify the object reference count. It becomes a dangling + In Python's C API, a borrowed reference is a reference to an object, + where the code using the object does not own the reference. + It becomes a dangling pointer if the object is destroyed. For example, a garbage collection can remove the last :term:`strong reference` to the object and so destroy it. @@ -1131,8 +1132,10 @@ Glossary strong reference In Python's C API, a strong reference is a reference to an object - which increments the object's reference count when it is created and - decrements the object's reference count when it is deleted. + which is owned by the code holding the reference. The strong + reference is taken by calling :c:func:`Py_INCREF` when the + reference is created and released with :c:func:`Py_DECREF` + when the reference is deleted. The :c:func:`Py_NewRef` function can be used to create a strong reference to an object. Usually, the :c:func:`Py_DECREF` function must be called on diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index b03860603c286..641be05a16fb3 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -759,6 +759,9 @@ always available. higher than you might expect, because it includes the (temporary) reference as an argument to :func:`getrefcount`. + Note that the returned value may not actually reflect how many + references to the object are actually held. Consequently, do not rely + on the returned value to be accurate, other than a value of 0 or 1. .. function:: getrecursionlimit() From webhook-mailer at python.org Mon Aug 7 19:11:00 2023 From: webhook-mailer at python.org (ericsnowcurrently) Date: Mon, 07 Aug 2023 23:11:00 -0000 Subject: [Python-checkins] gh-107630: Fix Remaining Subinterpreters Crashes on Py_TRACE_REFS Builds (gh-107750) Message-ID: https://github.com/python/cpython/commit/707018cc75558d6695e9e199a3ed0c8a4ff7cbcc commit: 707018cc75558d6695e9e199a3ed0c8a4ff7cbcc branch: main author: Eric Snow committer: ericsnowcurrently date: 2023-08-07T17:10:57-06:00 summary: gh-107630: Fix Remaining Subinterpreters Crashes on Py_TRACE_REFS Builds (gh-107750) This is a follow-up to gh-107567 and gh-107733. We skip test_basic_multiple_interpreters_deleted_no_reset on tracerefs builds. The test breaks interpreter isolation a little, which doesn't work well with Py_TRACE_REFS builds, so I feel fine about skipping the test. files: M Lib/test/test_import/__init__.py diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 163ed824ff1fb..051711bfd1fe2 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2555,6 +2555,12 @@ def test_basic_multiple_interpreters_main_no_reset(self): def test_basic_multiple_interpreters_deleted_no_reset(self): # without resetting; already loaded in a deleted interpreter + if hasattr(sys, 'getobjects'): + # It's a Py_TRACE_REFS build. + # This test breaks interpreter isolation a little, + # which causes problems on Py_TRACE_REF builds. + raise unittest.SkipTest('crashes on Py_TRACE_REFS builds') + # At this point: # * alive in 0 interpreters # * module def may or may not be loaded already From webhook-mailer at python.org Tue Aug 8 00:32:46 2023 From: webhook-mailer at python.org (gvanrossum) Date: Tue, 08 Aug 2023 04:32:46 -0000 Subject: [Python-checkins] gh-106812: Small stack effect fixes (#107759) Message-ID: https://github.com/python/cpython/commit/2df58dcd500dbedc61d0630374f9e94c522fe523 commit: 2df58dcd500dbedc61d0630374f9e94c522fe523 branch: main author: Guido van Rossum committer: gvanrossum date: 2023-08-07T21:32:42-07:00 summary: gh-106812: Small stack effect fixes (#107759) - Generalize the syntax for the type of a stack effect to allow a trailing `*`, so we can declare something as e.g. `PyCodeObject *`. - When generating assignments for stack effects, the type of the value on the stack should be the default (i.e., `PyObject *`) even when the variable copied to/from it has a different type, so that an appropriate cast is generated However, not when the variable is an array -- then the type is taken from the variable (as it is always `PyObject **`). files: M Tools/cases_generator/interpreter_definition.md M Tools/cases_generator/parsing.py M Tools/cases_generator/stacking.py diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index f141848631d04..5c4238756748a 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -108,7 +108,7 @@ and a piece of C code describing its semantics:: NAME [":" type] [ "if" "(" C-expression ")" ] type: - NAME + NAME ["*"] stream: NAME "/" size diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index 5610ac661a894..cdd20d7a0b3f5 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -252,12 +252,14 @@ def cache_effect(self) -> CacheEffect | None: @contextual def stack_effect(self) -> StackEffect | None: - # IDENTIFIER [':' IDENTIFIER] ['if' '(' expression ')'] + # IDENTIFIER [':' IDENTIFIER [TIMES]] ['if' '(' expression ')'] # | IDENTIFIER '[' expression ']' if tkn := self.expect(lx.IDENTIFIER): type_text = "" if self.expect(lx.COLON): type_text = self.require(lx.IDENTIFIER).text.strip() + if self.expect(lx.TIMES): + type_text += " *" cond_text = "" if self.expect(lx.IF): self.require(lx.LPAREN) diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 23eca3037f896..d457ce01a8f43 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -120,6 +120,14 @@ def as_variable(self, lax: bool = False) -> str: ), f"Push or pop above current stack level: {res}" return res + def as_stack_effect(self, lax: bool = False) -> StackEffect: + return StackEffect( + self.as_variable(lax=lax), + self.effect.type if self.effect.size else "", + self.effect.cond, + self.effect.size, + ) + @dataclasses.dataclass class CopyEffect: @@ -356,24 +364,14 @@ def write_components( for peek in mgr.peeks: out.assign( peek.effect, - StackEffect( - peek.as_variable(), - peek.effect.type, - peek.effect.cond, - peek.effect.size, - ), + peek.as_stack_effect(), ) # Initialize array outputs for poke in mgr.pokes: if poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: out.assign( poke.effect, - StackEffect( - poke.as_variable(lax=True), - poke.effect.type, - poke.effect.cond, - poke.effect.size, - ), + poke.as_stack_effect(lax=True), ) if len(parts) == 1: @@ -390,11 +388,6 @@ def write_components( for poke in mgr.pokes: if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: out.assign( - StackEffect( - poke.as_variable(), - poke.effect.type, - poke.effect.cond, - poke.effect.size, - ), + poke.as_stack_effect(), poke.effect, ) From webhook-mailer at python.org Tue Aug 8 00:36:28 2023 From: webhook-mailer at python.org (gvanrossum) Date: Tue, 08 Aug 2023 04:36:28 -0000 Subject: [Python-checkins] gh-107758: Improvements to lltrace feature (#107757) Message-ID: https://github.com/python/cpython/commit/328d925244511b2134d5ac926e307e4486ff4500 commit: 328d925244511b2134d5ac926e307e4486ff4500 branch: main author: Guido van Rossum committer: gvanrossum date: 2023-08-07T21:36:25-07:00 summary: gh-107758: Improvements to lltrace feature (#107757) - The `dump_stack()` method could call a `__repr__` method implemented in Python, causing (infinite) recursion. I rewrote it to only print out the values for some fundamental types (`int`, `str`, etc.); for everything else it just prints ``. - The lltrace-like feature for uops wrote to `stderr`, while the one in `ceval.c` writes to `stdout`; I changed the uops to write to stdout as well. files: A Misc/NEWS.d/next/Core and Builtins/2023-08-08-02-46-46.gh-issue-107758.R5kyBI.rst M Python/ceval.c M Python/executor.c M Python/optimizer.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-08-02-46-46.gh-issue-107758.R5kyBI.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-08-02-46-46.gh-issue-107758.R5kyBI.rst new file mode 100644 index 0000000000000..192f1df26e613 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-08-02-46-46.gh-issue-107758.R5kyBI.rst @@ -0,0 +1 @@ +Make the ``dump_stack()`` routine used by the ``lltrace`` feature (low-level interpreter debugging) robust against recursion by ensuring that it never calls a ``__repr__`` method implemented in Python. Also make the similar output for Tier-2 uops appear on ``stdout`` (instead of ``stderr``), to match the ``lltrace`` code in ceval.c. diff --git a/Python/ceval.c b/Python/ceval.c index 30f722e45e476..1dc4fd80224bb 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -109,11 +109,24 @@ dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer) if (ptr != stack_base) { printf(", "); } - if (PyObject_Print(*ptr, stdout, 0) != 0) { + if (*ptr == NULL) { + printf(""); + continue; + } + if ( + *ptr == Py_None + || PyBool_Check(*ptr) + || PyLong_CheckExact(*ptr) + || PyFloat_CheckExact(*ptr) + || PyUnicode_CheckExact(*ptr) + ) { + if (PyObject_Print(*ptr, stdout, 0) == 0) { + continue; + } PyErr_Clear(); - printf("<%s object at %p>", - Py_TYPE(*ptr)->tp_name, (void *)(*ptr)); } + // Don't call __repr__(), it might recurse into the interpreter. + printf("<%s at %p>", Py_TYPE(*ptr)->tp_name, (void *)(*ptr)); } printf("]\n"); fflush(stdout); @@ -128,9 +141,6 @@ lltrace_instruction(_PyInterpreterFrame *frame, if (frame->owner == FRAME_OWNED_BY_CSTACK) { return; } - /* This dump_stack() operation is risky, since the repr() of some - objects enters the interpreter recursively. It is also slow. - So you might want to comment it out. */ dump_stack(frame, stack_pointer); int oparg = next_instr->op.arg; int opcode = next_instr->op.code; @@ -729,6 +739,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto exit_unwind; } lltrace = r; + if (!lltrace) { + // When tracing executed uops, also trace bytecode + char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); + if (uop_debug != NULL && *uop_debug >= '0') { + lltrace = (*uop_debug - '0') >= 4; // TODO: Parse an int and all that + } + } } if (lltrace) { lltrace_resume_frame(frame); @@ -896,6 +913,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto exception_unwind; } /* Resume normal execution */ +#ifdef LLTRACE + if (lltrace) { + lltrace_resume_frame(frame); + } +#endif DISPATCH(); } } diff --git a/Python/executor.c b/Python/executor.c index 57525df202d86..4a18618c0c6c0 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -41,7 +41,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject lltrace = *uop_debug - '0'; // TODO: Parse an int and all that } #define DPRINTF(level, ...) \ - if (lltrace >= (level)) { fprintf(stderr, __VA_ARGS__); } + if (lltrace >= (level)) { printf(__VA_ARGS__); } #else #define DPRINTF(level, ...) #endif diff --git a/Python/optimizer.c b/Python/optimizer.c index d2ed8dfcd2cff..6c730aa14b9a4 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -391,7 +391,7 @@ translate_bytecode_to_trace( #ifdef Py_DEBUG #define DPRINTF(level, ...) \ - if (lltrace >= (level)) { fprintf(stderr, __VA_ARGS__); } + if (lltrace >= (level)) { printf(__VA_ARGS__); } #else #define DPRINTF(level, ...) #endif From webhook-mailer at python.org Tue Aug 8 02:41:13 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 08 Aug 2023 06:41:13 -0000 Subject: [Python-checkins] gh-95065: Make Argument Clinic append deprecation warnings to docstrings (#107745) Message-ID: https://github.com/python/cpython/commit/0db043dd5a34f7b5968476011e36396737d09030 commit: 0db043dd5a34f7b5968476011e36396737d09030 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-08T08:41:09+02:00 summary: gh-95065: Make Argument Clinic append deprecation warnings to docstrings (#107745) files: M Lib/test/clinic.test.c M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index c5c37b7e9e216..386d1dac3f115 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5477,7 +5477,11 @@ test_deprecate_positional_pos1_len1_optional PyDoc_STRVAR(test_deprecate_positional_pos1_len1_optional__doc__, "test_deprecate_positional_pos1_len1_optional($module, /, a, b=None)\n" "--\n" -"\n"); +"\n" +"Note: Passing 2 positional arguments to\n" +"test_deprecate_positional_pos1_len1_optional() is deprecated.\n" +"Parameter \'b\' will become a keyword-only parameter in Python 3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS1_LEN1_OPTIONAL_METHODDEF \ {"test_deprecate_positional_pos1_len1_optional", _PyCFunction_CAST(test_deprecate_positional_pos1_len1_optional), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len1_optional__doc__}, @@ -5567,7 +5571,7 @@ test_deprecate_positional_pos1_len1_optional(PyObject *module, PyObject *const * static PyObject * test_deprecate_positional_pos1_len1_optional_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=09a6edec1ddcd469 input=89099f3dacd757da]*/ +/*[clinic end generated code: output=6c0fd9d94fa1e765 input=89099f3dacd757da]*/ /*[clinic input] @@ -5580,7 +5584,11 @@ test_deprecate_positional_pos1_len1 PyDoc_STRVAR(test_deprecate_positional_pos1_len1__doc__, "test_deprecate_positional_pos1_len1($module, /, a, b)\n" "--\n" -"\n"); +"\n" +"Note: Passing 2 positional arguments to\n" +"test_deprecate_positional_pos1_len1() is deprecated. Parameter \'b\'\n" +"will become a keyword-only parameter in Python 3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS1_LEN1_METHODDEF \ {"test_deprecate_positional_pos1_len1", _PyCFunction_CAST(test_deprecate_positional_pos1_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len1__doc__}, @@ -5661,7 +5669,7 @@ test_deprecate_positional_pos1_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=52a2618293df747d input=1702bbab1e9b3b99]*/ +/*[clinic end generated code: output=22821a0fa9945d0c input=1702bbab1e9b3b99]*/ /*[clinic input] @@ -5677,7 +5685,12 @@ test_deprecate_positional_pos1_len2_with_kwd PyDoc_STRVAR(test_deprecate_positional_pos1_len2_with_kwd__doc__, "test_deprecate_positional_pos1_len2_with_kwd($module, /, a, b, c, *, d)\n" "--\n" -"\n"); +"\n" +"Note: Passing more than 1 positional argument to\n" +"test_deprecate_positional_pos1_len2_with_kwd() is deprecated.\n" +"Parameters \'b\' and \'c\' will become keyword-only parameters in Python\n" +"3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS1_LEN2_WITH_KWD_METHODDEF \ {"test_deprecate_positional_pos1_len2_with_kwd", _PyCFunction_CAST(test_deprecate_positional_pos1_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len2_with_kwd__doc__}, @@ -5768,7 +5781,7 @@ static PyObject * test_deprecate_positional_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=550aabea548589b4 input=28cdb885f6c34eab]*/ +/*[clinic end generated code: output=061d554ccc6b8f51 input=28cdb885f6c34eab]*/ /*[clinic input] @@ -5780,7 +5793,11 @@ test_deprecate_positional_pos0_len1 PyDoc_STRVAR(test_deprecate_positional_pos0_len1__doc__, "test_deprecate_positional_pos0_len1($module, /, a)\n" "--\n" -"\n"); +"\n" +"Note: Passing positional arguments to\n" +"test_deprecate_positional_pos0_len1() is deprecated. Parameter \'a\'\n" +"will become a keyword-only parameter in Python 3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS0_LEN1_METHODDEF \ {"test_deprecate_positional_pos0_len1", _PyCFunction_CAST(test_deprecate_positional_pos0_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len1__doc__}, @@ -5857,7 +5874,7 @@ test_deprecate_positional_pos0_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos0_len1_impl(PyObject *module, PyObject *a) -/*[clinic end generated code: output=66c63ec8d6903bde input=678206db25c0652c]*/ +/*[clinic end generated code: output=3e512117a5eda970 input=678206db25c0652c]*/ /*[clinic input] @@ -5870,7 +5887,11 @@ test_deprecate_positional_pos0_len2 PyDoc_STRVAR(test_deprecate_positional_pos0_len2__doc__, "test_deprecate_positional_pos0_len2($module, /, a, b)\n" "--\n" -"\n"); +"\n" +"Note: Passing positional arguments to\n" +"test_deprecate_positional_pos0_len2() is deprecated. Parameters \'a\'\n" +"and \'b\' will become keyword-only parameters in Python 3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS0_LEN2_METHODDEF \ {"test_deprecate_positional_pos0_len2", _PyCFunction_CAST(test_deprecate_positional_pos0_len2), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len2__doc__}, @@ -5954,7 +5975,7 @@ test_deprecate_positional_pos0_len2(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=6b6df40aaf751b2e input=fae0d0b1d480c939]*/ +/*[clinic end generated code: output=d41da050a5b82dd0 input=fae0d0b1d480c939]*/ /*[clinic input] @@ -5971,7 +5992,12 @@ PyDoc_STRVAR(test_deprecate_positional_pos0_len3_with_kwdonly__doc__, "test_deprecate_positional_pos0_len3_with_kwdonly($module, /, a, b, c,\n" " *, e)\n" "--\n" -"\n"); +"\n" +"Note: Passing positional arguments to\n" +"test_deprecate_positional_pos0_len3_with_kwdonly() is deprecated.\n" +"Parameters \'a\', \'b\' and \'c\' will become keyword-only parameters in\n" +"Python 3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS0_LEN3_WITH_KWDONLY_METHODDEF \ {"test_deprecate_positional_pos0_len3_with_kwdonly", _PyCFunction_CAST(test_deprecate_positional_pos0_len3_with_kwdonly), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len3_with_kwdonly__doc__}, @@ -6069,7 +6095,7 @@ test_deprecate_positional_pos0_len3_with_kwdonly_impl(PyObject *module, PyObject *b, PyObject *c, PyObject *e) -/*[clinic end generated code: output=5c936993846d01a3 input=1b0121770c0c52e0]*/ +/*[clinic end generated code: output=c5d7ddfc139ddf31 input=1b0121770c0c52e0]*/ /*[clinic input] @@ -6083,7 +6109,11 @@ test_deprecate_positional_pos2_len1 PyDoc_STRVAR(test_deprecate_positional_pos2_len1__doc__, "test_deprecate_positional_pos2_len1($module, /, a, b, c)\n" "--\n" -"\n"); +"\n" +"Note: Passing 3 positional arguments to\n" +"test_deprecate_positional_pos2_len1() is deprecated. Parameter \'c\'\n" +"will become a keyword-only parameter in Python 3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS2_LEN1_METHODDEF \ {"test_deprecate_positional_pos2_len1", _PyCFunction_CAST(test_deprecate_positional_pos2_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len1__doc__}, @@ -6166,7 +6196,7 @@ test_deprecate_positional_pos2_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=2641e037296e3b61 input=e1d129689e69ec7c]*/ +/*[clinic end generated code: output=d80609e6b37ffb8a input=e1d129689e69ec7c]*/ /*[clinic input] @@ -6181,7 +6211,11 @@ test_deprecate_positional_pos2_len2 PyDoc_STRVAR(test_deprecate_positional_pos2_len2__doc__, "test_deprecate_positional_pos2_len2($module, /, a, b, c, d)\n" "--\n" -"\n"); +"\n" +"Note: Passing more than 2 positional arguments to\n" +"test_deprecate_positional_pos2_len2() is deprecated. Parameters \'c\'\n" +"and \'d\' will become keyword-only parameters in Python 3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS2_LEN2_METHODDEF \ {"test_deprecate_positional_pos2_len2", _PyCFunction_CAST(test_deprecate_positional_pos2_len2), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len2__doc__}, @@ -6271,7 +6305,7 @@ static PyObject * test_deprecate_positional_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=4a9068ef8fee61f6 input=0d53533463a12792]*/ +/*[clinic end generated code: output=1c10d6197562319f input=0d53533463a12792]*/ /*[clinic input] @@ -6289,7 +6323,12 @@ PyDoc_STRVAR(test_deprecate_positional_pos2_len3_with_kwdonly__doc__, "test_deprecate_positional_pos2_len3_with_kwdonly($module, /, a, b, c,\n" " d, *, e)\n" "--\n" -"\n"); +"\n" +"Note: Passing more than 2 positional arguments to\n" +"test_deprecate_positional_pos2_len3_with_kwdonly() is deprecated.\n" +"Parameters \'c\' and \'d\' will become keyword-only parameters in Python\n" +"3.14.\n" +""); #define TEST_DEPRECATE_POSITIONAL_POS2_LEN3_WITH_KWDONLY_METHODDEF \ {"test_deprecate_positional_pos2_len3_with_kwdonly", _PyCFunction_CAST(test_deprecate_positional_pos2_len3_with_kwdonly), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len3_with_kwdonly__doc__}, @@ -6388,4 +6427,4 @@ test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=1154c2e3e798948c input=154fd450448d8935]*/ +/*[clinic end generated code: output=d32375ffce63d3db input=154fd450448d8935]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 57ad4e41ba24a..82e5b804c2e3e 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -941,6 +941,12 @@ def deprecate_positional_use( f"{func.full_name}() is deprecated. Parameters {pstr} will " f"become keyword-only parameters in Python {major}.{minor}." ) + + # Append deprecation warning to docstring. + lines = textwrap.wrap(f"Note: {depr_message}") + docstring = "\n".join(lines) + func.docstring += f"\n\n{docstring}\n" + # Format and return the code block. code = self.DEPRECATED_POSITIONAL_PROTOTYPE.format( condition=condition, From webhook-mailer at python.org Tue Aug 8 02:42:12 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 08 Aug 2023 06:42:12 -0000 Subject: [Python-checkins] gh-86457: Add docs for Argument Clinic @text_signature directive (#107747) Message-ID: https://github.com/python/cpython/commit/a9aeb99579f24bbce1dd553d605a5a5e2f37a3a2 commit: a9aeb99579f24bbce1dd553d605a5a5e2f37a3a2 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-08T08:42:08+02:00 summary: gh-86457: Add docs for Argument Clinic @text_signature directive (#107747) Co-authored-by: Alex Waygood files: M Doc/howto/clinic.rst M Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 286623c241014..743c7c9cb3000 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1900,6 +1900,40 @@ blocks embedded in Python files look slightly different. They look like this: #/*[python checksum:...]*/ +.. _clinic-howto-override-signature: + +How to override the generated signature +--------------------------------------- + +You can use the ``@text_signature`` directive to override the default generated +signature in the docstring. +This can be useful for complex signatures that Argument Clinic cannot handle. +The ``@text_signature`` directive takes one argument: +the custom signature as a string. +The provided signature is copied verbatim to the generated docstring. + +Example from :source:`Objects/codeobject.c`:: + + /*[clinic input] + @text_signature "($self, /, **changes)" + code.replace + * + co_argcount: int(c_default="self->co_argcount") = unchanged + co_posonlyargcount: int(c_default="self->co_posonlyargcount") = unchanged + # etc ... + + Return a copy of the code object with new values for the specified fields. + [clinic start generated output]*/ + +The generated docstring ends up looking like this:: + + Doc_STRVAR(code_replace__doc__, + "replace($self, /, **changes)\n" + "--\n" + "\n" + "Return a copy of the code object with new values for the specified fields."); + + .. _clinic-howto-deprecate-positional: How to deprecate passing parameters positionally diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst index 41cfd72cb3676..df1893e9c808e 100644 --- a/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst @@ -1,2 +1,2 @@ Argument Clinic now supports overriding automatically generated signature by -using directive `@text_signature`. +using directive `@text_signature`. See :ref:`clinic-howto-override-signature`. From webhook-mailer at python.org Tue Aug 8 03:37:49 2023 From: webhook-mailer at python.org (methane) Date: Tue, 08 Aug 2023 07:37:49 -0000 Subject: [Python-checkins] README: remove unmaintained sections (#107703) Message-ID: https://github.com/python/cpython/commit/7a250fdc16bb6f1fe0a6b0df8bb502870405b5d6 commit: 7a250fdc16bb6f1fe0a6b0df8bb502870405b5d6 branch: main author: Inada Naoki committer: methane date: 2023-08-08T16:37:45+09:00 summary: README: remove unmaintained sections (#107703) files: M README.rst diff --git a/README.rst b/README.rst index 4ab26565a13e0..208bf8cec444a 100644 --- a/README.rst +++ b/README.rst @@ -211,30 +211,6 @@ primary version, you would execute ``make install`` in your 3.13 build directory and ``make altinstall`` in the others. -Issue Tracker and Mailing List ------------------------------- - -Bug reports are welcome! You can use Github to `report bugs -`_, and/or `submit pull requests -`_. - -You can also follow development discussion on the `python-dev mailing list -`_. - - -Proposals for enhancement -------------------------- - -If you have a proposal to change Python, you may want to send an email to the -`comp.lang.python`_ or `python-ideas`_ mailing lists for initial feedback. A -Python Enhancement Proposal (PEP) may be submitted if your idea gains ground. -All current PEPs, as well as guidelines for submitting a new PEP, are listed at -`peps.python.org `_. - -.. _python-ideas: https://mail.python.org/mailman/listinfo/python-ideas/ -.. _comp.lang.python: https://mail.python.org/mailman/listinfo/python-list - - Release Schedule ---------------- From webhook-mailer at python.org Tue Aug 8 03:51:47 2023 From: webhook-mailer at python.org (methane) Date: Tue, 08 Aug 2023 07:51:47 -0000 Subject: [Python-checkins] README: remove unmaintained sections (GH-107703) Message-ID: https://github.com/python/cpython/commit/648d42643eeca18206825f0f4c58c58f71e0ae1a commit: 648d42643eeca18206825f0f4c58c58f71e0ae1a branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: methane date: 2023-08-08T16:51:43+09:00 summary: README: remove unmaintained sections (GH-107703) (cherry picked from commit 7a250fdc16bb6f1fe0a6b0df8bb502870405b5d6) files: M README.rst diff --git a/README.rst b/README.rst index e70800554784b..5e492ff2d2f3e 100644 --- a/README.rst +++ b/README.rst @@ -211,30 +211,6 @@ primary version, you would execute ``make install`` in your 3.11 build directory and ``make altinstall`` in the others. -Issue Tracker and Mailing List ------------------------------- - -Bug reports are welcome! You can use Github to `report bugs -`_, and/or `submit pull requests -`_. - -You can also follow development discussion on the `python-dev mailing list -`_. - - -Proposals for enhancement -------------------------- - -If you have a proposal to change Python, you may want to send an email to the -`comp.lang.python`_ or `python-ideas`_ mailing lists for initial feedback. A -Python Enhancement Proposal (PEP) may be submitted if your idea gains ground. -All current PEPs, as well as guidelines for submitting a new PEP, are listed at -`peps.python.org `_. - -.. _python-ideas: https://mail.python.org/mailman/listinfo/python-ideas/ -.. _comp.lang.python: https://mail.python.org/mailman/listinfo/python-list - - Release Schedule ---------------- From webhook-mailer at python.org Tue Aug 8 04:20:14 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 08 Aug 2023 08:20:14 -0000 Subject: [Python-checkins] gh-107659: Add docstrings for ctypes.pointer and ctypes.POINTER (#107660) Message-ID: https://github.com/python/cpython/commit/de72677f8a27083b2072b83b3737f891b660bb5c commit: de72677f8a27083b2072b83b3737f891b660bb5c branch: main author: Tomas R committer: erlend-aasland date: 2023-08-08T08:20:10Z summary: gh-107659: Add docstrings for ctypes.pointer and ctypes.POINTER (#107660) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-05-15-45-07.gh-issue-107659.QgtQ5M.rst A Modules/_ctypes/clinic/callproc.c.h M Modules/_ctypes/callproc.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-05-15-45-07.gh-issue-107659.QgtQ5M.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-05-15-45-07.gh-issue-107659.QgtQ5M.rst new file mode 100644 index 0000000000000..31cc6982400d5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-05-15-45-07.gh-issue-107659.QgtQ5M.rst @@ -0,0 +1 @@ +Add docstrings for :func:`ctypes.pointer` and :func:`ctypes.POINTER`. diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 69cf8a98af663..f9535db4f57c0 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -54,6 +54,11 @@ */ +/*[clinic input] +module _ctypes +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=476a19c49b31a75c]*/ + #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif @@ -98,6 +103,7 @@ #include "pycore_runtime.h" // _PyRuntime #include "pycore_global_objects.h" // _Py_ID() +#include "clinic/callproc.c.h" #define CTYPES_CAPSULE_NAME_PYMEM "_ctypes pymem" @@ -1893,8 +1899,22 @@ unpickle(PyObject *self, PyObject *args) return NULL; } +/*[clinic input] +_ctypes.POINTER as create_pointer_type + + type as cls: object + A ctypes type. + / + +Create and return a new ctypes pointer type. + +Pointer types are cached and reused internally, +so calling this function repeatedly is cheap. +[clinic start generated code]*/ + static PyObject * -POINTER(PyObject *self, PyObject *cls) +create_pointer_type(PyObject *module, PyObject *cls) +/*[clinic end generated code: output=98c3547ab6f4f40b input=3b81cff5ff9b9d5b]*/ { PyObject *result; PyTypeObject *typ; @@ -1944,8 +1964,22 @@ POINTER(PyObject *self, PyObject *cls) return result; } +/*[clinic input] +_ctypes.pointer as create_pointer_inst + + obj as arg: object + / + +Create a new pointer instance, pointing to 'obj'. + +The returned object is of the type POINTER(type(obj)). Note that if you +just want to pass a pointer to an object to a foreign function call, you +should use byref(obj) which is much faster. +[clinic start generated code]*/ + static PyObject * -pointer(PyObject *self, PyObject *arg) +create_pointer_inst(PyObject *module, PyObject *arg) +/*[clinic end generated code: output=3b543bc9f0de2180 input=713685fdb4d9bc27]*/ { PyObject *result; PyObject *typ; @@ -1957,7 +1991,7 @@ pointer(PyObject *self, PyObject *arg) else if (PyErr_Occurred()) { return NULL; } - typ = POINTER(NULL, (PyObject *)Py_TYPE(arg)); + typ = create_pointer_type(NULL, (PyObject *)Py_TYPE(arg)); if (typ == NULL) return NULL; result = PyObject_CallOneArg(typ, arg); @@ -1997,8 +2031,8 @@ buffer_info(PyObject *self, PyObject *arg) PyMethodDef _ctypes_module_methods[] = { {"get_errno", get_errno, METH_NOARGS}, {"set_errno", set_errno, METH_VARARGS}, - {"POINTER", POINTER, METH_O }, - {"pointer", pointer, METH_O }, + CREATE_POINTER_TYPE_METHODDEF + CREATE_POINTER_INST_METHODDEF {"_unpickle", unpickle, METH_VARARGS }, {"buffer_info", buffer_info, METH_O, "Return buffer interface information"}, {"resize", resize, METH_VARARGS, "Resize the memory buffer of a ctypes instance"}, diff --git a/Modules/_ctypes/clinic/callproc.c.h b/Modules/_ctypes/clinic/callproc.c.h new file mode 100644 index 0000000000000..6f036bb66b25a --- /dev/null +++ b/Modules/_ctypes/clinic/callproc.c.h @@ -0,0 +1,38 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(create_pointer_type__doc__, +"POINTER($module, type, /)\n" +"--\n" +"\n" +"Create and return a new ctypes pointer type.\n" +"\n" +" type\n" +" A ctypes type.\n" +"\n" +"Pointer types are cached and reused internally,\n" +"so calling this function repeatedly is cheap."); + +#define CREATE_POINTER_TYPE_METHODDEF \ + {"POINTER", (PyCFunction)create_pointer_type, METH_O, create_pointer_type__doc__}, + +PyDoc_STRVAR(create_pointer_inst__doc__, +"pointer($module, obj, /)\n" +"--\n" +"\n" +"Create a new pointer instance, pointing to \'obj\'.\n" +"\n" +"The returned object is of the type POINTER(type(obj)). Note that if you\n" +"just want to pass a pointer to an object to a foreign function call, you\n" +"should use byref(obj) which is much faster."); + +#define CREATE_POINTER_INST_METHODDEF \ + {"pointer", (PyCFunction)create_pointer_inst, METH_O, create_pointer_inst__doc__}, +/*[clinic end generated code: output=ae26452a759ba56d input=a9049054013a1b77]*/ From webhook-mailer at python.org Tue Aug 8 04:43:46 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 08 Aug 2023 08:43:46 -0000 Subject: [Python-checkins] gh-95065: Argument Clinic: Add comment to preprocessor warning code (#107766) Message-ID: https://github.com/python/cpython/commit/5df8b0d5c71a168a94fb64ad9d8190377b6e73da commit: 5df8b0d5c71a168a94fb64ad9d8190377b6e73da branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-08T08:43:41Z summary: gh-95065: Argument Clinic: Add comment to preprocessor warning code (#107766) files: M Lib/test/clinic.test.c M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 386d1dac3f115..9fcee0dad7118 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5524,6 +5524,7 @@ test_deprecate_positional_pos1_len1_optional(PyObject *module, PyObject *const * PyObject *a; PyObject *b = Py_None; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ @@ -5571,7 +5572,7 @@ test_deprecate_positional_pos1_len1_optional(PyObject *module, PyObject *const * static PyObject * test_deprecate_positional_pos1_len1_optional_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=6c0fd9d94fa1e765 input=89099f3dacd757da]*/ +/*[clinic end generated code: output=144cbf1adc574dd9 input=89099f3dacd757da]*/ /*[clinic input] @@ -5630,6 +5631,7 @@ test_deprecate_positional_pos1_len1(PyObject *module, PyObject *const *args, Py_ PyObject *a; PyObject *b; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ @@ -5669,7 +5671,7 @@ test_deprecate_positional_pos1_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=22821a0fa9945d0c input=1702bbab1e9b3b99]*/ +/*[clinic end generated code: output=994bd57c1c634709 input=1702bbab1e9b3b99]*/ /*[clinic input] @@ -5735,6 +5737,7 @@ test_deprecate_positional_pos1_len2_with_kwd(PyObject *module, PyObject *const * PyObject *c; PyObject *d; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic " \ @@ -5781,7 +5784,7 @@ static PyObject * test_deprecate_positional_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=061d554ccc6b8f51 input=28cdb885f6c34eab]*/ +/*[clinic end generated code: output=146c60ecbcdbf4b8 input=28cdb885f6c34eab]*/ /*[clinic input] @@ -5837,6 +5840,7 @@ test_deprecate_positional_pos0_len1(PyObject *module, PyObject *const *args, Py_ PyObject *argsbuf[1]; PyObject *a; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ @@ -5874,7 +5878,7 @@ test_deprecate_positional_pos0_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos0_len1_impl(PyObject *module, PyObject *a) -/*[clinic end generated code: output=3e512117a5eda970 input=678206db25c0652c]*/ +/*[clinic end generated code: output=dce99971a2494f9f input=678206db25c0652c]*/ /*[clinic input] @@ -5933,6 +5937,7 @@ test_deprecate_positional_pos0_len2(PyObject *module, PyObject *const *args, Py_ PyObject *a; PyObject *b; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic " \ @@ -5975,7 +5980,7 @@ test_deprecate_positional_pos0_len2(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b) -/*[clinic end generated code: output=d41da050a5b82dd0 input=fae0d0b1d480c939]*/ +/*[clinic end generated code: output=06999692e0c8dac4 input=fae0d0b1d480c939]*/ /*[clinic input] @@ -6044,6 +6049,7 @@ test_deprecate_positional_pos0_len3_with_kwdonly(PyObject *module, PyObject *con PyObject *c; PyObject *e; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the " \ @@ -6095,7 +6101,7 @@ test_deprecate_positional_pos0_len3_with_kwdonly_impl(PyObject *module, PyObject *b, PyObject *c, PyObject *e) -/*[clinic end generated code: output=c5d7ddfc139ddf31 input=1b0121770c0c52e0]*/ +/*[clinic end generated code: output=a553e33101dc42b2 input=1b0121770c0c52e0]*/ /*[clinic input] @@ -6156,6 +6162,7 @@ test_deprecate_positional_pos2_len1(PyObject *module, PyObject *const *args, Py_ PyObject *b; PyObject *c; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'c' in the clinic input of" \ @@ -6196,7 +6203,7 @@ test_deprecate_positional_pos2_len1(PyObject *module, PyObject *const *args, Py_ static PyObject * test_deprecate_positional_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c) -/*[clinic end generated code: output=d80609e6b37ffb8a input=e1d129689e69ec7c]*/ +/*[clinic end generated code: output=f96454a4970b443c input=e1d129689e69ec7c]*/ /*[clinic input] @@ -6260,6 +6267,7 @@ test_deprecate_positional_pos2_len2(PyObject *module, PyObject *const *args, Py_ PyObject *c; PyObject *d; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ @@ -6305,7 +6313,7 @@ static PyObject * test_deprecate_positional_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, PyObject *d) -/*[clinic end generated code: output=1c10d6197562319f input=0d53533463a12792]*/ +/*[clinic end generated code: output=5e648e887da0a804 input=0d53533463a12792]*/ /*[clinic input] @@ -6377,6 +6385,7 @@ test_deprecate_positional_pos2_len3_with_kwdonly(PyObject *module, PyObject *con PyObject *d; PyObject *e; + // Emit compiler warnings when we get to Python 3.14. #if PY_VERSION_HEX >= 0x030e00C0 # error \ "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ @@ -6427,4 +6436,4 @@ test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, PyObject *c, PyObject *d, PyObject *e) -/*[clinic end generated code: output=d32375ffce63d3db input=154fd450448d8935]*/ +/*[clinic end generated code: output=383d56b03f7c2dcb input=154fd450448d8935]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 82e5b804c2e3e..c6cf43ab40fb1 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -850,6 +850,7 @@ class CLanguage(Language): #endif /* !defined({methoddef_name}) */ """) DEPRECATED_POSITIONAL_PROTOTYPE: Final[str] = r""" + // Emit compiler warnings when we get to Python {major}.{minor}. #if PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00C0 # error \ {cpp_message} From webhook-mailer at python.org Tue Aug 8 06:17:32 2023 From: webhook-mailer at python.org (hugovk) Date: Tue, 08 Aug 2023 10:17:32 -0000 Subject: [Python-checkins] [3.11] GH-84435: Make pyspecific directives translatable (GH-19470) (#107682) Message-ID: https://github.com/python/cpython/commit/0aa3b9d76c036fd306c0fd1c270d5d88d02962a1 commit: 0aa3b9d76c036fd306c0fd1c270d5d88d02962a1 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: hugovk date: 2023-08-08T12:17:28+02:00 summary: [3.11] GH-84435: Make pyspecific directives translatable (GH-19470) (#107682) GH-84435: Make pyspecific directives translatable (GH-19470) (cherry picked from commit ecb05e0b9842ba03b42b4dec8767b1c18a4e28b3) Co-authored-by: cocoatomo Co-authored-by: Jelle Zijlstra Co-authored-by: Adam Turner <9087854+aa-turner at users.noreply.github.com> files: M Doc/tools/extensions/pyspecific.py diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index f59331419e508..d13a9dfd4c8eb 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -103,14 +103,13 @@ class ImplementationDetail(Directive): final_argument_whitespace = True # This text is copied to templates/dummy.html - label_text = 'CPython implementation detail:' + label_text = sphinx_gettext('CPython implementation detail:') def run(self): self.assert_has_content() pnode = nodes.compound(classes=['impl-detail']) - label = sphinx_gettext(self.label_text) content = self.content - add_text = nodes.strong(label, label) + add_text = nodes.strong(self.label_text, self.label_text) self.state.nested_parse(content, self.content_offset, pnode) content = nodes.inline(pnode[0].rawsource, translatable=True) content.source = pnode[0].source @@ -239,9 +238,9 @@ class AuditEvent(Directive): final_argument_whitespace = True _label = [ - "Raises an :ref:`auditing event ` {name} with no arguments.", - "Raises an :ref:`auditing event ` {name} with argument {args}.", - "Raises an :ref:`auditing event ` {name} with arguments {args}.", + sphinx_gettext("Raises an :ref:`auditing event ` {name} with no arguments."), + sphinx_gettext("Raises an :ref:`auditing event ` {name} with argument {args}."), + sphinx_gettext("Raises an :ref:`auditing event ` {name} with arguments {args}."), ] @property @@ -257,7 +256,7 @@ def run(self): else: args = [] - label = sphinx_gettext(self._label[min(2, len(args))]) + label = self._label[min(2, len(args))] text = label.format(name="``{}``".format(name), args=", ".join("``{}``".format(a) for a in args if a)) @@ -419,8 +418,8 @@ class DeprecatedRemoved(Directive): final_argument_whitespace = True option_spec = {} - _deprecated_label = 'Deprecated since version {deprecated}, will be removed in version {removed}' - _removed_label = 'Deprecated since version {deprecated}, removed in version {removed}' + _deprecated_label = sphinx_gettext('Deprecated since version {deprecated}, will be removed in version {removed}') + _removed_label = sphinx_gettext('Deprecated since version {deprecated}, removed in version {removed}') def run(self): node = addnodes.versionmodified() @@ -436,7 +435,6 @@ def run(self): else: label = self._removed_label - label = sphinx_gettext(label) text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) if len(self.arguments) == 3: inodes, messages = self.state.inline_text(self.arguments[2], From webhook-mailer at python.org Tue Aug 8 08:12:53 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Tue, 08 Aug 2023 12:12:53 -0000 Subject: [Python-checkins] gh-106368: Argument clinic: add tests for more failure paths (#107731) Message-ID: https://github.com/python/cpython/commit/7c5153de5a2bd2c886173a317f116885a925cfce commit: 7c5153de5a2bd2c886173a317f116885a925cfce branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-08T12:12:49Z summary: gh-106368: Argument clinic: add tests for more failure paths (#107731) files: M Lib/test/test_clinic.py M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index d13d8623f8093..6c2411f9a57b6 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -45,6 +45,7 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None): tc.assertEqual(cm.exception.filename, filename) if lineno is not None: tc.assertEqual(cm.exception.lineno, lineno) + return cm.exception class ClinicWholeFileTest(TestCase): @@ -222,6 +223,15 @@ def test_directive_output_print(self): last_line.startswith("/*[clinic end generated code: output=") ) + def test_directive_wrong_arg_number(self): + raw = dedent(""" + /*[clinic input] + preserve foo bar baz eggs spam ham mushrooms + [clinic start generated code]*/ + """) + err = "takes 1 positional argument but 8 were given" + self.expect_failure(raw, err) + def test_unknown_destination_command(self): raw = """ /*[clinic input] @@ -600,6 +610,31 @@ def test_directive_output_invalid_command(self): self.expect_failure(block, err, lineno=2) +class ParseFileUnitTest(TestCase): + def expect_parsing_failure( + self, *, filename, expected_error, verify=True, output=None + ): + errmsg = re.escape(dedent(expected_error).strip()) + with self.assertRaisesRegex(clinic.ClinicError, errmsg): + clinic.parse_file(filename) + + def test_parse_file_no_extension(self) -> None: + self.expect_parsing_failure( + filename="foo", + expected_error="Can't extract file type for file 'foo'" + ) + + def test_parse_file_strange_extension(self) -> None: + filenames_to_errors = { + "foo.rs": "Can't identify file type for file 'foo.rs'", + "foo.hs": "Can't identify file type for file 'foo.hs'", + "foo.js": "Can't identify file type for file 'foo.js'", + } + for filename, errmsg in filenames_to_errors.items(): + with self.subTest(filename=filename): + self.expect_parsing_failure(filename=filename, expected_error=errmsg) + + class ClinicGroupPermuterTest(TestCase): def _test(self, l, m, r, output): computed = clinic.permute_optional_groups(l, m, r) @@ -794,8 +829,8 @@ def parse_function(self, text, signatures_in_block=2, function_index=1): return s[function_index] def expect_failure(self, block, err, *, filename=None, lineno=None): - _expect_failure(self, self.parse_function, block, err, - filename=filename, lineno=lineno) + return _expect_failure(self, self.parse_function, block, err, + filename=filename, lineno=lineno) def checkDocstring(self, fn, expected): self.assertTrue(hasattr(fn, "docstring")) @@ -877,6 +912,41 @@ def test_param_default_expr_named_constant(self): """ self.expect_failure(block, err, lineno=2) + def test_param_with_bizarre_default_fails_correctly(self): + template = """ + module os + os.access + follow_symlinks: int = {default} + """ + err = "Unsupported expression as default value" + for bad_default_value in ( + "{1, 2, 3}", + "3 if bool() else 4", + "[x for x in range(42)]" + ): + with self.subTest(bad_default=bad_default_value): + block = template.format(default=bad_default_value) + self.expect_failure(block, err, lineno=2) + + def test_unspecified_not_allowed_as_default_value(self): + block = """ + module os + os.access + follow_symlinks: int(c_default='MAXSIZE') = unspecified + """ + err = "'unspecified' is not a legal default value!" + exc = self.expect_failure(block, err, lineno=2) + self.assertNotIn('Malformed expression given as default value', str(exc)) + + def test_malformed_expression_as_default_value(self): + block = """ + module os + os.access + follow_symlinks: int(c_default='MAXSIZE') = 1/0 + """ + err = "Malformed expression given as default value" + self.expect_failure(block, err, lineno=2) + def test_param_default_expr_binop(self): err = ( "When you specify an expression ('a + b') as your default value, " @@ -1041,6 +1111,28 @@ def test_c_name(self): """) self.assertEqual("os_stat_fn", function.c_basename) + def test_base_invalid_syntax(self): + block = """ + module os + os.stat + invalid syntax: int = 42 + """ + err = dedent(r""" + Function 'stat' has an invalid parameter declaration: + \s+'invalid syntax: int = 42' + """).strip() + with self.assertRaisesRegex(clinic.ClinicError, err): + self.parse_function(block) + + def test_param_default_invalid_syntax(self): + block = """ + module os + os.stat + x: int = invalid syntax + """ + err = r"Syntax error: 'x = invalid syntax\n'" + self.expect_failure(block, err, lineno=2) + def test_cloning_nonexistent_function_correctly_fails(self): block = """ cloned = fooooooooooooooooo @@ -1414,18 +1506,6 @@ def test_parameters_required_after_star(self): with self.subTest(block=block): self.expect_failure(block, err) - def test_parameters_required_after_depr_star(self): - dataset = ( - "module foo\nfoo.bar\n * [from 3.14]", - "module foo\nfoo.bar\n * [from 3.14]\nDocstring here.", - "module foo\nfoo.bar\n this: int\n * [from 3.14]", - "module foo\nfoo.bar\n this: int\n * [from 3.14]\nDocstring.", - ) - err = "Function 'foo.bar' specifies '* [from 3.14]' without any parameters afterwards." - for block in dataset: - with self.subTest(block=block): - self.expect_failure(block, err) - def test_depr_star_invalid_format_1(self): block = """ module foo diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index c6cf43ab40fb1..0b336d9ac5a60 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -5207,13 +5207,14 @@ def bad_node(self, node: ast.AST) -> None: # but at least make an attempt at ensuring it's a valid expression. try: value = eval(default) - if value is unspecified: - fail("'unspecified' is not a legal default value!") except NameError: pass # probably a named constant except Exception as e: fail("Malformed expression given as default value " f"{default!r} caused {e!r}") + else: + if value is unspecified: + fail("'unspecified' is not a legal default value!") if bad: fail(f"Unsupported expression as default value: {default!r}") From webhook-mailer at python.org Tue Aug 8 08:47:20 2023 From: webhook-mailer at python.org (zooba) Date: Tue, 08 Aug 2023 12:47:20 -0000 Subject: [Python-checkins] gh-91795: Update build optimization part of PCbuild/readme.txt (GH-91849) Message-ID: https://github.com/python/cpython/commit/906b73be5eada1995bd667a02c59f7a11998310f commit: 906b73be5eada1995bd667a02c59f7a11998310f branch: main author: Fatih <77548106+fatihkabakk at users.noreply.github.com> committer: zooba date: 2023-08-08T13:47:15+01:00 summary: gh-91795: Update build optimization part of PCbuild/readme.txt (GH-91849) files: M PCbuild/readme.txt diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 86ad3ab1a40d9..1d3b45b7912a3 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -251,9 +251,11 @@ against a profiling library and contain extra debug information. The PGUpdate configuration takes the profiling data and generates optimized binaries. -The build_pgo.bat script automates the creation of optimized binaries. -It creates the PGI files, runs the unit test suite or PyBench with the -PGI python, and finally creates the optimized files. +The build.bat script has an argument `--pgo` that automate the creation +of optimized binaries. +It creates the PGI files, runs the unit test suite with the PGI python, +and finally creates the optimized files. +You can customize the job for profiling with `--pgo-job ` option. See https://docs.microsoft.com/en-us/cpp/build/profile-guided-optimizations From webhook-mailer at python.org Tue Aug 8 10:25:59 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 08 Aug 2023 14:25:59 -0000 Subject: [Python-checkins] [3.12] gh-98154: Clarify Usage of "Reference Count" In the Docs (gh-107552) (#107752) Message-ID: https://github.com/python/cpython/commit/aa2ecef22a66938cba072ea57c27c63c11f79c9a commit: aa2ecef22a66938cba072ea57c27c63c11f79c9a branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-08T16:25:35+02:00 summary: [3.12] gh-98154: Clarify Usage of "Reference Count" In the Docs (gh-107552) (#107752) * gh-98154: Clarify Usage of "Reference Count" In the Docs (gh-107552) PEP 683 (immortal objects) revealed some ways in which the Python documentation has been unnecessarily coupled to the implementation details of reference counts. In the end users should focus on reference ownership, including taking references and releasing them, rather than on how many reference counts an object has. This change updates the documentation to reflect that perspective. It also updates the docs relative to immortal objects in a handful of places. (cherry picked from commit 5dc825d504ad08d64c9d1ce578f9deebbe012604) Co-authored-by: Eric Snow * Fix a typo. --------- Co-authored-by: Eric Snow files: M Doc/c-api/allocation.rst M Doc/c-api/arg.rst M Doc/c-api/buffer.rst M Doc/c-api/bytes.rst M Doc/c-api/exceptions.rst M Doc/c-api/init_config.rst M Doc/c-api/intro.rst M Doc/c-api/module.rst M Doc/c-api/object.rst M Doc/c-api/refcounting.rst M Doc/c-api/sys.rst M Doc/c-api/typeobj.rst M Doc/c-api/unicode.rst M Doc/glossary.rst M Doc/library/sys.rst diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 44747e2964366..b3609c233156b 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -29,12 +29,13 @@ Allocating Objects on the Heap .. c:macro:: PyObject_New(TYPE, typeobj) - Allocate a new Python object using the C structure type *TYPE* and the - Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header - are not initialized; the object's reference count will be one. The size of - the memory allocation is determined from the :c:member:`~PyTypeObject.tp_basicsize` field of - the type object. + Allocate a new Python object using the C structure type *TYPE* + and the Python type object *typeobj* (``PyTypeObject*``). + Fields not defined by the Python object header are not initialized. + The caller will own the only reference to the object + (i.e. its reference count will be one). + The size of the memory allocation is determined from the + :c:member:`~PyTypeObject.tp_basicsize` field of the type object. .. c:macro:: PyObject_NewVar(TYPE, typeobj, size) diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index c09c2d5261ca0..657b10d3e0ac4 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -293,8 +293,10 @@ Other objects ``O`` (object) [PyObject \*] Store a Python object (without any conversion) in a C object pointer. The C - program thus receives the actual object that was passed. The object's reference - count is not increased. The pointer stored is not ``NULL``. + program thus receives the actual object that was passed. A new + :term:`strong reference` to the object is not created + (i.e. its reference count is not increased). + The pointer stored is not ``NULL``. ``O!`` (object) [*typeobject*, PyObject \*] Store a Python object in a C object pointer. This is similar to ``O``, but @@ -378,7 +380,8 @@ inside nested parentheses. They are: mutually exclude each other. Note that any Python object references which are provided to the caller are -*borrowed* references; do not decrement their reference count! +*borrowed* references; do not release them +(i.e. do not decrement their reference count)! Additional arguments passed to these functions must be addresses of variables whose type is determined by the format string; these are used to store values @@ -613,8 +616,10 @@ Building values Convert a C :c:type:`Py_complex` structure to a Python complex number. ``O`` (object) [PyObject \*] - Pass a Python object untouched (except for its reference count, which is - incremented by one). If the object passed in is a ``NULL`` pointer, it is assumed + Pass a Python object untouched but create a new + :term:`strong reference` to it + (i.e. its reference count is incremented by one). + If the object passed in is a ``NULL`` pointer, it is assumed that this was caused because the call producing the argument found an error and set an exception. Therefore, :c:func:`Py_BuildValue` will return ``NULL`` but won't raise an exception. If no exception has been raised yet, :exc:`SystemError` is @@ -624,7 +629,7 @@ Building values Same as ``O``. ``N`` (object) [PyObject \*] - Same as ``O``, except it doesn't increment the reference count on the object. + Same as ``O``, except it doesn't create a new :term:`strong reference`. Useful when the object is created by a call to an object constructor in the argument list. diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 02b53ec149c73..8ca1c190dab9a 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -102,7 +102,9 @@ a buffer, see :c:func:`PyObject_GetBuffer`. .. c:member:: PyObject *obj A new reference to the exporting object. The reference is owned by - the consumer and automatically decremented and set to ``NULL`` by + the consumer and automatically released + (i.e. reference count decremented) + and set to ``NULL`` by :c:func:`PyBuffer_Release`. The field is the equivalent of the return value of any standard C-API function. @@ -454,7 +456,8 @@ Buffer-related functions .. c:function:: void PyBuffer_Release(Py_buffer *view) - Release the buffer *view* and decrement the reference count for + Release the buffer *view* and release the :term:`strong reference` + (i.e. decrement the reference count) to the view's supporting object, ``view->obj``. This function MUST be called when the buffer is no longer being used, otherwise reference leaks may occur. diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 4e3ffc7e23e3f..61a68f5277388 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -184,8 +184,8 @@ called with a non-bytes parameter. .. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) Create a new bytes object in *\*bytes* containing the contents of *newpart* - appended to *bytes*. This version decrements the reference count of - *newpart*. + appended to *bytes*. This version releases the :term:`strong reference` + to *newpart* (i.e. decrements its reference count). .. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index a2126ffc559ab..f1d6c995188ab 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -110,7 +110,8 @@ For convenience, some of these functions will always return a This is the most common way to set the error indicator. The first argument specifies the exception type; it is normally one of the standard exceptions, - e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. + e.g. :c:data:`PyExc_RuntimeError`. You need not create a new + :term:`strong reference` to it (e.g. with :c:func:`Py_INCREF`). The second argument is an error message; it is decoded from ``'utf-8'``. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 9da9f873e7448..96705c804f11d 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1109,7 +1109,7 @@ PyConfig .. c:member:: int show_ref_count - Show total reference count at exit? + Show total reference count at exit (excluding immortal objects)? Set to ``1`` by :option:`-X showrefcount <-X>` command line option. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 42a5db1893472..4a6cb7ab3b838 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -287,52 +287,58 @@ true if (and only if) the object pointed to by *a* is a Python list. Reference Counts ---------------- -The reference count is important because today's computers have a finite (and -often severely limited) memory size; it counts how many different places there -are that have a reference to an object. Such a place could be another object, -or a global (or static) C variable, or a local variable in some C function. -When an object's reference count becomes zero, the object is deallocated. If -it contains references to other objects, their reference count is decremented. -Those other objects may be deallocated in turn, if this decrement makes their -reference count become zero, and so on. (There's an obvious problem with -objects that reference each other here; for now, the solution is "don't do -that.") +The reference count is important because today's computers have a finite +(and often severely limited) memory size; it counts how many different +places there are that have a :term:`strong reference` to an object. +Such a place could be another object, or a global (or static) C variable, +or a local variable in some C function. +When the last :term:`strong reference` to an object is released +(i.e. its reference count becomes zero), the object is deallocated. +If it contains references to other objects, those references are released. +Those other objects may be deallocated in turn, if there are no more +references to them, and so on. (There's an obvious problem with +objects that reference each other here; for now, the solution +is "don't do that.") .. index:: single: Py_INCREF() single: Py_DECREF() -Reference counts are always manipulated explicitly. The normal way is to use -the macro :c:func:`Py_INCREF` to increment an object's reference count by one, -and :c:func:`Py_DECREF` to decrement it by one. The :c:func:`Py_DECREF` macro +Reference counts are always manipulated explicitly. The normal way is +to use the macro :c:func:`Py_INCREF` to take a new reference to an +object (i.e. increment its reference count by one), +and :c:func:`Py_DECREF` to release that reference (i.e. decrement the +reference count by one). The :c:func:`Py_DECREF` macro is considerably more complex than the incref one, since it must check whether the reference count becomes zero and then cause the object's deallocator to be -called. The deallocator is a function pointer contained in the object's type -structure. The type-specific deallocator takes care of decrementing the -reference counts for other objects contained in the object if this is a compound +called. The deallocator is a function pointer contained in the object's type +structure. The type-specific deallocator takes care of releasing references +for other objects contained in the object if this is a compound object type, such as a list, as well as performing any additional finalization that's needed. There's no chance that the reference count can overflow; at least as many bits are used to hold the reference count as there are distinct memory locations in virtual memory (assuming ``sizeof(Py_ssize_t) >= sizeof(void*)``). Thus, the reference count increment is a simple operation. -It is not necessary to increment an object's reference count for every local -variable that contains a pointer to an object. In theory, the object's +It is not necessary to hold a :term:`strong reference` (i.e. increment +the reference count) for every local variable that contains a pointer +to an object. In theory, the object's reference count goes up by one when the variable is made to point to it and it goes down by one when the variable goes out of scope. However, these two cancel each other out, so at the end the reference count hasn't changed. The only real reason to use the reference count is to prevent the object from being deallocated as long as our variable is pointing to it. If we know that there is at least one other reference to the object that lives at least as long as -our variable, there is no need to increment the reference count temporarily. +our variable, there is no need to take a new :term:`strong reference` +(i.e. increment the reference count) temporarily. An important situation where this arises is in objects that are passed as arguments to C functions in an extension module that are called from Python; the call mechanism guarantees to hold a reference to every argument for the duration of the call. However, a common pitfall is to extract an object from a list and hold on to it -for a while without incrementing its reference count. Some other operation might -conceivably remove the object from the list, decrementing its reference count +for a while without taking a new reference. Some other operation might +conceivably remove the object from the list, releasing that reference, and possibly deallocating it. The real danger is that innocent-looking operations may invoke arbitrary Python code which could do this; there is a code path which allows control to flow back to the user from a :c:func:`Py_DECREF`, so @@ -340,7 +346,8 @@ almost any operation is potentially dangerous. A safe approach is to always use the generic operations (functions whose name begins with ``PyObject_``, ``PyNumber_``, ``PySequence_`` or ``PyMapping_``). -These operations always increment the reference count of the object they return. +These operations always create a new :term:`strong reference` +(i.e. increment the reference count) of the object they return. This leaves the caller with the responsibility to call :c:func:`Py_DECREF` when they are done with the result; this soon becomes second nature. @@ -356,7 +363,7 @@ to objects (objects are not owned: they are always shared). "Owning a reference" means being responsible for calling Py_DECREF on it when the reference is no longer needed. Ownership can also be transferred, meaning that the code that receives ownership of the reference then becomes responsible for -eventually decref'ing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF` +eventually releasing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF` when it's no longer needed---or passing on this responsibility (usually to its caller). When a function passes ownership of a reference on to its caller, the caller is said to receive a *new* reference. When no ownership is transferred, @@ -414,9 +421,9 @@ For example, the above two blocks of code could be replaced by the following It is much more common to use :c:func:`PyObject_SetItem` and friends with items whose references you are only borrowing, like arguments that were passed in to -the function you are writing. In that case, their behaviour regarding reference -counts is much saner, since you don't have to increment a reference count so you -can give a reference away ("have it be stolen"). For example, this function +the function you are writing. In that case, their behaviour regarding references +is much saner, since you don't have to take a new reference just so you +can give that reference away ("have it be stolen"). For example, this function sets all items of a list (actually, any mutable sequence) to a given item:: int diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 2b6875d533c82..cb6b0a47681e9 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -529,7 +529,7 @@ state: .. note:: Unlike other functions that steal references, ``PyModule_AddObject()`` - only decrements the reference count of *value* **on success**. + only releases the reference to *value* **on success**. This means that its return value must be checked, and calling code must :c:func:`Py_DECREF` *value* manually on error. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 5b3c3615f7581..4a3fed1a6087a 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -15,8 +15,8 @@ Object Protocol .. c:macro:: Py_RETURN_NOTIMPLEMENTED Properly handle returning :c:data:`Py_NotImplemented` from within a C - function (that is, increment the reference count of NotImplemented and - return it). + function (that is, create a new :term:`strong reference` + to NotImplemented and return it). .. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags) @@ -320,11 +320,12 @@ Object Protocol When *o* is non-``NULL``, returns a type object corresponding to the object type of object *o*. On failure, raises :exc:`SystemError` and returns ``NULL``. This - is equivalent to the Python expression ``type(o)``. This function increments the - reference count of the return value. There's really no reason to use this + is equivalent to the Python expression ``type(o)``. + This function creates a new :term:`strong reference` to the return value. + There's really no reason to use this function instead of the :c:func:`Py_TYPE()` function, which returns a - pointer of type :c:expr:`PyTypeObject*`, except when the incremented reference - count is needed. + pointer of type :c:expr:`PyTypeObject*`, except when a new + :term:`strong reference` is needed. .. c:function:: int PyObject_TypeCheck(PyObject *o, PyTypeObject *type) diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 640c5c610899f..4ea0378d02a65 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -15,6 +15,12 @@ of Python objects. Get the reference count of the Python object *o*. + Note that the returned value may not actually reflect how many + references to the object are actually held. For example, some + objects are "immortal" and have a very high refcount that does not + reflect the actual number of references. Consequently, do not rely + on the returned value to be accurate, other than a value of 0 or 1. + Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count. .. versionchanged:: 3.11 @@ -28,36 +34,53 @@ of Python objects. Set the object *o* reference counter to *refcnt*. + Note that this function has no effect on + `immortal `_ + objects. + .. versionadded:: 3.9 + .. versionchanged:: 3.12 + Immortal objects are not modified. + .. c:function:: void Py_INCREF(PyObject *o) - Increment the reference count for object *o*. + Indicate taking a new :term:`strong reference` to object *o*, + indicating it is in use and should not be destroyed. This function is usually used to convert a :term:`borrowed reference` to a :term:`strong reference` in-place. The :c:func:`Py_NewRef` function can be used to create a new :term:`strong reference`. + When done using the object, release it by calling :c:func:`Py_DECREF`. + The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, use :c:func:`Py_XINCREF`. + Do not expect this function to actually modify *o* in any way. + For at least `some objects `_, + this function has no effect. + + .. versionchanged:: 3.12 + Immortal objects are not modified. + .. c:function:: void Py_XINCREF(PyObject *o) - Increment the reference count for object *o*. The object may be ``NULL``, in - which case the macro has no effect. + Similar to :c:func:`Py_INCREF`, but the object *o* can be ``NULL``, + in which case this has no effect. See also :c:func:`Py_XNewRef`. .. c:function:: PyObject* Py_NewRef(PyObject *o) - Create a new :term:`strong reference` to an object: increment the reference - count of the object *o* and return the object *o*. + Create a new :term:`strong reference` to an object: + call :c:func:`Py_INCREF` on *o* and return the object *o*. When the :term:`strong reference` is no longer needed, :c:func:`Py_DECREF` - should be called on it to decrement the object reference count. + should be called on it to release the reference. The object *o* must not be ``NULL``; use :c:func:`Py_XNewRef` if *o* can be ``NULL``. @@ -87,9 +110,12 @@ of Python objects. .. c:function:: void Py_DECREF(PyObject *o) - Decrement the reference count for object *o*. + Release a :term:`strong reference` to object *o*, indicating the + reference is no longer used. - If the reference count reaches zero, the object's type's deallocation + Once the last :term:`strong reference` is released + (i.e. the object's reference count reaches 0), + the object's type's deallocation function (which must not be ``NULL``) is invoked. This function is usually used to delete a :term:`strong reference` before @@ -98,6 +124,10 @@ of Python objects. The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, use :c:func:`Py_XDECREF`. + Do not expect this function to actually modify *o* in any way. + For at least `some objects `_, + this function has no effect. + .. warning:: The deallocation function can cause arbitrary Python code to be invoked (e.g. @@ -109,25 +139,29 @@ of Python objects. reference to the deleted object in a temporary variable, update the list data structure, and then call :c:func:`Py_DECREF` for the temporary variable. + .. versionchanged:: 3.12 + Immortal objects are not modified. + .. c:function:: void Py_XDECREF(PyObject *o) - Decrement the reference count for object *o*. The object may be ``NULL``, in - which case the macro has no effect; otherwise the effect is the same as for - :c:func:`Py_DECREF`, and the same warning applies. + Similar to :c:func:`Py_DECREF`, but the object *o* can be ``NULL``, + in which case this has no effect. + The same warning from :c:func:`Py_DECREF` applies here as well. .. c:function:: void Py_CLEAR(PyObject *o) - Decrement the reference count for object *o*. The object may be ``NULL``, in + Release a :term:`strong reference` for object *o*. + The object may be ``NULL``, in which case the macro has no effect; otherwise the effect is the same as for :c:func:`Py_DECREF`, except that the argument is also set to ``NULL``. The warning for :c:func:`Py_DECREF` does not apply with respect to the object passed because the macro carefully uses a temporary variable and sets the argument to ``NULL`` - before decrementing its reference count. + before releasing the reference. - It is a good idea to use this macro whenever decrementing the reference - count of an object that might be traversed during garbage collection. + It is a good idea to use this macro whenever releasing a reference + to an object that might be traversed during garbage collection. .. versionchanged:: 3.12 The macro argument is now only evaluated once. If the argument has side @@ -136,20 +170,22 @@ of Python objects. .. c:function:: void Py_IncRef(PyObject *o) - Increment the reference count for object *o*. A function version of :c:func:`Py_XINCREF`. + Indicate taking a new :term:`strong reference` to object *o*. + A function version of :c:func:`Py_XINCREF`. It can be used for runtime dynamic embedding of Python. .. c:function:: void Py_DecRef(PyObject *o) - Decrement the reference count for object *o*. A function version of :c:func:`Py_XDECREF`. + Release a :term:`strong reference` to object *o*. + A function version of :c:func:`Py_XDECREF`. It can be used for runtime dynamic embedding of Python. .. c:macro:: Py_SETREF(dst, src) - Macro safely decrementing the `dst` reference count and setting `dst` to - `src`. + Macro safely releasing a :term:`strong reference` to object *dst* + and setting *dst* to *src*. As in case of :c:func:`Py_CLEAR`, "the obvious" code can be deadly:: @@ -160,9 +196,10 @@ of Python objects. Py_SETREF(dst, src); - That arranges to set `dst` to `src` _before_ decrementing reference count of - *dst* old value, so that any code triggered as a side-effect of `dst` - getting torn down no longer believes `dst` points to a valid object. + That arranges to set *dst* to *src* _before_ releasing the reference + to the old value of *dst*, so that any code triggered as a side-effect + of *dst* getting torn down no longer believes *dst* points + to a valid object. .. versionadded:: 3.6 diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 1cec0666f5319..a8a284e6e1cf9 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -8,8 +8,9 @@ Operating System Utilities .. c:function:: PyObject* PyOS_FSPath(PyObject *path) Return the file system representation for *path*. If the object is a - :class:`str` or :class:`bytes` object, then its reference count is - incremented. If the object implements the :class:`os.PathLike` interface, + :class:`str` or :class:`bytes` object, then a new + :term:`strong reference` is returned. + If the object implements the :class:`os.PathLike` interface, then :meth:`~os.PathLike.__fspath__` is returned as long as it is a :class:`str` or :class:`bytes` object. Otherwise :exc:`TypeError` is raised and ``NULL`` is returned. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 26e6133aebaa8..221a05b192240 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -690,7 +690,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) } Finally, if the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the - deallocator should decrement the reference count for its type object after + deallocator should release the owned reference to its type object + (via :c:func:`Py_DECREF`) after calling the type deallocator. In order to avoid dangling pointers, the recommended way to achieve this is: @@ -1461,9 +1462,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) } The :c:func:`Py_CLEAR` macro should be used, because clearing references is - delicate: the reference to the contained object must not be decremented until + delicate: the reference to the contained object must not be released + (via :c:func:`Py_DECREF`) until after the pointer to the contained object is set to ``NULL``. This is because - decrementing the reference count may cause the contained object to become trash, + releasing the reference may cause the contained object to become trash, triggering a chain of reclamation activity that may include invoking arbitrary Python code (due to finalizers, or weakref callbacks, associated with the contained object). If it's possible for such code to reference *self* again, @@ -1541,7 +1543,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) they may be C ints or floats). The third argument specifies the requested operation, as for :c:func:`PyObject_RichCompare`. - The return value's reference count is properly incremented. + The returned value is a new :term:`strong reference`. On error, sets an exception and returns ``NULL`` from the function. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 02f9597c345a0..586ad182e3a45 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -571,7 +571,7 @@ APIs: Copy an instance of a Unicode subtype to a new true Unicode object if necessary. If *obj* is already a true Unicode object (not a subtype), - return the reference with incremented refcount. + return a new :term:`strong reference` to the object. Objects other than Unicode or its subtypes will cause a :exc:`TypeError`. @@ -1445,11 +1445,11 @@ They all return ``NULL`` or ``-1`` if an exception occurs. Intern the argument *\*string* in place. The argument must be the address of a pointer variable pointing to a Python Unicode string object. If there is an existing interned string that is the same as *\*string*, it sets *\*string* to - it (decrementing the reference count of the old string object and incrementing - the reference count of the interned string object), otherwise it leaves - *\*string* alone and interns it (incrementing its reference count). - (Clarification: even though there is a lot of talk about reference counts, think - of this function as reference-count-neutral; you own the object after the call + it (releasing the reference to the old string object and creating a new + :term:`strong reference` to the interned string object), otherwise it leaves + *\*string* alone and interns it (creating a new :term:`strong reference`). + (Clarification: even though there is a lot of talk about references, think + of this function as reference-neutral; you own the object after the call if and only if you owned it before the call.) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 5c0f0f15217a0..f3d5c5eede970 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -168,8 +168,9 @@ Glossary :class:`str` objects. borrowed reference - In Python's C API, a borrowed reference is a reference to an object. - It does not modify the object reference count. It becomes a dangling + In Python's C API, a borrowed reference is a reference to an object, + where the code using the object does not own the reference. + It becomes a dangling pointer if the object is destroyed. For example, a garbage collection can remove the last :term:`strong reference` to the object and so destroy it. @@ -1063,7 +1064,9 @@ Glossary reference count The number of references to an object. When the reference count of an - object drops to zero, it is deallocated. Reference counting is + object drops to zero, it is deallocated. Some objects are + "immortal" and have reference counts that are never modified, and + therefore the objects are never deallocated. Reference counting is generally not visible to Python code, but it is a key element of the :term:`CPython` implementation. Programmers can call the :func:`sys.getrefcount` function to return the @@ -1131,8 +1134,10 @@ Glossary strong reference In Python's C API, a strong reference is a reference to an object - which increments the object's reference count when it is created and - decrements the object's reference count when it is deleted. + which is owned by the code holding the reference. The strong + reference is taken by calling :c:func:`Py_INCREF` when the + reference is created and released with :c:func:`Py_DECREF` + when the reference is deleted. The :c:func:`Py_NewRef` function can be used to create a strong reference to an object. Usually, the :c:func:`Py_DECREF` function must be called on diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index a61737383e393..33391d11ab392 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -770,6 +770,15 @@ always available. higher than you might expect, because it includes the (temporary) reference as an argument to :func:`getrefcount`. + Note that the returned value may not actually reflect how many + references to the object are actually held. For example, some + objects are "immortal" and have a very high refcount that does not + reflect the actual number of references. Consequently, do not rely + on the returned value to be accurate, other than a value of 0 or 1. + + .. versionchanged:: 3.12 + Immortal objects have very large refcounts that do not match + the actual number of references to the object. .. function:: getrecursionlimit() From webhook-mailer at python.org Tue Aug 8 11:52:17 2023 From: webhook-mailer at python.org (ericsnowcurrently) Date: Tue, 08 Aug 2023 15:52:17 -0000 Subject: [Python-checkins] gh-105699: Re-enable the Multiple-Interpreters Stress Tests (gh-107572) Message-ID: https://github.com/python/cpython/commit/f9e3ff1ea4b2c8b787360409d55f2037652b7456 commit: f9e3ff1ea4b2c8b787360409d55f2037652b7456 branch: main author: Eric Snow committer: ericsnowcurrently date: 2023-08-08T09:52:13-06:00 summary: gh-105699: Re-enable the Multiple-Interpreters Stress Tests (gh-107572) We had disabled them due to crashes they exposed, which have since been fixed. files: M Lib/test/test_interpreters.py diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 662eafa3b7a10..5981d96de8de0 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -464,7 +464,6 @@ def test_bytes_for_script(self): # test_xxsubinterpreters covers the remaining Interpreter.run() behavior. - at unittest.skip('these are crashing, likely just due just to _xxsubinterpreters (see gh-105699)') class StressTests(TestBase): # In these tests we generally want a lot of interpreters, From webhook-mailer at python.org Tue Aug 8 12:12:56 2023 From: webhook-mailer at python.org (rhettinger) Date: Tue, 08 Aug 2023 16:12:56 -0000 Subject: [Python-checkins] Minor accuracy improvement for statistics.correlation() (GH-107781) Message-ID: https://github.com/python/cpython/commit/d4ac094cf9d15ec5705ec0fe8771df9e6ba915b9 commit: d4ac094cf9d15ec5705ec0fe8771df9e6ba915b9 branch: main author: Raymond Hettinger committer: rhettinger date: 2023-08-08T17:12:52+01:00 summary: Minor accuracy improvement for statistics.correlation() (GH-107781) files: M Lib/statistics.py diff --git a/Lib/statistics.py b/Lib/statistics.py index 6bd214bbfe2ff..066669d25ddb1 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -1004,6 +1004,14 @@ def _mean_stdev(data): # Handle Nans and Infs gracefully return float(xbar), float(xbar) / float(ss) +def _sqrtprod(x: float, y: float) -> float: + "Return sqrt(x * y) computed with high accuracy." + # Square root differential correction: + # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 + h = sqrt(x * y) + x = sumprod((x, h), (y, -h)) + return h + x / (2.0 * h) + # === Statistics for relations between two inputs === @@ -1083,7 +1091,7 @@ def correlation(x, y, /, *, method='linear'): sxx = sumprod(x, x) syy = sumprod(y, y) try: - return sxy / sqrt(sxx * syy) + return sxy / _sqrtprod(sxx, syy) except ZeroDivisionError: raise StatisticsError('at least one of the inputs is constant') From webhook-mailer at python.org Tue Aug 8 13:30:37 2023 From: webhook-mailer at python.org (rhettinger) Date: Tue, 08 Aug 2023 17:30:37 -0000 Subject: [Python-checkins] GH-100425: Note improved commutativity in sum(). (GH-107785) Message-ID: https://github.com/python/cpython/commit/aab6f7173a3b825599629dd6fa5cb7e477421595 commit: aab6f7173a3b825599629dd6fa5cb7e477421595 branch: main author: Raymond Hettinger committer: rhettinger date: 2023-08-08T18:30:33+01:00 summary: GH-100425: Note improved commutativity in sum(). (GH-107785) files: M Doc/library/functions.rst M Doc/whatsnew/3.12.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index b271067ae639c..88a7fdfe6f0d5 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1752,7 +1752,7 @@ are always available. They are listed here in alphabetical order. The *start* parameter can be specified as a keyword argument. .. versionchanged:: 3.12 Summation of floats switched to an algorithm - that gives higher accuracy on most builds. + that gives higher accuracy and better commutativity on most builds. .. class:: super() diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 6553013552d00..8ed435476c9ce 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -504,8 +504,8 @@ Other Language Changes * :class:`slice` objects are now hashable, allowing them to be used as dict keys and set items. (Contributed by Will Bradshaw, Furkan Onder, and Raymond Hettinger in :gh:`101264`.) -* :func:`sum` now uses Neumaier summation to improve accuracy when summing - floats or mixed ints and floats. +* :func:`sum` now uses Neumaier summation to improve accuracy and commutativity + when summing floats or mixed ints and floats. (Contributed by Raymond Hettinger in :gh:`100425`.) * Exceptions raised in a typeobject's ``__set_name__`` method are no longer From webhook-mailer at python.org Tue Aug 8 16:42:47 2023 From: webhook-mailer at python.org (brandtbucher) Date: Tue, 08 Aug 2023 20:42:47 -0000 Subject: [Python-checkins] GH-107596: Specialize str[int] (GH-107597) Message-ID: https://github.com/python/cpython/commit/ea72c6fe3b6db5f4e8ce3d3405c0ea65dc002faf commit: ea72c6fe3b6db5f4e8ce3d3405c0ea65dc002faf branch: main author: Brandt Bucher committer: brandtbucher date: 2023-08-08T13:42:43-07:00 summary: GH-107596: Specialize str[int] (GH-107597) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-03-11-13-09.gh-issue-107596.T3yPGI.rst M Include/internal/pycore_opcode.h M Include/internal/pycore_opcode_metadata.h M Include/opcode.h M Lib/_opcode_metadata.py M Python/bytecodes.c M Python/executor_cases.c.h M Python/generated_cases.c.h M Python/opcode_targets.h M Python/specialize.c diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index aff09a2a926ae..a187da6e24730 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -52,6 +52,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [BINARY_SUBSCR_DICT] = BINARY_SUBSCR, [BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR, [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, + [BINARY_SUBSCR_STR_INT] = BINARY_SUBSCR, [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR, [BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP, [BUILD_LIST] = BUILD_LIST, @@ -292,12 +293,12 @@ const char *const _PyOpcode_OpName[268] = { [FORMAT_SIMPLE] = "FORMAT_SIMPLE", [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", + [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [SEND_GEN] = "SEND_GEN", [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -305,39 +306,39 @@ const char *const _PyOpcode_OpName[268] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", + [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [GET_ITER] = "GET_ITER", [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", [RETURN_GENERATOR] = "RETURN_GENERATOR", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", [RETURN_VALUE] = "RETURN_VALUE", - [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [COMPARE_OP_INT] = "COMPARE_OP_INT", + [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", [LOAD_LOCALS] = "LOAD_LOCALS", - [COMPARE_OP_STR] = "COMPARE_OP_STR", + [COMPARE_OP_INT] = "COMPARE_OP_INT", [POP_EXCEPT] = "POP_EXCEPT", [STORE_NAME] = "STORE_NAME", [DELETE_NAME] = "DELETE_NAME", @@ -360,9 +361,9 @@ const char *const _PyOpcode_OpName[268] = { [IMPORT_NAME] = "IMPORT_NAME", [IMPORT_FROM] = "IMPORT_FROM", [JUMP_FORWARD] = "JUMP_FORWARD", + [COMPARE_OP_STR] = "COMPARE_OP_STR", [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", - [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -381,11 +382,11 @@ const char *const _PyOpcode_OpName[268] = { [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", [RAISE_VARARGS] = "RAISE_VARARGS", [GET_AWAITABLE] = "GET_AWAITABLE", - [FOR_ITER_GEN] = "FOR_ITER_GEN", + [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [BUILD_SLICE] = "BUILD_SLICE", [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", [MAKE_CELL] = "MAKE_CELL", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [FOR_ITER_GEN] = "FOR_ITER_GEN", [LOAD_DEREF] = "LOAD_DEREF", [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", @@ -397,26 +398,26 @@ const char *const _PyOpcode_OpName[268] = { [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", [MAP_ADD] = "MAP_ADD", - [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", [COPY_FREE_VARS] = "COPY_FREE_VARS", [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", + [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", - [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", [CONVERT_VALUE] = "CONVERT_VALUE", + [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", - [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", + [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [LOAD_FAST_LOAD_FAST] = "LOAD_FAST_LOAD_FAST", [STORE_FAST_LOAD_FAST] = "STORE_FAST_LOAD_FAST", [STORE_FAST_STORE_FAST] = "STORE_FAST_STORE_FAST", @@ -427,6 +428,7 @@ const char *const _PyOpcode_OpName[268] = { [LOAD_FROM_DICT_OR_GLOBALS] = "LOAD_FROM_DICT_OR_GLOBALS", [LOAD_FROM_DICT_OR_DEREF] = "LOAD_FROM_DICT_OR_DEREF", [SET_FUNCTION_ATTRIBUTE] = "SET_FUNCTION_ATTRIBUTE", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", [CALL_NO_KW_LIST_APPEND] = "CALL_NO_KW_LIST_APPEND", @@ -435,7 +437,6 @@ const char *const _PyOpcode_OpName[268] = { [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = "CALL_NO_KW_ALLOC_AND_ENTER_INIT", - [186] = "<186>", [187] = "<187>", [188] = "<188>", [189] = "<189>", @@ -521,7 +522,6 @@ const char *const _PyOpcode_OpName[268] = { #endif // NEED_OPCODE_TABLES #define EXTRA_CASES \ - case 186: \ case 187: \ case 188: \ case 189: \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 1cab6c984f3ac..02303c42c75c8 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -144,6 +144,8 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 4; case BINARY_SUBSCR_LIST_INT: return 2; + case BINARY_SUBSCR_STR_INT: + return 2; case BINARY_SUBSCR_TUPLE_INT: return 2; case BINARY_SUBSCR_DICT: @@ -588,6 +590,8 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case BINARY_SUBSCR_LIST_INT: return 1; + case BINARY_SUBSCR_STR_INT: + return 1; case BINARY_SUBSCR_TUPLE_INT: return 1; case BINARY_SUBSCR_DICT: @@ -1047,6 +1051,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [BINARY_SLICE] = { true, INSTR_FMT_IX, 0 }, [STORE_SLICE] = { true, INSTR_FMT_IX, 0 }, [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, 0 }, [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, 0 }, [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, 0 }, [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, 0 }, @@ -1258,6 +1263,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [BINARY_SLICE] = { .nuops = 1, .uops = { { BINARY_SLICE, 0, 0 } } }, [STORE_SLICE] = { .nuops = 1, .uops = { { STORE_SLICE, 0, 0 } } }, [BINARY_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_LIST_INT, 0, 0 } } }, + [BINARY_SUBSCR_STR_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_STR_INT, 0, 0 } } }, [BINARY_SUBSCR_TUPLE_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_TUPLE_INT, 0, 0 } } }, [BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_DICT, 0, 0 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { LIST_APPEND, 0, 0 } } }, diff --git a/Include/opcode.h b/Include/opcode.h index ede1518b6fd25..b3d6cba63096c 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -175,57 +175,58 @@ extern "C" { #define BINARY_SUBSCR_DICT 38 #define BINARY_SUBSCR_GETITEM 39 #define BINARY_SUBSCR_LIST_INT 42 -#define BINARY_SUBSCR_TUPLE_INT 43 -#define STORE_SUBSCR_DICT 44 -#define STORE_SUBSCR_LIST_INT 45 -#define SEND_GEN 46 -#define UNPACK_SEQUENCE_TWO_TUPLE 47 -#define UNPACK_SEQUENCE_TUPLE 48 -#define UNPACK_SEQUENCE_LIST 56 -#define STORE_ATTR_INSTANCE_VALUE 57 -#define STORE_ATTR_SLOT 58 -#define STORE_ATTR_WITH_HINT 59 -#define LOAD_GLOBAL_MODULE 62 -#define LOAD_GLOBAL_BUILTIN 63 -#define LOAD_SUPER_ATTR_ATTR 64 -#define LOAD_SUPER_ATTR_METHOD 65 -#define LOAD_ATTR_INSTANCE_VALUE 66 -#define LOAD_ATTR_MODULE 67 -#define LOAD_ATTR_WITH_HINT 70 -#define LOAD_ATTR_SLOT 72 -#define LOAD_ATTR_CLASS 73 -#define LOAD_ATTR_PROPERTY 76 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 77 -#define LOAD_ATTR_METHOD_WITH_VALUES 78 -#define LOAD_ATTR_METHOD_NO_DICT 79 -#define LOAD_ATTR_METHOD_LAZY_DICT 80 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 81 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 82 -#define COMPARE_OP_FLOAT 84 -#define COMPARE_OP_INT 86 -#define COMPARE_OP_STR 88 -#define FOR_ITER_LIST 111 -#define FOR_ITER_TUPLE 112 -#define FOR_ITER_RANGE 113 -#define FOR_ITER_GEN 132 -#define CALL_BOUND_METHOD_EXACT_ARGS 136 -#define CALL_PY_EXACT_ARGS 148 -#define CALL_PY_WITH_DEFAULTS 153 -#define CALL_NO_KW_TYPE_1 154 -#define CALL_NO_KW_STR_1 155 -#define CALL_NO_KW_TUPLE_1 159 -#define CALL_BUILTIN_CLASS 160 -#define CALL_NO_KW_BUILTIN_O 161 -#define CALL_NO_KW_BUILTIN_FAST 166 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 167 -#define CALL_NO_KW_LEN 178 -#define CALL_NO_KW_ISINSTANCE 179 -#define CALL_NO_KW_LIST_APPEND 180 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 181 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 182 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 183 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 184 -#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 185 +#define BINARY_SUBSCR_STR_INT 43 +#define BINARY_SUBSCR_TUPLE_INT 44 +#define STORE_SUBSCR_DICT 45 +#define STORE_SUBSCR_LIST_INT 46 +#define SEND_GEN 47 +#define UNPACK_SEQUENCE_TWO_TUPLE 48 +#define UNPACK_SEQUENCE_TUPLE 56 +#define UNPACK_SEQUENCE_LIST 57 +#define STORE_ATTR_INSTANCE_VALUE 58 +#define STORE_ATTR_SLOT 59 +#define STORE_ATTR_WITH_HINT 62 +#define LOAD_GLOBAL_MODULE 63 +#define LOAD_GLOBAL_BUILTIN 64 +#define LOAD_SUPER_ATTR_ATTR 65 +#define LOAD_SUPER_ATTR_METHOD 66 +#define LOAD_ATTR_INSTANCE_VALUE 67 +#define LOAD_ATTR_MODULE 70 +#define LOAD_ATTR_WITH_HINT 72 +#define LOAD_ATTR_SLOT 73 +#define LOAD_ATTR_CLASS 76 +#define LOAD_ATTR_PROPERTY 77 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 78 +#define LOAD_ATTR_METHOD_WITH_VALUES 79 +#define LOAD_ATTR_METHOD_NO_DICT 80 +#define LOAD_ATTR_METHOD_LAZY_DICT 81 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 82 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 84 +#define COMPARE_OP_FLOAT 86 +#define COMPARE_OP_INT 88 +#define COMPARE_OP_STR 111 +#define FOR_ITER_LIST 112 +#define FOR_ITER_TUPLE 113 +#define FOR_ITER_RANGE 132 +#define FOR_ITER_GEN 136 +#define CALL_BOUND_METHOD_EXACT_ARGS 148 +#define CALL_PY_EXACT_ARGS 153 +#define CALL_PY_WITH_DEFAULTS 154 +#define CALL_NO_KW_TYPE_1 155 +#define CALL_NO_KW_STR_1 159 +#define CALL_NO_KW_TUPLE_1 160 +#define CALL_BUILTIN_CLASS 161 +#define CALL_NO_KW_BUILTIN_O 166 +#define CALL_NO_KW_BUILTIN_FAST 167 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 178 +#define CALL_NO_KW_LEN 179 +#define CALL_NO_KW_ISINSTANCE 180 +#define CALL_NO_KW_LIST_APPEND 181 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 182 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 183 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 184 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 185 +#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 186 #define NB_ADD 0 #define NB_AND 1 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index fd8ecdb5c980f..17101d1d94757 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -25,6 +25,7 @@ "BINARY_SUBSCR_DICT", "BINARY_SUBSCR_GETITEM", "BINARY_SUBSCR_LIST_INT", + "BINARY_SUBSCR_STR_INT", "BINARY_SUBSCR_TUPLE_INT", ], "STORE_SUBSCR": [ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-03-11-13-09.gh-issue-107596.T3yPGI.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-03-11-13-09.gh-issue-107596.T3yPGI.rst new file mode 100644 index 0000000000000..8912de73680b4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-03-11-13-09.gh-issue-107596.T3yPGI.rst @@ -0,0 +1 @@ +Specialize subscripting :class:`str` objects by :class:`int` indexes. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 90e26d3c86b38..d6bfb624c7713 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -509,6 +509,7 @@ dummy_func( BINARY_SUBSCR_DICT, BINARY_SUBSCR_GETITEM, BINARY_SUBSCR_LIST_INT, + BINARY_SUBSCR_STR_INT, BINARY_SUBSCR_TUPLE_INT, }; @@ -574,6 +575,21 @@ dummy_func( Py_DECREF(list); } + inst(BINARY_SUBSCR_STR_INT, (unused/1, str, sub -- res)) { + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(str); + } + inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple, sub -- res)) { DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9363b4955087d..27be8a383989e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -462,6 +462,29 @@ break; } + case BINARY_SUBSCR_STR_INT: { + PyObject *sub; + PyObject *str; + PyObject *res; + sub = stack_pointer[-1]; + str = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(str); + STACK_SHRINK(1); + stack_pointer[-1] = res; + break; + } + case BINARY_SUBSCR_TUPLE_INT: { PyObject *sub; PyObject *tuple; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7250240ac2396..d7db8b07005fd 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -685,6 +685,30 @@ DISPATCH(); } + TARGET(BINARY_SUBSCR_STR_INT) { + PyObject *sub; + PyObject *str; + PyObject *res; + sub = stack_pointer[-1]; + str = stack_pointer[-2]; + DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); + DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(str); + STACK_SHRINK(1); + stack_pointer[-1] = res; + next_instr += 1; + DISPATCH(); + } + TARGET(BINARY_SUBSCR_TUPLE_INT) { PyObject *sub; PyObject *tuple; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index d84d253c912a2..210c37b37225b 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -42,12 +42,12 @@ static void *opcode_targets[256] = { &&TARGET_FORMAT_SIMPLE, &&TARGET_FORMAT_WITH_SPEC, &&TARGET_BINARY_SUBSCR_LIST_INT, + &&TARGET_BINARY_SUBSCR_STR_INT, &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_STORE_SUBSCR_DICT, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_SEND_GEN, &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, - &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,39 +55,39 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, + &&TARGET_UNPACK_SEQUENCE_TUPLE, &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, + &&TARGET_STORE_ATTR_WITH_HINT, &&TARGET_LOAD_GLOBAL_MODULE, &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_SUPER_ATTR_ATTR, &&TARGET_LOAD_SUPER_ATTR_METHOD, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_LOAD_ATTR_MODULE, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, - &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_SLOT, - &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ASSERTION_ERROR, &&TARGET_RETURN_GENERATOR, + &&TARGET_LOAD_ATTR_CLASS, &&TARGET_LOAD_ATTR_PROPERTY, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, &&TARGET_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, - &&TARGET_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, &&TARGET_RETURN_VALUE, - &&TARGET_COMPARE_OP_FLOAT, + &&TARGET_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_COMPARE_OP_INT, + &&TARGET_COMPARE_OP_FLOAT, &&TARGET_LOAD_LOCALS, - &&TARGET_COMPARE_OP_STR, + &&TARGET_COMPARE_OP_INT, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, &&TARGET_DELETE_NAME, @@ -110,9 +110,9 @@ static void *opcode_targets[256] = { &&TARGET_IMPORT_NAME, &&TARGET_IMPORT_FROM, &&TARGET_JUMP_FORWARD, + &&TARGET_COMPARE_OP_STR, &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_TUPLE, - &&TARGET_FOR_ITER_RANGE, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -131,11 +131,11 @@ static void *opcode_targets[256] = { &&TARGET_POP_JUMP_IF_NONE, &&TARGET_RAISE_VARARGS, &&TARGET_GET_AWAITABLE, - &&TARGET_FOR_ITER_GEN, + &&TARGET_FOR_ITER_RANGE, &&TARGET_BUILD_SLICE, &&TARGET_JUMP_BACKWARD_NO_INTERRUPT, &&TARGET_MAKE_CELL, - &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_FOR_ITER_GEN, &&TARGET_LOAD_DEREF, &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, @@ -147,26 +147,26 @@ static void *opcode_targets[256] = { &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, - &&TARGET_CALL_PY_EXACT_ARGS, + &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, &&TARGET_COPY_FREE_VARS, &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, + &&TARGET_CALL_PY_EXACT_ARGS, &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_CALL_NO_KW_TYPE_1, - &&TARGET_CALL_NO_KW_STR_1, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, &&TARGET_CONVERT_VALUE, + &&TARGET_CALL_NO_KW_STR_1, &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_CALL_BUILTIN_CLASS, - &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, + &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_BUILTIN_FAST, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_LOAD_FAST_LOAD_FAST, &&TARGET_STORE_FAST_LOAD_FAST, &&TARGET_STORE_FAST_STORE_FAST, @@ -177,6 +177,7 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_FROM_DICT_OR_GLOBALS, &&TARGET_LOAD_FROM_DICT_OR_DEREF, &&TARGET_SET_FUNCTION_ATTRIBUTE, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_CALL_NO_KW_LEN, &&TARGET_CALL_NO_KW_ISINSTANCE, &&TARGET_CALL_NO_KW_LIST_APPEND, @@ -228,7 +229,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_ENTER_EXECUTOR, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index de329ef1195cb..855252e066dea 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -363,7 +363,6 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_SUBSCR_ARRAY_SLICE 10 #define SPEC_FAIL_SUBSCR_LIST_SLICE 11 #define SPEC_FAIL_SUBSCR_TUPLE_SLICE 12 -#define SPEC_FAIL_SUBSCR_STRING_INT 13 #define SPEC_FAIL_SUBSCR_STRING_SLICE 14 #define SPEC_FAIL_SUBSCR_BUFFER_INT 15 #define SPEC_FAIL_SUBSCR_BUFFER_SLICE 16 @@ -1260,16 +1259,7 @@ _Py_Specialize_LoadGlobal( static int binary_subscr_fail_kind(PyTypeObject *container_type, PyObject *sub) { - if (container_type == &PyUnicode_Type) { - if (PyLong_CheckExact(sub)) { - return SPEC_FAIL_SUBSCR_STRING_INT; - } - if (PySlice_Check(sub)) { - return SPEC_FAIL_SUBSCR_STRING_SLICE; - } - return SPEC_FAIL_OTHER; - } - else if (strcmp(container_type->tp_name, "array.array") == 0) { + if (strcmp(container_type->tp_name, "array.array") == 0) { if (PyLong_CheckExact(sub)) { return SPEC_FAIL_SUBSCR_ARRAY_INT; } @@ -1376,6 +1366,19 @@ _Py_Specialize_BinarySubscr( PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_TUPLE_SLICE : SPEC_FAIL_OTHER); goto fail; } + if (container_type == &PyUnicode_Type) { + if (PyLong_CheckExact(sub)) { + if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + instr->op.code = BINARY_SUBSCR_STR_INT; + goto success; + } + SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); + goto fail; + } + SPECIALIZATION_FAIL(BINARY_SUBSCR, + PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_STRING_SLICE : SPEC_FAIL_OTHER); + goto fail; + } if (container_type == &PyDict_Type) { instr->op.code = BINARY_SUBSCR_DICT; goto success; From webhook-mailer at python.org Tue Aug 8 16:50:58 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 08 Aug 2023 20:50:58 -0000 Subject: [Python-checkins] gh-104683: Add --exclude option to Argument Clinic CLI (#107770) Message-ID: https://github.com/python/cpython/commit/0be3743f54569475c40e00c8dcdded928d429d58 commit: 0be3743f54569475c40e00c8dcdded928d429d58 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-08T20:50:54Z summary: gh-104683: Add --exclude option to Argument Clinic CLI (#107770) Co-authored-by: Serhiy Storchaka files: A Misc/NEWS.d/next/Tools-Demos/2023-08-08-12-21-41.gh-issue-104683.DRsAQE.rst M Doc/howto/clinic.rst M Lib/test/test_clinic.py M Makefile.pre.in M Tools/clinic/clinic.py diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 743c7c9cb3000..2d89ccc203b6d 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -188,6 +188,11 @@ The CLI supports the following options: The directory tree to walk in :option:`--make` mode. +.. option:: --exclude EXCLUDE + + A file to exclude in :option:`--make` mode. + This option can be given multiple times. + .. option:: FILE ... The list of files to process. diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 6c2411f9a57b6..8ed7e214742d5 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2216,6 +2216,35 @@ def create_files(files, srcdir, code): path = os.path.join(ext_path, filename) self.assertNotIn(path, out) + def test_cli_make_exclude(self): + code = dedent(""" + /*[clinic input] + [clinic start generated code]*/ + """) + with os_helper.temp_dir(quiet=False) as tmp_dir: + # add some folders, some C files and a Python file + for fn in "file1.c", "file2.c", "file3.c", "file4.c": + path = os.path.join(tmp_dir, fn) + with open(path, "w", encoding="utf-8") as f: + f.write(code) + + # Run clinic in verbose mode with --make on tmpdir. + # Exclude file2.c and file3.c. + out = self.expect_success( + "-v", "--make", "--srcdir", tmp_dir, + "--exclude", os.path.join(tmp_dir, "file2.c"), + # The added ./ should be normalised away. + "--exclude", os.path.join(tmp_dir, "./file3.c"), + # Relative paths should also work. + "--exclude", "file4.c" + ) + + # expect verbose mode to only mention the C files in tmp_dir + self.assertIn("file1.c", out) + self.assertNotIn("file2.c", out) + self.assertNotIn("file3.c", out) + self.assertNotIn("file4.c", out) + def test_cli_verbose(self): with os_helper.temp_dir() as tmp_dir: fn = os.path.join(tmp_dir, "test.c") diff --git a/Makefile.pre.in b/Makefile.pre.in index 12409774746a3..d8fdb34747011 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -775,9 +775,13 @@ coverage-report: regen-token regen-frozen # Run "Argument Clinic" over all source files .PHONY: clinic clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --srcdir $(srcdir) + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --exclude Lib/test/clinic.test.c --srcdir $(srcdir) $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_global_objects.py +.PHONY: clinic-tests +clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py -f $(srcdir)/Lib/test/clinic.test.c + # Build the interpreter $(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-08-12-21-41.gh-issue-104683.DRsAQE.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-08-12-21-41.gh-issue-104683.DRsAQE.rst new file mode 100644 index 0000000000000..ee3a70967b098 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-08-12-21-41.gh-issue-104683.DRsAQE.rst @@ -0,0 +1 @@ +Add ``--exclude`` option to Argument Clinic CLI. diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 0b336d9ac5a60..3b26a706bafd3 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -5834,6 +5834,9 @@ def create_cli() -> argparse.ArgumentParser: help="walk --srcdir to run over all relevant files") cmdline.add_argument("--srcdir", type=str, default=os.curdir, help="the directory tree to walk in --make mode") + cmdline.add_argument("--exclude", type=str, action="append", + help=("a file to exclude in --make mode; " + "can be given multiple times")) cmdline.add_argument("filename", metavar="FILE", type=str, nargs="*", help="the list of files to process") return cmdline @@ -5905,6 +5908,11 @@ def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: parser.error("can't use -o or filenames with --make") if not ns.srcdir: parser.error("--srcdir must not be empty with --make") + if ns.exclude: + excludes = [os.path.join(ns.srcdir, f) for f in ns.exclude] + excludes = [os.path.normpath(f) for f in excludes] + else: + excludes = [] for root, dirs, files in os.walk(ns.srcdir): for rcs_dir in ('.svn', '.git', '.hg', 'build', 'externals'): if rcs_dir in dirs: @@ -5914,6 +5922,9 @@ def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: if not filename.endswith(('.c', '.cpp', '.h')): continue path = os.path.join(root, filename) + path = os.path.normpath(path) + if path in excludes: + continue if ns.verbose: print(path) parse_file(path, verify=not ns.force) From webhook-mailer at python.org Tue Aug 8 17:12:39 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 08 Aug 2023 21:12:39 -0000 Subject: [Python-checkins] gh-104683: Argument Clinic: refactor format_docstring() (#107623) Message-ID: https://github.com/python/cpython/commit/73507382ac184a72b59ebb0c2f85e8b1d2dfa58e commit: 73507382ac184a72b59ebb0c2f85e8b1d2dfa58e branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-08T21:12:35Z summary: gh-104683: Argument Clinic: refactor format_docstring() (#107623) Extract helper methods for formatting the signature and parameter sections, and clean up the remaining function body. Co-authored-by: Alex Waygood Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Tools/clinic/clinic.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 3b26a706bafd3..16f31717080e1 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -5506,23 +5506,11 @@ def state_function_docstring(self, line: str) -> None: self.docstring_append(self.function, line) - def format_docstring(self) -> str: - f = self.function - assert f is not None - - new_or_init = f.kind.new_or_init - if new_or_init and not f.docstring: - # don't render a docstring at all, no signature, nothing. - return f.docstring - + def format_docstring_signature( + self, f: Function, parameters: list[Parameter] + ) -> str: text, add, output = _text_accumulator() - parameters = f.render_parameters - - ## - ## docstring first line - ## - - if new_or_init: + if f.kind.new_or_init: # classes get *just* the name of the class # not __new__, not __init__, and not module.classname assert f.cls @@ -5685,35 +5673,39 @@ def add_parameter(text: str) -> None: if not f.docstring_only: add("\n" + sig_end_marker + "\n") - docstring_first_line = output() + signature_line = output() # now fix up the places where the brackets look wrong - docstring_first_line = docstring_first_line.replace(', ]', ',] ') + return signature_line.replace(', ]', ',] ') - # okay. now we're officially building the "parameters" section. - # create substitution text for {parameters} + @staticmethod + def format_docstring_parameters(params: list[Parameter]) -> str: + """Create substitution text for {parameters}""" + text, add, output = _text_accumulator() spacer_line = False - for p in parameters: - if not p.docstring.strip(): + for param in params: + docstring = param.docstring.strip() + if not docstring: continue if spacer_line: add('\n') else: spacer_line = True add(" ") - add(p.name) + add(param.name) add('\n') - add(textwrap.indent(rstrip_lines(p.docstring.rstrip()), " ")) - parameters_output = output() - if parameters_output: - parameters_output += '\n' - - ## - ## docstring body - ## + stripped = rstrip_lines(docstring) + add(textwrap.indent(stripped, " ")) + if text: + add('\n') + return output() - docstring = f.docstring.rstrip() - lines = [line.rstrip() for line in docstring.split('\n')] + def format_docstring(self) -> str: + assert self.function is not None + f = self.function + if f.kind.new_or_init and not f.docstring: + # don't render a docstring at all, no signature, nothing. + return f.docstring # Enforce the summary line! # The first line of a docstring should be a summary of the function. @@ -5727,6 +5719,7 @@ def add_parameter(text: str) -> None: # Guido said Clinic should enforce this: # http://mail.python.org/pipermail/python-dev/2013-June/127110.html + lines = f.docstring.split('\n') if len(lines) >= 2: if lines[1]: fail(f"Docstring for {f.full_name!r} does not have a summary line!\n" @@ -5738,26 +5731,23 @@ def add_parameter(text: str) -> None: # between it and the {parameters} we're about to add. lines.append('') - parameters_marker_count = len(docstring.split('{parameters}')) - 1 + parameters_marker_count = len(f.docstring.split('{parameters}')) - 1 if parameters_marker_count > 1: fail('You may not specify {parameters} more than once in a docstring!') + # insert signature at front and params after the summary line if not parameters_marker_count: - # insert after summary line lines.insert(2, '{parameters}') + lines.insert(0, '{signature}') - # insert at front of docstring - lines.insert(0, docstring_first_line) - + # finalize docstring + params = f.render_parameters + parameters = self.format_docstring_parameters(params) + signature = self.format_docstring_signature(f, params) docstring = "\n".join(lines) - - add(docstring) - docstring = output() - - docstring = linear_format(docstring, parameters=parameters_output) - docstring = docstring.rstrip() - - return docstring + return linear_format(docstring, + signature=signature, + parameters=parameters).rstrip() def do_post_block_processing_cleanup(self, lineno: int) -> None: """ From webhook-mailer at python.org Wed Aug 9 01:48:01 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 09 Aug 2023 05:48:01 -0000 Subject: [Python-checkins] gh-106052: Fix bug in the matching of possessive quantifiers (gh-106515) Message-ID: https://github.com/python/cpython/commit/7b6e34e5baeb4162815ffa4d943b09a58e3f6580 commit: 7b6e34e5baeb4162815ffa4d943b09a58e3f6580 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-09T08:47:57+03:00 summary: gh-106052: Fix bug in the matching of possessive quantifiers (gh-106515) It did not work in the case of a subpattern containing backtracking. Temporary implement possessive quantifiers as equivalent greedy qualifiers in atomic groups. files: A Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst M Lib/re/_compiler.py M Lib/test/test_re.py diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index d0a4c55caf6e4..f5fd160ba0043 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -100,6 +100,13 @@ def _compile(code, pattern, flags): emit(ANY_ALL) else: emit(ANY) + elif op is POSSESSIVE_REPEAT: + # gh-106052: Possessive quantifiers do not work when the + # subpattern contains backtracking, i.e. "(?:ab?c)*+". + # Implement it as equivalent greedy qualifier in atomic group. + p = [(MAX_REPEAT, av)] + p = [(ATOMIC_GROUP, p)] + _compile(code, p, flags) elif op in REPEATING_CODES: if _simple(av[2]): emit(REPEATING_CODES[op][2]) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index a565cbe1a8d81..bf3698ac78a88 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2342,6 +2342,16 @@ def test_bug_gh91616(self): self.assertTrue(re.fullmatch(r'(?s:(?>.*?\.).*)\Z', "a.txt")) # reproducer self.assertTrue(re.fullmatch(r'(?s:(?=(?P.*?\.))(?P=g0).*)\Z', "a.txt")) + def test_bug_gh106052(self): + self.assertEqual(re.match("(?>(?:ab?c)+)", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?:ab?c)++", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?>(?:ab?c)*)", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?:ab?c)*+", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?>(?:ab?c)?)", "a").span(), (0, 0)) + self.assertEqual(re.match("(?:ab?c)?+", "a").span(), (0, 0)) + self.assertEqual(re.match("(?>(?:ab?c){1,3})", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?:ab?c){1,3}+", "aca").span(), (0, 2)) + @unittest.skipIf(multiprocessing is None, 'test requires multiprocessing') def test_regression_gh94675(self): pattern = re.compile(r'(?<=[({}])(((//[^\n]*)?[\n])([\000-\040])*)*' @@ -2441,6 +2451,7 @@ def test_atomic_group(self): 17: SUCCESS ''') + @unittest.expectedFailure # gh-106052 def test_possesive_repeat_one(self): self.assertEqual(get_debug_out(r'a?+'), '''\ POSSESSIVE_REPEAT 0 1 @@ -2453,6 +2464,7 @@ def test_possesive_repeat_one(self): 12: SUCCESS ''') + @unittest.expectedFailure # gh-106052 def test_possesive_repeat(self): self.assertEqual(get_debug_out(r'(?:ab)?+'), '''\ POSSESSIVE_REPEAT 0 1 diff --git a/Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst b/Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst new file mode 100644 index 0000000000000..f2d4c2f7b18ec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst @@ -0,0 +1,2 @@ +:mod:`re` module: fix the matching of possessive quantifiers in the case of +a subpattern containing backtracking. From webhook-mailer at python.org Wed Aug 9 01:55:39 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 09 Aug 2023 05:55:39 -0000 Subject: [Python-checkins] gh-80282: Argument Clinic: Add clarifying comment about ASCII docstring limitation (#107764) Message-ID: https://github.com/python/cpython/commit/925bbc2166d169962210314772d13270e9ba60a2 commit: 925bbc2166d169962210314772d13270e9ba60a2 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-09T07:55:34+02:00 summary: gh-80282: Argument Clinic: Add clarifying comment about ASCII docstring limitation (#107764) Co-authored-by: Alex Waygood files: M Tools/clinic/clinic.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 16f31717080e1..059c2db27288d 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -5461,6 +5461,11 @@ def state_parameter_docstring_start(self, line: str) -> None: def docstring_append(self, obj: Function | Parameter, line: str) -> None: """Add a rstripped line to the current docstring.""" + # gh-80282: We filter out non-ASCII characters from the docstring, + # since historically, some compilers may balk on non-ASCII input. + # If you're using Argument Clinic in an external project, + # you may not need to support the same array of platforms as CPython, + # so you may be able to remove this restriction. matches = re.finditer(r'[^\x00-\x7F]', line) if offending := ", ".join([repr(m[0]) for m in matches]): warn("Non-ascii characters are not allowed in docstrings:", From webhook-mailer at python.org Wed Aug 9 02:12:06 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 09 Aug 2023 06:12:06 -0000 Subject: [Python-checkins] [3.11] gh-86457: Fix signature for code.replace() (GH-23199) (GH-107746) Message-ID: https://github.com/python/cpython/commit/edaa0db93ee23b1d936631dedde3adf5a1a0fb13 commit: edaa0db93ee23b1d936631dedde3adf5a1a0fb13 branch: 3.11 author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-09T06:12:02Z summary: [3.11] gh-86457: Fix signature for code.replace() (GH-23199) (GH-107746) Also add support of @text_signature in Argument Clinic. (cherry picked from commit 0e6e32fb84b2f7cb668e0b9927637587081e38cd) files: A Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst M Objects/clinic/codeobject.c.h M Objects/codeobject.c M Tools/clinic/clinic.py diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst new file mode 100644 index 0000000000000..4768e6767574d --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst @@ -0,0 +1,2 @@ +Argument Clinic now supports overriding automatically generated signature by +using directive ``@text_signature``. diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index df82524a86afe..9bf9e14b17e51 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -157,12 +157,7 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(code_replace__doc__, -"replace($self, /, *, co_argcount=-1, co_posonlyargcount=-1,\n" -" co_kwonlyargcount=-1, co_nlocals=-1, co_stacksize=-1,\n" -" co_flags=-1, co_firstlineno=-1, co_code=None, co_consts=None,\n" -" co_names=None, co_varnames=None, co_freevars=None,\n" -" co_cellvars=None, co_filename=None, co_name=None,\n" -" co_qualname=None, co_linetable=None, co_exceptiontable=None)\n" +"replace($self, /, **changes)\n" "--\n" "\n" "Return a copy of the code object with new values for the specified fields."); @@ -174,13 +169,12 @@ static PyObject * code_replace_impl(PyCodeObject *self, int co_argcount, int co_posonlyargcount, int co_kwonlyargcount, int co_nlocals, int co_stacksize, int co_flags, - int co_firstlineno, PyBytesObject *co_code, - PyObject *co_consts, PyObject *co_names, - PyObject *co_varnames, PyObject *co_freevars, - PyObject *co_cellvars, PyObject *co_filename, - PyObject *co_name, PyObject *co_qualname, - PyBytesObject *co_linetable, - PyBytesObject *co_exceptiontable); + int co_firstlineno, PyObject *co_code, PyObject *co_consts, + PyObject *co_names, PyObject *co_varnames, + PyObject *co_freevars, PyObject *co_cellvars, + PyObject *co_filename, PyObject *co_name, + PyObject *co_qualname, PyObject *co_linetable, + PyObject *co_exceptiontable); static PyObject * code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -197,7 +191,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje int co_stacksize = self->co_stacksize; int co_flags = self->co_flags; int co_firstlineno = self->co_firstlineno; - PyBytesObject *co_code = NULL; + PyObject *co_code = NULL; PyObject *co_consts = self->co_consts; PyObject *co_names = self->co_names; PyObject *co_varnames = NULL; @@ -206,8 +200,8 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *co_filename = self->co_filename; PyObject *co_name = self->co_name; PyObject *co_qualname = self->co_qualname; - PyBytesObject *co_linetable = (PyBytesObject *)self->co_linetable; - PyBytesObject *co_exceptiontable = (PyBytesObject *)self->co_exceptiontable; + PyObject *co_linetable = self->co_linetable; + PyObject *co_exceptiontable = self->co_exceptiontable; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); if (!args) { @@ -284,7 +278,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("replace", "argument 'co_code'", "bytes", args[7]); goto exit; } - co_code = (PyBytesObject *)args[7]; + co_code = args[7]; if (!--noptargs) { goto skip_optional_kwonly; } @@ -383,7 +377,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("replace", "argument 'co_linetable'", "bytes", args[16]); goto exit; } - co_linetable = (PyBytesObject *)args[16]; + co_linetable = args[16]; if (!--noptargs) { goto skip_optional_kwonly; } @@ -392,7 +386,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("replace", "argument 'co_exceptiontable'", "bytes", args[17]); goto exit; } - co_exceptiontable = (PyBytesObject *)args[17]; + co_exceptiontable = args[17]; skip_optional_kwonly: return_value = code_replace_impl(self, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, co_stacksize, co_flags, co_firstlineno, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_qualname, co_linetable, co_exceptiontable); @@ -436,4 +430,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=9c521b6c79f90ff7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d1bbf51b746ca2d0 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 77ea4abf35dca..c4a0d9ad1cf98 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1875,27 +1875,28 @@ code_linesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args)) } /*[clinic input] + at text_signature "($self, /, **changes)" code.replace * - co_argcount: int(c_default="self->co_argcount") = -1 - co_posonlyargcount: int(c_default="self->co_posonlyargcount") = -1 - co_kwonlyargcount: int(c_default="self->co_kwonlyargcount") = -1 - co_nlocals: int(c_default="self->co_nlocals") = -1 - co_stacksize: int(c_default="self->co_stacksize") = -1 - co_flags: int(c_default="self->co_flags") = -1 - co_firstlineno: int(c_default="self->co_firstlineno") = -1 - co_code: PyBytesObject(c_default="NULL") = None - co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = None - co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = None - co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = None - co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None - co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None - co_filename: unicode(c_default="self->co_filename") = None - co_name: unicode(c_default="self->co_name") = None - co_qualname: unicode(c_default="self->co_qualname") = None - co_linetable: PyBytesObject(c_default="(PyBytesObject *)self->co_linetable") = None - co_exceptiontable: PyBytesObject(c_default="(PyBytesObject *)self->co_exceptiontable") = None + co_argcount: int(c_default="self->co_argcount") = unchanged + co_posonlyargcount: int(c_default="self->co_posonlyargcount") = unchanged + co_kwonlyargcount: int(c_default="self->co_kwonlyargcount") = unchanged + co_nlocals: int(c_default="self->co_nlocals") = unchanged + co_stacksize: int(c_default="self->co_stacksize") = unchanged + co_flags: int(c_default="self->co_flags") = unchanged + co_firstlineno: int(c_default="self->co_firstlineno") = unchanged + co_code: object(subclass_of="&PyBytes_Type", c_default="NULL") = unchanged + co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = unchanged + co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = unchanged + co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged + co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged + co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged + co_filename: unicode(c_default="self->co_filename") = unchanged + co_name: unicode(c_default="self->co_name") = unchanged + co_qualname: unicode(c_default="self->co_qualname") = unchanged + co_linetable: object(subclass_of="&PyBytes_Type", c_default="self->co_linetable") = unchanged + co_exceptiontable: object(subclass_of="&PyBytes_Type", c_default="self->co_exceptiontable") = unchanged Return a copy of the code object with new values for the specified fields. [clinic start generated code]*/ @@ -1904,14 +1905,13 @@ static PyObject * code_replace_impl(PyCodeObject *self, int co_argcount, int co_posonlyargcount, int co_kwonlyargcount, int co_nlocals, int co_stacksize, int co_flags, - int co_firstlineno, PyBytesObject *co_code, - PyObject *co_consts, PyObject *co_names, - PyObject *co_varnames, PyObject *co_freevars, - PyObject *co_cellvars, PyObject *co_filename, - PyObject *co_name, PyObject *co_qualname, - PyBytesObject *co_linetable, - PyBytesObject *co_exceptiontable) -/*[clinic end generated code: output=b6cd9988391d5711 input=f6f68e03571f8d7c]*/ + int co_firstlineno, PyObject *co_code, PyObject *co_consts, + PyObject *co_names, PyObject *co_varnames, + PyObject *co_freevars, PyObject *co_cellvars, + PyObject *co_filename, PyObject *co_name, + PyObject *co_qualname, PyObject *co_linetable, + PyObject *co_exceptiontable) +/*[clinic end generated code: output=e75c48a15def18b9 input=18e280e07846c122]*/ { #define CHECK_INT_ARG(ARG) \ if (ARG < 0) { \ @@ -1936,7 +1936,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount, if (code == NULL) { return NULL; } - co_code = (PyBytesObject *)code; + co_code = code; } if (PySys_Audit("code.__new__", "OOOiiiiii", @@ -1975,10 +1975,10 @@ code_replace_impl(PyCodeObject *self, int co_argcount, co = PyCode_NewWithPosOnlyArgs( co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, - co_stacksize, co_flags, (PyObject*)co_code, co_consts, co_names, + co_stacksize, co_flags, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_qualname, co_firstlineno, - (PyObject*)co_linetable, (PyObject*)co_exceptiontable); + co_linetable, co_exceptiontable); error: Py_XDECREF(code); diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 4d30e66b4822a..84636b92e38a3 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4072,6 +4072,7 @@ def reset(self): self.indent = IndentStack() self.kind = CALLABLE self.coexist = False + self.forced_text_signature: str | None = None self.parameter_continuation = '' self.preserve_output = False @@ -4201,6 +4202,11 @@ def at_coexist(self): fail("Called @coexist twice!") self.coexist = True + def at_text_signature(self, text_signature): + if self.forced_text_signature: + fail("Called @text_signature twice!") + self.forced_text_signature = text_signature + def parse(self, block): self.reset() self.block = block @@ -4903,142 +4909,145 @@ def format_docstring(self): add(f.cls.name) else: add(f.name) - add('(') + if self.forced_text_signature: + add(self.forced_text_signature) + else: + add('(') + + # populate "right_bracket_count" field for every parameter + assert parameters, "We should always have a self parameter. " + repr(f) + assert isinstance(parameters[0].converter, self_converter) + # self is always positional-only. + assert parameters[0].is_positional_only() + parameters[0].right_bracket_count = 0 + positional_only = True + for p in parameters[1:]: + if not p.is_positional_only(): + positional_only = False + else: + assert positional_only + if positional_only: + p.right_bracket_count = abs(p.group) + else: + # don't put any right brackets around non-positional-only parameters, ever. + p.right_bracket_count = 0 + + right_bracket_count = 0 + + def fix_right_bracket_count(desired): + nonlocal right_bracket_count + s = '' + while right_bracket_count < desired: + s += '[' + right_bracket_count += 1 + while right_bracket_count > desired: + s += ']' + right_bracket_count -= 1 + return s + + need_slash = False + added_slash = False + need_a_trailing_slash = False + + # we only need a trailing slash: + # * if this is not a "docstring_only" signature + # * and if the last *shown* parameter is + # positional only + if not f.docstring_only: + for p in reversed(parameters): + if not p.converter.show_in_signature: + continue + if p.is_positional_only(): + need_a_trailing_slash = True + break - # populate "right_bracket_count" field for every parameter - assert parameters, "We should always have a self parameter. " + repr(f) - assert isinstance(parameters[0].converter, self_converter) - # self is always positional-only. - assert parameters[0].is_positional_only() - parameters[0].right_bracket_count = 0 - positional_only = True - for p in parameters[1:]: - if not p.is_positional_only(): - positional_only = False - else: - assert positional_only - if positional_only: - p.right_bracket_count = abs(p.group) - else: - # don't put any right brackets around non-positional-only parameters, ever. - p.right_bracket_count = 0 - - right_bracket_count = 0 - - def fix_right_bracket_count(desired): - nonlocal right_bracket_count - s = '' - while right_bracket_count < desired: - s += '[' - right_bracket_count += 1 - while right_bracket_count > desired: - s += ']' - right_bracket_count -= 1 - return s - need_slash = False - added_slash = False - need_a_trailing_slash = False + added_star = False - # we only need a trailing slash: - # * if this is not a "docstring_only" signature - # * and if the last *shown* parameter is - # positional only - if not f.docstring_only: - for p in reversed(parameters): + first_parameter = True + last_p = parameters[-1] + line_length = len(''.join(text)) + indent = " " * line_length + def add_parameter(text): + nonlocal line_length + nonlocal first_parameter + if first_parameter: + s = text + first_parameter = False + else: + s = ' ' + text + if line_length + len(s) >= 72: + add('\n') + add(indent) + line_length = len(indent) + s = text + line_length += len(s) + add(s) + + for p in parameters: if not p.converter.show_in_signature: continue - if p.is_positional_only(): - need_a_trailing_slash = True - break + assert p.name + is_self = isinstance(p.converter, self_converter) + if is_self and f.docstring_only: + # this isn't a real machine-parsable signature, + # so let's not print the "self" parameter + continue - added_star = False - - first_parameter = True - last_p = parameters[-1] - line_length = len(''.join(text)) - indent = " " * line_length - def add_parameter(text): - nonlocal line_length - nonlocal first_parameter - if first_parameter: - s = text - first_parameter = False - else: - s = ' ' + text - if line_length + len(s) >= 72: - add('\n') - add(indent) - line_length = len(indent) - s = text - line_length += len(s) - add(s) - - for p in parameters: - if not p.converter.show_in_signature: - continue - assert p.name - - is_self = isinstance(p.converter, self_converter) - if is_self and f.docstring_only: - # this isn't a real machine-parsable signature, - # so let's not print the "self" parameter - continue - - if p.is_positional_only(): - need_slash = not f.docstring_only - elif need_slash and not (added_slash or p.is_positional_only()): - added_slash = True - add_parameter('/,') - - if p.is_keyword_only() and not added_star: - added_star = True - add_parameter('*,') - - p_add, p_output = text_accumulator() - p_add(fix_right_bracket_count(p.right_bracket_count)) - - if isinstance(p.converter, self_converter): - # annotate first parameter as being a "self". - # - # if inspect.Signature gets this function, - # and it's already bound, the self parameter - # will be stripped off. - # - # if it's not bound, it should be marked - # as positional-only. - # - # note: we don't print "self" for __init__, - # because this isn't actually the signature - # for __init__. (it can't be, __init__ doesn't - # have a docstring.) if this is an __init__ - # (or __new__), then this signature is for - # calling the class to construct a new instance. - p_add('$') + if p.is_positional_only(): + need_slash = not f.docstring_only + elif need_slash and not (added_slash or p.is_positional_only()): + added_slash = True + add_parameter('/,') + + if p.is_keyword_only() and not added_star: + added_star = True + add_parameter('*,') + + p_add, p_output = text_accumulator() + p_add(fix_right_bracket_count(p.right_bracket_count)) + + if isinstance(p.converter, self_converter): + # annotate first parameter as being a "self". + # + # if inspect.Signature gets this function, + # and it's already bound, the self parameter + # will be stripped off. + # + # if it's not bound, it should be marked + # as positional-only. + # + # note: we don't print "self" for __init__, + # because this isn't actually the signature + # for __init__. (it can't be, __init__ doesn't + # have a docstring.) if this is an __init__ + # (or __new__), then this signature is for + # calling the class to construct a new instance. + p_add('$') - if p.is_vararg(): - p_add("*") + if p.is_vararg(): + p_add("*") - name = p.converter.signature_name or p.name - p_add(name) + name = p.converter.signature_name or p.name + p_add(name) - if not p.is_vararg() and p.converter.is_optional(): - p_add('=') - value = p.converter.py_default - if not value: - value = repr(p.converter.default) - p_add(value) + if not p.is_vararg() and p.converter.is_optional(): + p_add('=') + value = p.converter.py_default + if not value: + value = repr(p.converter.default) + p_add(value) - if (p != last_p) or need_a_trailing_slash: - p_add(',') + if (p != last_p) or need_a_trailing_slash: + p_add(',') - add_parameter(p_output()) + add_parameter(p_output()) - add(fix_right_bracket_count(0)) - if need_a_trailing_slash: - add_parameter('/') - add(')') + add(fix_right_bracket_count(0)) + if need_a_trailing_slash: + add_parameter('/') + add(')') # PEP 8 says: # From webhook-mailer at python.org Wed Aug 9 02:15:31 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 09 Aug 2023 06:15:31 -0000 Subject: [Python-checkins] [3.11] gh-106052: Fix bug in the matching of possessive quantifiers (GH-106515) (GH-107795) Message-ID: https://github.com/python/cpython/commit/5b76eaf02e1f7450c2b8525de0f7f0b65022850a commit: 5b76eaf02e1f7450c2b8525de0f7f0b65022850a branch: 3.11 author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-09T06:15:27Z summary: [3.11] gh-106052: Fix bug in the matching of possessive quantifiers (GH-106515) (GH-107795) It did not work in the case of a subpattern containing backtracking. Temporary implement possessive quantifiers as equivalent greedy qualifiers in atomic groups. (cherry picked from commit 7b6e34e5baeb4162815ffa4d943b09a58e3f6580) files: A Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst M Lib/re/_compiler.py M Lib/test/test_re.py diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index d8e0d2fdefdcc..e30740b9c30b0 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -100,6 +100,13 @@ def _compile(code, pattern, flags): emit(ANY_ALL) else: emit(ANY) + elif op is POSSESSIVE_REPEAT: + # gh-106052: Possessive quantifiers do not work when the + # subpattern contains backtracking, i.e. "(?:ab?c)*+". + # Implement it as equivalent greedy qualifier in atomic group. + p = [(MAX_REPEAT, av)] + p = [(ATOMIC_GROUP, p)] + _compile(code, p, flags) elif op in REPEATING_CODES: if flags & SRE_FLAG_TEMPLATE: raise error("internal: unsupported template operator %r" % (op,)) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 38d6db7c4091b..d2736dae20b1b 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2396,6 +2396,16 @@ def test_template_function_and_flag_is_deprecated(self): self.assertTrue(template_re1.match('ahoy')) self.assertFalse(template_re1.match('nope')) + def test_bug_gh106052(self): + self.assertEqual(re.match("(?>(?:ab?c)+)", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?:ab?c)++", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?>(?:ab?c)*)", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?:ab?c)*+", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?>(?:ab?c)?)", "a").span(), (0, 0)) + self.assertEqual(re.match("(?:ab?c)?+", "a").span(), (0, 0)) + self.assertEqual(re.match("(?>(?:ab?c){1,3})", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?:ab?c){1,3}+", "aca").span(), (0, 2)) + @unittest.skipIf(multiprocessing is None, 'test requires multiprocessing') def test_regression_gh94675(self): pattern = re.compile(r'(?<=[({}])(((//[^\n]*)?[\n])([\000-\040])*)*' @@ -2492,6 +2502,7 @@ def test_atomic_group(self): 17: SUCCESS ''') + @unittest.expectedFailure # gh-106052 def test_possesive_repeat_one(self): self.assertEqual(get_debug_out(r'a?+'), '''\ POSSESSIVE_REPEAT 0 1 @@ -2504,6 +2515,7 @@ def test_possesive_repeat_one(self): 12: SUCCESS ''') + @unittest.expectedFailure # gh-106052 def test_possesive_repeat(self): self.assertEqual(get_debug_out(r'(?:ab)?+'), '''\ POSSESSIVE_REPEAT 0 1 diff --git a/Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst b/Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst new file mode 100644 index 0000000000000..f2d4c2f7b18ec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst @@ -0,0 +1,2 @@ +:mod:`re` module: fix the matching of possessive quantifiers in the case of +a subpattern containing backtracking. From webhook-mailer at python.org Wed Aug 9 03:25:29 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 09 Aug 2023 07:25:29 -0000 Subject: [Python-checkins] [3.11] gh-86457: Add docs for Argument Clinic @text_signature directive (#107747) (#107799) Message-ID: https://github.com/python/cpython/commit/b0b26af822e601461988d0765cf7b33d3791ff2e commit: b0b26af822e601461988d0765cf7b33d3791ff2e branch: 3.11 author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-09T09:25:25+02:00 summary: [3.11] gh-86457: Add docs for Argument Clinic @text_signature directive (#107747) (#107799) (cherry picked from commit a9aeb99579f24bbce1dd553d605a5a5e2f37a3a2) Co-authored-by: Alex Waygood files: M Doc/howto/clinic.rst M Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index d75ba8a19cff2..23be8a7a85c09 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1893,3 +1893,36 @@ blocks embedded in Python files look slightly different. They look like this: #[python start generated code]*/ def foo(): pass #/*[python checksum:...]*/ + + +.. _clinic-howto-override-signature: + +How to override the generated signature +--------------------------------------- + +You can use the ``@text_signature`` directive to override the default generated +signature in the docstring. +This can be useful for complex signatures that Argument Clinic cannot handle. +The ``@text_signature`` directive takes one argument: +the custom signature as a string. +The provided signature is copied verbatim to the generated docstring. + +Example from :source:`Objects/codeobject.c`:: + + /*[clinic input] + @text_signature "($self, /, **changes)" + code.replace + * + co_argcount: int(c_default="self->co_argcount") = unchanged + co_posonlyargcount: int(c_default="self->co_posonlyargcount") = unchanged + # etc ... + + Return a copy of the code object with new values for the specified fields. + [clinic start generated output]*/ + +The generated docstring ends up looking like this:: + + replace($self, /, **changes) + -- + + Return a copy of the code object with new values for the specified fields. diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst index 4768e6767574d..7284f5bd54881 100644 --- a/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst @@ -1,2 +1,2 @@ Argument Clinic now supports overriding automatically generated signature by -using directive ``@text_signature``. +using directive ``@text_signature``. See :ref:`clinic-howto-override-signature`. From webhook-mailer at python.org Wed Aug 9 03:43:06 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 09 Aug 2023 07:43:06 -0000 Subject: [Python-checkins] Docs: clean up Argument Clinic howto's (#107797) Message-ID: https://github.com/python/cpython/commit/34cafd35e35dcb44b4347a6478fdbf88b900240c commit: 34cafd35e35dcb44b4347a6478fdbf88b900240c branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-09T09:43:02+02:00 summary: Docs: clean up Argument Clinic howto's (#107797) - fix formatting in @text_signature howto and NEWS entry - fix formatting and example text in deprecate-positionals howto files: M Doc/howto/clinic.rst M Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 2d89ccc203b6d..463a938fafa8d 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1930,13 +1930,14 @@ Example from :source:`Objects/codeobject.c`:: Return a copy of the code object with new values for the specified fields. [clinic start generated output]*/ -The generated docstring ends up looking like this:: +The generated docstring ends up looking like this: - Doc_STRVAR(code_replace__doc__, - "replace($self, /, **changes)\n" - "--\n" - "\n" - "Return a copy of the code object with new values for the specified fields."); +.. code-block:: none + + replace($self, /, **changes) + -- + + Return a copy of the code object with new values for the specified fields. .. _clinic-howto-deprecate-positional: @@ -1968,7 +1969,7 @@ whenever the *b* parameter is passed positionally, and we'll be allowed to make it keyword-only in Python 3.14 at the earliest. We can use Argument Clinic to emit the desired deprecation warnings -using the ``* [from ...]``` syntax, +using the ``* [from ...]`` syntax, by adding the line ``* [from 3.14]`` right above the *b* parameter:: /*[clinic input] @@ -2000,12 +2001,12 @@ Luckily for us, compiler warnings are now generated: .. code-block:: none In file included from Modules/foomodule.c:139: - Modules/clinic/foomodule.c.h:83:8: warning: Update 'b' in 'myfunc' in 'foomodule.c' to be keyword-only. [-W#warnings] - # warning "Update 'b' in 'myfunc' in 'foomodule.c' to be keyword-only." + Modules/clinic/foomodule.c.h:139:8: warning: In 'foomodule.c', update parameter(s) 'a' and 'b' in the clinic input of 'mymod.myfunc' to be keyword-only. [-W#warnings] + # warning "In 'foomodule.c', update parameter(s) 'a' and 'b' in the clinic input of 'mymod.myfunc' to be keyword-only. [-W#warnings]" ^ We now close the deprecation phase by making *b* keyword-only; -replace the ``* [from ...]``` line above *b* +replace the ``* [from ...]`` line above *b* with the ``*`` from the line above *c*:: /*[clinic input] diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst index df1893e9c808e..7284f5bd54881 100644 --- a/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst @@ -1,2 +1,2 @@ Argument Clinic now supports overriding automatically generated signature by -using directive `@text_signature`. See :ref:`clinic-howto-override-signature`. +using directive ``@text_signature``. See :ref:`clinic-howto-override-signature`. From webhook-mailer at python.org Wed Aug 9 03:44:47 2023 From: webhook-mailer at python.org (rhettinger) Date: Wed, 09 Aug 2023 07:44:47 -0000 Subject: [Python-checkins] Future-proof helper function with zero handling. (GH-107798) Message-ID: https://github.com/python/cpython/commit/2fb484e62518d15fc9a19565c2ab096bd741219a commit: 2fb484e62518d15fc9a19565c2ab096bd741219a branch: main author: Raymond Hettinger committer: rhettinger date: 2023-08-09T08:44:43+01:00 summary: Future-proof helper function with zero handling. (GH-107798) files: M Lib/statistics.py diff --git a/Lib/statistics.py b/Lib/statistics.py index 066669d25ddb1..93c44f1f13fab 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -1009,6 +1009,8 @@ def _sqrtprod(x: float, y: float) -> float: # Square root differential correction: # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 h = sqrt(x * y) + if not h: + return 0.0 x = sumprod((x, h), (y, -h)) return h + x / (2.0 * h) From webhook-mailer at python.org Wed Aug 9 03:54:36 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 09 Aug 2023 07:54:36 -0000 Subject: [Python-checkins] [3.11] Docs: clean up Argument Clinic howto's (#107797) (#107800) Message-ID: https://github.com/python/cpython/commit/ec0c0c8692e32f1c694a53ab00b17893fc03666b commit: ec0c0c8692e32f1c694a53ab00b17893fc03666b branch: 3.11 author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-09T07:54:32Z summary: [3.11] Docs: clean up Argument Clinic howto's (#107797) (#107800) (cherry picked from commit 34cafd35e35dcb44b4347a6478fdbf88b900240c) - fix formatting in @text_signature howto and NEWS entry files: M Doc/howto/clinic.rst diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 23be8a7a85c09..5f4d3977bc160 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1920,7 +1920,9 @@ Example from :source:`Objects/codeobject.c`:: Return a copy of the code object with new values for the specified fields. [clinic start generated output]*/ -The generated docstring ends up looking like this:: +The generated docstring ends up looking like this: + +.. code-block:: none replace($self, /, **changes) -- From webhook-mailer at python.org Wed Aug 9 04:30:55 2023 From: webhook-mailer at python.org (markshannon) Date: Wed, 09 Aug 2023 08:30:55 -0000 Subject: [Python-checkins] GH-107724: Fix the signature of `PY_THROW` callback functions. (GH-107725) Message-ID: https://github.com/python/cpython/commit/52fbcf61b5a70993c2d32332ff0ad9f369d968d3 commit: 52fbcf61b5a70993c2d32332ff0ad9f369d968d3 branch: main author: Mark Shannon committer: markshannon date: 2023-08-09T09:30:50+01:00 summary: GH-107724: Fix the signature of `PY_THROW` callback functions. (GH-107725) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-04-21-25-26.gh-issue-107724.EbBXMr.rst M Include/internal/pycore_instruments.h M Lib/test/test_monitoring.py M Python/ceval.c M Python/instrumentation.c M Python/legacy_tracing.c diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index ccccd54a2f70a..56de9f8717148 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -90,10 +90,6 @@ extern int _Py_call_instrumentation_2args(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); -extern void -_Py_call_instrumentation_exc0(PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); - extern void _Py_call_instrumentation_exc2(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index b36382c8d0982..845185be737eb 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -743,6 +743,13 @@ class ExceptionHandledRecorder(ExceptionRecorder): def __call__(self, code, offset, exc): self.events.append(("handled", type(exc))) +class ThrowRecorder(ExceptionRecorder): + + event_type = E.PY_THROW + + def __call__(self, code, offset, exc): + self.events.append(("throw", type(exc))) + class ExceptionMonitoringTest(CheckEvents): @@ -888,6 +895,31 @@ async def async_loop(): func, recorders = self.exception_recorders) + def test_throw(self): + + def gen(): + yield 1 + yield 2 + + def func(): + try: + g = gen() + next(g) + g.throw(IndexError) + except IndexError: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + events = self.get_events( + func, + TEST_TOOL, + self.exception_recorders + (ThrowRecorder,) + ) + self.assertEqual(events[0], ("throw", IndexError)) + class LineRecorder: event_type = E.LINE diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-04-21-25-26.gh-issue-107724.EbBXMr.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-04-21-25-26.gh-issue-107724.EbBXMr.rst new file mode 100644 index 0000000000000..6e853cf72a334 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-04-21-25-26.gh-issue-107724.EbBXMr.rst @@ -0,0 +1,3 @@ +In pre-release versions of 3.12, up to rc1, the sys.monitoring callback +function for the ``PY_THROW`` event was missing the third, exception +argument. That is now fixed. diff --git a/Python/ceval.c b/Python/ceval.c index 1dc4fd80224bb..b966399a342d0 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2039,7 +2039,7 @@ monitor_throw(PyThreadState *tstate, if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) { return; } - _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_THROW, frame, instr); + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); } void diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 123c20dfe1a99..65ea7902a80a6 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1081,16 +1081,6 @@ call_instrumentation_vector_protected( assert(_PyErr_Occurred(tstate)); } -void -_Py_call_instrumentation_exc0( - PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) -{ - assert(_PyErr_Occurred(tstate)); - PyObject *args[3] = { NULL, NULL, NULL }; - call_instrumentation_vector_protected(tstate, event, frame, instr, 2, args); -} - void _Py_call_instrumentation_exc2( PyThreadState *tstate, int event, diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index 48db51731cfdb..7774d10b10172 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -163,7 +163,7 @@ sys_trace_func2( } static PyObject * -sys_trace_unwind( +sys_trace_func3( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { @@ -445,7 +445,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - (vectorcallfunc)sys_trace_func2, PyTrace_CALL, + (vectorcallfunc)sys_trace_func3, PyTrace_CALL, PY_MONITORING_EVENT_PY_THROW, -1)) { return -1; } @@ -470,7 +470,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - (vectorcallfunc)sys_trace_unwind, PyTrace_RETURN, + (vectorcallfunc)sys_trace_func3, PyTrace_RETURN, PY_MONITORING_EVENT_PY_UNWIND, -1)) { return -1; } From webhook-mailer at python.org Wed Aug 9 06:24:10 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Wed, 09 Aug 2023 10:24:10 -0000 Subject: [Python-checkins] gh-104683: Remove unused variables from `Tools/clinic` and tests for `Tools/clinic` (#107771) Message-ID: https://github.com/python/cpython/commit/65ce3652fa47a34acf715ee07bf0a2fae2e0da51 commit: 65ce3652fa47a34acf715ee07bf0a2fae2e0da51 branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-09T11:24:05+01:00 summary: gh-104683: Remove unused variables from `Tools/clinic` and tests for `Tools/clinic` (#107771) files: M Lib/test/test_clinic.py M Tools/clinic/cpp.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 8ed7e214742d5..026cd18d646ad 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -567,7 +567,6 @@ def test_directive_preserve_input(self): self.expect_failure(block, err, lineno=6) def test_directive_preserve_output(self): - err = "'preserve' only works for blocks that don't produce any output!" block = dedent(""" /*[clinic input] output everything buffer @@ -965,7 +964,6 @@ def test_param_no_docstring(self): follow_symlinks: bool = True something_else: str = '' """) - p = function.parameters['follow_symlinks'] self.assertEqual(3, len(function.parameters)) conv = function.parameters['something_else'].converter self.assertIsInstance(conv, clinic.str_converter) @@ -1929,10 +1927,6 @@ def test_scaffolding(self): self.assertEqual(repr(clinic.NULL), '') # test that fail fails - expected = ( - 'Error in file "clown.txt" on line 69:\n' - 'The igloos are melting!\n' - ) with support.captured_stdout() as stdout: errmsg = 'The igloos are melting' with self.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: @@ -1940,6 +1934,7 @@ def test_scaffolding(self): exc = cm.exception self.assertEqual(exc.filename, 'clown.txt') self.assertEqual(exc.lineno, 69) + self.assertEqual(stdout.getvalue(), "") def test_non_ascii_character_in_docstring(self): block = """ @@ -2408,7 +2403,7 @@ def test_file_dest(self): "not overwriting!") self.assertIn(expected_err, err) # Run clinic again, this time with the -f option. - out = self.expect_success("-f", in_fn) + _ = self.expect_success("-f", in_fn) # Read back the generated output. with open(in_fn, encoding="utf-8") as f: data = f.read() diff --git a/Tools/clinic/cpp.py b/Tools/clinic/cpp.py index 3d9c61ac67896..16eee6fc39949 100644 --- a/Tools/clinic/cpp.py +++ b/Tools/clinic/cpp.py @@ -186,7 +186,7 @@ def _main(filenames: list[str] | None = None) -> None: cpp = Monitor(filename, verbose=True) print() print(filename) - for line_number, line in enumerate(f.read().split('\n'), 1): + for line in f: cpp.writeline(line) From webhook-mailer at python.org Wed Aug 9 06:26:55 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 09 Aug 2023 10:26:55 -0000 Subject: [Python-checkins] [3.12] gh-106052: Fix bug in the matching of possessive quantifiers (GH-106515) (#107796) Message-ID: https://github.com/python/cpython/commit/3bb43b7b1b75154bc4e94b1fa81afe296a8150d0 commit: 3bb43b7b1b75154bc4e94b1fa81afe296a8150d0 branch: 3.12 author: Serhiy Storchaka committer: Yhg1s date: 2023-08-09T12:26:51+02:00 summary: [3.12] gh-106052: Fix bug in the matching of possessive quantifiers (GH-106515) (#107796) [3.12] gh-106052: Fix bug in the matching of possessive quantifiers (gh-106515) It did not work in the case of a subpattern containing backtracking. Temporary implement possessive quantifiers as equivalent greedy qualifiers in atomic groups.. (cherry picked from commit 7b6e34e5baeb4162815ffa4d943b09a58e3f6580) files: A Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst M Lib/re/_compiler.py M Lib/test/test_re.py diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index d8e0d2fdefdcc..e30740b9c30b0 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -100,6 +100,13 @@ def _compile(code, pattern, flags): emit(ANY_ALL) else: emit(ANY) + elif op is POSSESSIVE_REPEAT: + # gh-106052: Possessive quantifiers do not work when the + # subpattern contains backtracking, i.e. "(?:ab?c)*+". + # Implement it as equivalent greedy qualifier in atomic group. + p = [(MAX_REPEAT, av)] + p = [(ATOMIC_GROUP, p)] + _compile(code, p, flags) elif op in REPEATING_CODES: if flags & SRE_FLAG_TEMPLATE: raise error("internal: unsupported template operator %r" % (op,)) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 50b9ad701f0ce..85541f4451d03 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2365,6 +2365,16 @@ def test_template_function_and_flag_is_deprecated(self): self.assertTrue(template_re1.match('ahoy')) self.assertFalse(template_re1.match('nope')) + def test_bug_gh106052(self): + self.assertEqual(re.match("(?>(?:ab?c)+)", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?:ab?c)++", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?>(?:ab?c)*)", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?:ab?c)*+", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?>(?:ab?c)?)", "a").span(), (0, 0)) + self.assertEqual(re.match("(?:ab?c)?+", "a").span(), (0, 0)) + self.assertEqual(re.match("(?>(?:ab?c){1,3})", "aca").span(), (0, 2)) + self.assertEqual(re.match("(?:ab?c){1,3}+", "aca").span(), (0, 2)) + @unittest.skipIf(multiprocessing is None, 'test requires multiprocessing') def test_regression_gh94675(self): pattern = re.compile(r'(?<=[({}])(((//[^\n]*)?[\n])([\000-\040])*)*' @@ -2461,6 +2471,7 @@ def test_atomic_group(self): 17: SUCCESS ''') + @unittest.expectedFailure # gh-106052 def test_possesive_repeat_one(self): self.assertEqual(get_debug_out(r'a?+'), '''\ POSSESSIVE_REPEAT 0 1 @@ -2473,6 +2484,7 @@ def test_possesive_repeat_one(self): 12: SUCCESS ''') + @unittest.expectedFailure # gh-106052 def test_possesive_repeat(self): self.assertEqual(get_debug_out(r'(?:ab)?+'), '''\ POSSESSIVE_REPEAT 0 1 diff --git a/Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst b/Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst new file mode 100644 index 0000000000000..f2d4c2f7b18ec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-07-14-52-31.gh-issue-106052.ak8nbs.rst @@ -0,0 +1,2 @@ +:mod:`re` module: fix the matching of possessive quantifiers in the case of +a subpattern containing backtracking. From webhook-mailer at python.org Wed Aug 9 08:44:31 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 09 Aug 2023 12:44:31 -0000 Subject: [Python-checkins] gh-104683: Argument Clinic: Params now render their own docstrings (#107790) Message-ID: https://github.com/python/cpython/commit/1b3f5f24af95c0476ba339ed786a8a4a2578362a commit: 1b3f5f24af95c0476ba339ed786a8a4a2578362a branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-09T12:44:26Z summary: gh-104683: Argument Clinic: Params now render their own docstrings (#107790) Co-authored-by: Alex Waygood files: M Lib/test/test_clinic.py M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 026cd18d646ad..3921523af067f 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -833,8 +833,8 @@ def expect_failure(self, block, err, *, filename=None, lineno=None): def checkDocstring(self, fn, expected): self.assertTrue(hasattr(fn, "docstring")) - self.assertEqual(fn.docstring.strip(), - dedent(expected).strip()) + self.assertEqual(dedent(expected).strip(), + fn.docstring.strip()) def test_trivial(self): parser = DSLParser(_make_clinic()) @@ -997,8 +997,12 @@ def test_function_docstring(self): path: str Path to be examined + Ensure that multiple lines are indented correctly. Perform a stat system call on the given path. + + Ensure that multiple lines are indented correctly. + Ensure that multiple lines are indented correctly. """) self.checkDocstring(function, """ stat($module, /, path) @@ -1008,6 +1012,10 @@ def test_function_docstring(self): path Path to be examined + Ensure that multiple lines are indented correctly. + + Ensure that multiple lines are indented correctly. + Ensure that multiple lines are indented correctly. """) def test_docstring_trailing_whitespace(self): diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 059c2db27288d..3490d4871b891 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2775,6 +2775,13 @@ def get_displayname(self, i: int) -> str: else: return f'"argument {i}"' + def render_docstring(self) -> str: + add, out = text_accumulator() + add(f" {self.name}\n") + for line in self.docstring.split("\n"): + add(f" {line}\n") + return out().rstrip() + CConverterClassT = TypeVar("CConverterClassT", bound=type["CConverter"]) @@ -5686,23 +5693,11 @@ def add_parameter(text: str) -> None: @staticmethod def format_docstring_parameters(params: list[Parameter]) -> str: """Create substitution text for {parameters}""" - text, add, output = _text_accumulator() - spacer_line = False - for param in params: - docstring = param.docstring.strip() - if not docstring: - continue - if spacer_line: + add, output = text_accumulator() + for p in params: + if p.docstring: + add(p.render_docstring()) add('\n') - else: - spacer_line = True - add(" ") - add(param.name) - add('\n') - stripped = rstrip_lines(docstring) - add(textwrap.indent(stripped, " ")) - if text: - add('\n') return output() def format_docstring(self) -> str: From webhook-mailer at python.org Wed Aug 9 09:28:23 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 09 Aug 2023 13:28:23 -0000 Subject: [Python-checkins] gh-95065: Produce nicer deprecation messages in Argument Clinic (#107808) Message-ID: https://github.com/python/cpython/commit/0a7f48b9a8d54809f1e9272337aefe2444158e64 commit: 0a7f48b9a8d54809f1e9272337aefe2444158e64 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-09T13:28:18Z summary: gh-95065: Produce nicer deprecation messages in Argument Clinic (#107808) files: M Lib/test/clinic.test.c M Tools/clinic/clinic.py diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 9fcee0dad7118..c7063e1139b0a 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -4,9 +4,11 @@ output preset block /*[clinic end generated code: output=da39a3ee5e6b4b0d input=3c81ac2402d06a8b]*/ /*[clinic input] +module m +class m.T "TestObj *" "TestType" class Test "TestObj *" "TestType" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=fc7e50384d12b83f]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=f761b4d55cb179cf]*/ /*[clinic input] test_object_converter @@ -6437,3 +6439,192 @@ test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, PyObject *d, PyObject *e) /*[clinic end generated code: output=383d56b03f7c2dcb input=154fd450448d8935]*/ + + +/*[clinic input] + at classmethod +Test.__new__ + * [from 3.14] + a: object +The deprecation message should use the class name instead of __new__. +[clinic start generated code]*/ + +PyDoc_STRVAR(Test__doc__, +"Test(a)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __new__.\n" +"\n" +"Note: Passing positional arguments to Test() is deprecated. Parameter\n" +"\'a\' will become a keyword-only parameter in Python 3.14.\n" +""); + +static PyObject * +Test_impl(PyTypeObject *type, PyObject *a); + +static PyObject * +Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "Test", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'Test.__new__' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'Test.__new__' to be keyword-only.") + # else + # warning \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'Test.__new__' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to Test() is deprecated. Parameter " + "'a' will become a keyword-only parameter in Python 3.14.", 1)) + { + goto exit; + } + } + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + return_value = Test_impl(type, a); + +exit: + return return_value; +} + +static PyObject * +Test_impl(PyTypeObject *type, PyObject *a) +/*[clinic end generated code: output=d15a69ea37ec6502 input=f133dc077aef49ec]*/ + + +/*[clinic input] +m.T.__init__ + * [from 3.14] + a: object +The deprecation message should use the class name instead of __init__. +[clinic start generated code]*/ + +PyDoc_STRVAR(m_T___init____doc__, +"T(a)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __init__.\n" +"\n" +"Note: Passing positional arguments to m.T() is deprecated. Parameter\n" +"\'a\' will become a keyword-only parameter in Python 3.14.\n" +""); + +static int +m_T___init___impl(TestObj *self, PyObject *a); + +static int +m_T___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "T", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'm.T.__init__' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'm.T.__init__' to be keyword-only.") + # else + # warning \ + "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ + " 'm.T.__init__' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to m.T() is deprecated. Parameter " + "'a' will become a keyword-only parameter in Python 3.14.", 1)) + { + goto exit; + } + } + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + return_value = m_T___init___impl((TestObj *)self, a); + +exit: + return return_value; +} + +static int +m_T___init___impl(TestObj *self, PyObject *a) +/*[clinic end generated code: output=ef43c425816a549f input=f71b51dbe19fa657]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 3490d4871b891..70b066cce82fa 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -928,7 +928,7 @@ def deprecate_positional_use( if first_pos: preamble = f"Passing {first_pos+1} positional arguments to " depr_message = preamble + ( - f"{func.full_name}() is deprecated. Parameter {pstr} will " + f"{func.fulldisplayname}() is deprecated. Parameter {pstr} will " f"become a keyword-only parameter in Python {major}.{minor}." ) else: @@ -939,7 +939,7 @@ def deprecate_positional_use( f"argument{'s' if first_pos != 1 else ''} to " ) depr_message = preamble + ( - f"{func.full_name}() is deprecated. Parameters {pstr} will " + f"{func.fulldisplayname}() is deprecated. Parameters {pstr} will " f"become keyword-only parameters in Python {major}.{minor}." ) @@ -1673,14 +1673,7 @@ def render_function( full_name = f.full_name template_dict = {'full_name': full_name} - - if new_or_init: - assert isinstance(f.cls, Class) - name = f.cls.name - else: - name = f.name - - template_dict['name'] = name + template_dict['name'] = f.displayname if f.c_basename: c_basename = f.c_basename @@ -2678,6 +2671,21 @@ def __post_init__(self) -> None: self.self_converter: self_converter | None = None self.__render_parameters__: list[Parameter] | None = None + @functools.cached_property + def displayname(self) -> str: + """Pretty-printable name.""" + if self.kind.new_or_init: + assert isinstance(self.cls, Class) + return self.cls.name + else: + return self.name + + @functools.cached_property + def fulldisplayname(self) -> str: + if isinstance(self.module, Module): + return f"{self.module.name}.{self.displayname}" + return self.displayname + @property def render_parameters(self) -> list[Parameter]: if not self.__render_parameters__: @@ -5522,13 +5530,7 @@ def format_docstring_signature( self, f: Function, parameters: list[Parameter] ) -> str: text, add, output = _text_accumulator() - if f.kind.new_or_init: - # classes get *just* the name of the class - # not __new__, not __init__, and not module.classname - assert f.cls - add(f.cls.name) - else: - add(f.name) + add(f.displayname) if self.forced_text_signature: add(self.forced_text_signature) else: From webhook-mailer at python.org Wed Aug 9 14:19:44 2023 From: webhook-mailer at python.org (brandtbucher) Date: Wed, 09 Aug 2023 18:19:44 -0000 Subject: [Python-checkins] GH-105848: Simplify the arrangement of CALL's stack (GH-107788) Message-ID: https://github.com/python/cpython/commit/a9caf9cf9041d6d0b69f8be0fd778dd1f9b50e74 commit: a9caf9cf9041d6d0b69f8be0fd778dd1f9b50e74 branch: main author: Brandt Bucher committer: brandtbucher date: 2023-08-09T18:19:39Z summary: GH-105848: Simplify the arrangement of CALL's stack (GH-107788) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-05-09-06-56.gh-issue-105848.Drc-1-.rst M Doc/library/dis.rst M Include/internal/pycore_opcode_metadata.h M Lib/dis.py M Lib/importlib/_bootstrap_external.py M Lib/test/test_compiler_assemble.py M Lib/test/test_compiler_codegen.py M Lib/test/test_dis.py M Objects/exception_handling_notes.txt M Programs/test_frozenmain.h M Python/bytecodes.c M Python/compile.c M Python/executor_cases.c.h M Python/flowgraph.c M Python/generated_cases.c.h M Python/specialize.c diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 2217c3a803545..0b44d160de58a 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -56,7 +56,7 @@ the following command can be used to display the disassembly of >>> dis.dis(myfunc) 2 0 RESUME 0 - 3 2 LOAD_GLOBAL 1 (NULL + len) + 3 2 LOAD_GLOBAL 1 (len + NULL) 12 LOAD_FAST 0 (alist) 14 CALL 1 22 RETURN_VALUE diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 02303c42c75c8..9f4437c09e92c 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -273,11 +273,11 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { case BUILD_CONST_KEY_MAP: return oparg + 1; case DICT_UPDATE: - return 1; + return (oparg - 1) + 2; case DICT_MERGE: - return 1; + return (oparg - 1) + 5; case MAP_ADD: - return 2; + return (oparg - 1) + 3; case INSTRUMENTED_LOAD_SUPER_ATTR: return 3; case LOAD_SUPER_ATTR: @@ -719,11 +719,11 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case BUILD_CONST_KEY_MAP: return 1; case DICT_UPDATE: - return 0; + return (oparg - 1) + 1; case DICT_MERGE: - return 0; + return (oparg - 1) + 4; case MAP_ADD: - return 0; + return (oparg - 1) + 1; case INSTRUMENTED_LOAD_SUPER_ATTR: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_SUPER_ATTR: @@ -735,7 +735,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case LOAD_ZERO_SUPER_ATTR: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_SUPER_ATTR_ATTR: - return ((oparg & 1) ? 1 : 0) + 1; + return 1; case LOAD_SUPER_ATTR_METHOD: return 2; case LOAD_ATTR: @@ -753,9 +753,9 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { case LOAD_ATTR_CLASS: return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_PROPERTY: - return ((oparg & 1) ? 1 : 0) + 1; + return 1; case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: - return ((oparg & 1) ? 1 : 0) + 1; + return 1; case STORE_ATTR_INSTANCE_VALUE: return 0; case STORE_ATTR_WITH_HINT: diff --git a/Lib/dis.py b/Lib/dis.py index b167773dfe7b0..bf1a1e2ff7ac1 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -552,15 +552,15 @@ def _get_instructions_bytes(code, varname_from_oparg=None, if deop == LOAD_GLOBAL: argval, argrepr = _get_name_info(arg//2, get_name) if (arg & 1) and argrepr: - argrepr = "NULL + " + argrepr + argrepr = f"{argrepr} + NULL" elif deop == LOAD_ATTR: argval, argrepr = _get_name_info(arg//2, get_name) if (arg & 1) and argrepr: - argrepr = "NULL|self + " + argrepr + argrepr = f"{argrepr} + NULL|self" elif deop == LOAD_SUPER_ATTR: argval, argrepr = _get_name_info(arg//4, get_name) if (arg & 1) and argrepr: - argrepr = "NULL|self + " + argrepr + argrepr = f"{argrepr} + NULL|self" else: argval, argrepr = _get_name_info(arg, get_name) elif deop in hasjabs: diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 16a82bef2ba71..5f0d659b1ed53 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -453,6 +453,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.13a1 3555 (generate specialized opcodes metadata from bytecodes.c) # Python 3.13a1 3556 (Convert LOAD_CLOSURE to a pseudo-op) # Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit) +# Python 3.13a1 3558 (Reorder the stack items for CALL) # Python 3.14 will start with 3600 @@ -469,7 +470,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3557).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3558).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index 6df72cbc54666..5696433e529d0 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -94,12 +94,12 @@ def inner(): instructions = [ ('RESUME', 0,), - ('PUSH_NULL', 0, 1), ('LOAD_CLOSURE', 0, 1), ('BUILD_TUPLE', 1, 1), ('LOAD_CONST', 1, 1), ('MAKE_FUNCTION', 0, 2), ('SET_FUNCTION_ATTRIBUTE', 8, 2), + ('PUSH_NULL', 0, 1), ('CALL', 0, 2), # (lambda: x)() ('LOAD_CONST', 2, 2), # 2 ('BINARY_OP', 6, 2), # % diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index d99bb8c6cd472..6d7731ddba02c 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -41,8 +41,8 @@ def test_for_loop(self): loop_lbl := self.Label(), ('FOR_ITER', exit_lbl := self.Label(), 1), ('STORE_NAME', 1, 1), - ('PUSH_NULL', None, 2), ('LOAD_NAME', 2, 2), + ('PUSH_NULL', None, 2), ('LOAD_NAME', 1, 2), ('CALL', 1, 2), ('POP_TOP', None), diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 642b26163ab24..f49c60a01a54e 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -102,7 +102,7 @@ def _f(a): dis_f = """\ %3d RESUME 0 -%3d LOAD_GLOBAL 1 (NULL + print) +%3d LOAD_GLOBAL 1 (print + NULL) LOAD_FAST 0 (a) CALL 1 POP_TOP @@ -131,7 +131,7 @@ def bug708901(): dis_bug708901 = """\ %3d RESUME 0 -%3d LOAD_GLOBAL 1 (NULL + range) +%3d LOAD_GLOBAL 1 (range + NULL) LOAD_CONST 1 (1) %3d LOAD_CONST 2 (10) @@ -236,7 +236,7 @@ def wrap_func_w_kwargs(): dis_kw_names = """\ %3d RESUME 0 -%3d LOAD_GLOBAL 1 (NULL + func_w_kwargs) +%3d LOAD_GLOBAL 1 (func_w_kwargs + NULL) LOAD_CONST 1 (1) LOAD_CONST 2 (2) LOAD_CONST 3 (5) @@ -345,8 +345,8 @@ def wrap_func_w_kwargs(): LOAD_CONST 1 ('x') STORE_SUBSCR - 3 PUSH_NULL - LOAD_NAME 3 (fun) + 3 LOAD_NAME 3 (fun) + PUSH_NULL LOAD_CONST 0 (1) CALL 1 LOAD_NAME 2 (__annotations__) @@ -355,8 +355,8 @@ def wrap_func_w_kwargs(): 4 LOAD_CONST 0 (1) LOAD_NAME 4 (lst) - PUSH_NULL LOAD_NAME 3 (fun) + PUSH_NULL LOAD_CONST 3 (0) CALL 1 STORE_SUBSCR @@ -615,14 +615,14 @@ def _tryfinallyconst(b): %3d LOAD_FAST 0 (a) -%3d PUSH_NULL - LOAD_FAST 1 (b) +%3d LOAD_FAST 1 (b) + PUSH_NULL CALL 0 POP_TOP RETURN_VALUE >> PUSH_EXC_INFO - PUSH_NULL LOAD_FAST 1 (b) + PUSH_NULL CALL 0 POP_TOP RERAISE 0 @@ -644,14 +644,14 @@ def _tryfinallyconst(b): %3d NOP -%3d PUSH_NULL - LOAD_FAST 0 (b) +%3d LOAD_FAST 0 (b) + PUSH_NULL CALL 0 POP_TOP RETURN_CONST 1 (1) PUSH_EXC_INFO - PUSH_NULL LOAD_FAST 0 (b) + PUSH_NULL CALL 0 POP_TOP RERAISE 0 @@ -710,7 +710,7 @@ def foo(x): %3d RESUME 0 -%3d LOAD_GLOBAL 1 (NULL + list) +%3d LOAD_GLOBAL 1 (list + NULL) LOAD_FAST 0 (x) BUILD_TUPLE 1 LOAD_CONST 1 ( at 0x..., file "%s", line %d>) @@ -792,7 +792,7 @@ def loop_test(): >> FOR_ITER_LIST 14 (to 48) STORE_FAST 0 (i) -%3d LOAD_GLOBAL_MODULE 1 (NULL + load_test) +%3d LOAD_GLOBAL_MODULE 1 (load_test + NULL) LOAD_FAST 0 (i) CALL_PY_WITH_DEFAULTS 1 POP_TOP @@ -1230,8 +1230,8 @@ def test_call_specialize(self): call_quicken = """\ 0 RESUME 0 - 1 PUSH_NULL - LOAD_NAME 0 (str) + 1 LOAD_NAME 0 (str) + PUSH_NULL LOAD_CONST 0 (1) CALL_NO_KW_STR_1 1 RETURN_VALUE @@ -1629,7 +1629,7 @@ def _prepare_test_cases(): Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='NULL + print', offset=24, start_offset=24, starts_line=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=7, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=None, is_jump_target=False, positions=None), @@ -1659,7 +1659,7 @@ def _prepare_test_cases(): Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='NULL + print', offset=30, start_offset=30, starts_line=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=5, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=None, is_jump_target=False, positions=None), @@ -1673,7 +1673,7 @@ def _prepare_test_cases(): expected_opinfo_inner = [ Instruction(opname='COPY_FREE_VARS', opcode=149, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='NULL + print', offset=4, start_offset=4, starts_line=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=4, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), @@ -1686,13 +1686,13 @@ def _prepare_test_cases(): expected_opinfo_jumpy = [ Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=1, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='range', argrepr='NULL + range', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='FOR_ITER', opcode=93, arg=28, argval=84, argrepr='to 84', offset=24, start_offset=24, starts_line=None, is_jump_target=True, positions=None), Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=30, start_offset=30, starts_line=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=4, is_jump_target=False, positions=None), Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=None, is_jump_target=False, positions=None), @@ -1709,14 +1709,14 @@ def _prepare_test_cases(): Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=80, start_offset=80, starts_line=8, is_jump_target=True, positions=None), Instruction(opname='JUMP_FORWARD', opcode=110, arg=12, argval=108, argrepr='to 108', offset=82, start_offset=82, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='END_FOR', opcode=4, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=3, is_jump_target=True, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=86, start_offset=86, starts_line=10, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=86, start_offset=86, starts_line=10, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=96, start_offset=96, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=98, start_offset=98, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=106, start_offset=106, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_FAST_CHECK', opcode=127, arg=0, argval='i', argrepr='i', offset=108, start_offset=108, starts_line=11, is_jump_target=True, positions=None), Instruction(opname='TO_BOOL', opcode=6, arg=None, argval=None, argrepr='', offset=110, start_offset=110, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=37, argval=194, argrepr='to 194', offset=118, start_offset=118, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=120, start_offset=120, starts_line=12, is_jump_target=True, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=120, start_offset=120, starts_line=12, is_jump_target=True, positions=None), Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=130, start_offset=130, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=132, start_offset=132, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=140, start_offset=140, starts_line=None, is_jump_target=False, positions=None), @@ -1738,7 +1738,7 @@ def _prepare_test_cases(): Instruction(opname='TO_BOOL', opcode=6, arg=None, argval=None, argrepr='', offset=180, start_offset=180, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=2, argval=194, argrepr='to 194', offset=188, start_offset=188, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='JUMP_BACKWARD', opcode=140, arg=37, argval=120, argrepr='to 120', offset=190, start_offset=190, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=194, start_offset=194, starts_line=19, is_jump_target=True, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=194, start_offset=194, starts_line=19, is_jump_target=True, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=204, start_offset=204, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=206, start_offset=206, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=214, start_offset=214, starts_line=None, is_jump_target=False, positions=None), @@ -1750,7 +1750,7 @@ def _prepare_test_cases(): Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=228, start_offset=228, starts_line=25, is_jump_target=False, positions=None), Instruction(opname='BEFORE_WITH', opcode=53, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=232, start_offset=232, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=234, start_offset=234, starts_line=26, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=234, start_offset=234, starts_line=26, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=244, start_offset=244, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=246, start_offset=246, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=254, start_offset=254, starts_line=None, is_jump_target=False, positions=None), @@ -1759,7 +1759,7 @@ def _prepare_test_cases(): Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=260, start_offset=260, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=2, argval=2, argrepr='', offset=262, start_offset=262, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=270, start_offset=270, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=272, start_offset=272, starts_line=28, is_jump_target=True, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=272, start_offset=272, starts_line=28, is_jump_target=True, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=282, start_offset=282, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=284, start_offset=284, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=292, start_offset=292, starts_line=None, is_jump_target=False, positions=None), @@ -1782,7 +1782,7 @@ def _prepare_test_cases(): Instruction(opname='CHECK_EXC_MATCH', opcode=36, arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=15, argval=376, argrepr='to 376', offset=344, start_offset=344, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=348, start_offset=348, starts_line=23, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=348, start_offset=348, starts_line=23, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=358, start_offset=358, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=360, start_offset=360, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=None, is_jump_target=False, positions=None), @@ -1793,7 +1793,7 @@ def _prepare_test_cases(): Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=382, start_offset=382, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=386, start_offset=386, starts_line=28, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=386, start_offset=386, starts_line=28, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=396, start_offset=396, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=398, start_offset=398, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=406, start_offset=406, starts_line=None, is_jump_target=False, positions=None), diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-05-09-06-56.gh-issue-105848.Drc-1-.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-05-09-06-56.gh-issue-105848.Drc-1-.rst new file mode 100644 index 0000000000000..6c1c3229475f6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-05-09-06-56.gh-issue-105848.Drc-1-.rst @@ -0,0 +1,3 @@ +Modify the bytecode so that the actual callable for a :opcode:`CALL` is at a +consistent position on the stack (regardless of whether or not +bound-method-calling optimizations are active). diff --git a/Objects/exception_handling_notes.txt b/Objects/exception_handling_notes.txt index 7de01fdbf5ff4..387ef935ce739 100644 --- a/Objects/exception_handling_notes.txt +++ b/Objects/exception_handling_notes.txt @@ -47,7 +47,7 @@ a table to determine where to jump to when an exception is raised. 2 2 NOP - 3 4 LOAD_GLOBAL 1 (NULL + g) + 3 4 LOAD_GLOBAL 1 (g + NULL) 16 LOAD_CONST 1 (0) 18 PRECALL 1 22 CALL 1 diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 9058327e846dc..0dca507e28bc1 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -2,14 +2,14 @@ unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, 0,0,0,0,0,243,164,0,0,0,151,0,100,0,100,1, - 108,0,90,0,100,0,100,1,108,1,90,1,2,0,101,2, - 100,2,171,1,0,0,0,0,0,0,1,0,2,0,101,2, + 108,0,90,0,100,0,100,1,108,1,90,1,101,2,2,0, + 100,2,171,1,0,0,0,0,0,0,1,0,101,2,2,0, 100,3,101,0,106,6,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,171,2,0,0,0,0,0,0, - 1,0,2,0,101,1,106,8,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,171,0,0,0,0,0, + 1,0,101,1,106,8,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,2,0,171,0,0,0,0,0, 0,0,100,4,25,0,0,0,90,5,100,5,68,0,93,20, - 0,0,90,6,2,0,101,2,100,6,101,6,40,0,100,7, + 0,0,90,6,101,2,2,0,100,6,101,6,40,0,100,7, 101,5,101,6,25,0,0,0,40,0,157,4,171,1,0,0, 0,0,0,0,1,0,140,22,0,0,4,0,121,1,41,8, 233,0,0,0,0,78,122,18,70,114,111,122,101,110,32,72, @@ -27,12 +27,12 @@ unsigned char M_test_frozenmain[] = { 0,0,218,3,107,101,121,169,0,243,0,0,0,0,250,18, 116,101,115,116,95,102,114,111,122,101,110,109,97,105,110,46, 112,121,250,8,60,109,111,100,117,108,101,62,114,18,0,0, - 0,1,0,0,0,115,102,0,0,0,240,3,1,1,1,243, + 0,1,0,0,0,115,99,0,0,0,240,3,1,1,1,243, 8,0,1,11,219,0,24,225,0,5,208,6,26,212,0,27, 217,0,5,128,106,144,35,151,40,145,40,212,0,27,216,9, - 38,208,9,26,215,9,38,209,9,38,211,9,40,168,24,209, - 9,50,128,6,240,2,6,12,2,242,0,7,1,42,128,67, - 241,14,0,5,10,136,71,144,67,144,53,152,2,152,54,160, - 35,153,59,152,45,208,10,40,214,4,41,241,15,7,1,42, - 114,16,0,0,0, + 26,215,9,38,210,9,38,211,9,40,168,24,209,9,50,128, + 6,240,2,6,12,2,242,0,7,1,42,128,67,241,14,0, + 5,10,136,71,144,67,144,53,152,2,152,54,160,35,153,59, + 152,45,208,10,40,214,4,41,241,15,7,1,42,114,16,0, + 0,0, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d6bfb624c7713..b2281abc6663d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1315,7 +1315,7 @@ dummy_func( LOAD_GLOBAL_BUILTIN, }; - inst(LOAD_GLOBAL, (unused/1, unused/1, unused/1, unused/1 -- null if (oparg & 1), v)) { + inst(LOAD_GLOBAL, (unused/1, unused/1, unused/1, unused/1 -- res, null if (oparg & 1))) { #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1331,10 +1331,10 @@ dummy_func( if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (v == NULL) { + res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ @@ -1343,17 +1343,17 @@ dummy_func( } ERROR_IF(true, error); } - Py_INCREF(v); + Py_INCREF(res); } else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0, error); - if (v == NULL) { + ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0, error); + if (res == NULL) { /* namespace 2: builtins */ - ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0, error); - if (v == NULL) { + ERROR_IF(PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0, error); + if (res == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1378,7 +1378,7 @@ dummy_func( assert(DK_IS_UNICODE(dict->ma_keys)); } - op(_LOAD_GLOBAL_MODULE, (index/1 -- null if (oparg & 1), res)) { + op(_LOAD_GLOBAL_MODULE, (index/1 -- res, null if (oparg & 1))) { PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; @@ -1388,7 +1388,7 @@ dummy_func( null = NULL; } - op(_LOAD_GLOBAL_BUILTINS, (index/1 -- null if (oparg & 1), res)) { + op(_LOAD_GLOBAL_BUILTINS, (index/1 -- res, null if (oparg & 1))) { PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; @@ -1614,8 +1614,7 @@ dummy_func( ERROR_IF(map == NULL, error); } - inst(DICT_UPDATE, (update --)) { - PyObject *dict = PEEK(oparg + 1); // update is still on the stack + inst(DICT_UPDATE, (dict, unused[oparg - 1], update -- dict, unused[oparg - 1])) { if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -1628,26 +1627,23 @@ dummy_func( DECREF_INPUTS(); } - inst(DICT_MERGE, (update --)) { - PyObject *dict = PEEK(oparg + 1); // update is still on the stack - + inst(DICT_MERGE, (callable, unused, unused, dict, unused[oparg - 1], update -- callable, unused, unused, dict, unused[oparg - 1])) { if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, PEEK(3 + oparg), update); + _PyEval_FormatKwargsError(tstate, callable, update); DECREF_INPUTS(); ERROR_IF(true, error); } DECREF_INPUTS(); } - inst(MAP_ADD, (key, value --)) { - PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack + inst(MAP_ADD, (dict, unused[oparg - 1], key, value -- dict, unused[oparg - 1])) { assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references ERROR_IF(_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0, error); } - inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/9, unused, unused, unused -- unused if (oparg & 1), unused)) { + inst(INSTRUMENTED_LOAD_SUPER_ATTR, (unused/9, unused, unused, unused -- unused, unused if (oparg & 1))) { _PySuperAttrCache *cache = (_PySuperAttrCache *)next_instr; // cancel out the decrement that will happen in LOAD_SUPER_ATTR; we // don't want to specialize instrumented instructions @@ -1660,7 +1656,7 @@ dummy_func( LOAD_SUPER_ATTR_METHOD, }; - inst(LOAD_SUPER_ATTR, (unused/1, global_super, class, self -- res2 if (oparg & 1), res)) { + inst(LOAD_SUPER_ATTR, (unused/1, global_super, class, self -- attr, null if (oparg & 1))) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); int load_method = oparg & 1; #if ENABLE_SPECIALIZATION @@ -1704,9 +1700,10 @@ dummy_func( } DECREF_INPUTS(); ERROR_IF(super == NULL, error); - res = PyObject_GetAttr(super, name); + attr = PyObject_GetAttr(super, name); Py_DECREF(super); - ERROR_IF(res == NULL, error); + ERROR_IF(attr == NULL, error); + null = NULL; } pseudo(LOAD_SUPER_METHOD) = { @@ -1721,18 +1718,18 @@ dummy_func( LOAD_SUPER_ATTR, }; - inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super, class, self -- res2 if (oparg & 1), res)) { + inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super, class, self -- attr, unused if (0))) { assert(!(oparg & 1)); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(attr == NULL, error); } - inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super, class, self -- res2, res)) { + inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super, class, self -- attr, self_or_null)) { assert(oparg & 1); DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); @@ -1740,20 +1737,19 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; - res2 = _PySuper_Lookup(cls, self, name, + attr = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); Py_DECREF(global_super); Py_DECREF(class); - if (res2 == NULL) { + if (attr == NULL) { Py_DECREF(self); ERROR_IF(true, error); } if (method_found) { - res = self; // transfer ownership + self_or_null = self; // transfer ownership } else { Py_DECREF(self); - res = res2; - res2 = NULL; + self_or_null = NULL; } } @@ -1772,7 +1768,7 @@ dummy_func( LOAD_ATTR_NONDESCRIPTOR_NO_DICT, }; - inst(LOAD_ATTR, (unused/9, owner -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR, (unused/9, owner -- attr, self_or_null if (oparg & 1))) { #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1787,16 +1783,15 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - PyObject* meth = NULL; - if (_PyObject_GetMethod(owner, name, &meth)) { + attr = NULL; + if (_PyObject_GetMethod(owner, name, &attr)) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(meth != NULL); // No errors on this branch - res2 = meth; - res = owner; // Transfer ownership + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership } else { /* meth is not an unbound method (but a regular attr, or @@ -1807,16 +1802,15 @@ dummy_func( NULL | meth | arg1 | ... | argN */ DECREF_INPUTS(); - ERROR_IF(meth == NULL, error); - res2 = NULL; - res = meth; + ERROR_IF(attr == NULL, error); + self_or_null = NULL; } } else { /* Classic, pushes one value. */ - res = PyObject_GetAttr(owner, name); + attr = PyObject_GetAttr(owner, name); DECREF_INPUTS(); - ERROR_IF(res == NULL, error); + ERROR_IF(attr == NULL, error); } } @@ -1837,13 +1831,13 @@ dummy_func( DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); } - op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- res2 if (oparg & 1), res)) { + op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - res = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(res == NULL, LOAD_ATTR); + attr = _PyDictOrValues_GetValues(dorv)->values[index]; + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; DECREF_INPUTS(); } @@ -1854,7 +1848,7 @@ dummy_func( _LOAD_ATTR_INSTANCE_VALUE + unused/5; // Skip over rest of cache - inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) { DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); @@ -1862,15 +1856,15 @@ dummy_func( assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - res = ep->me_value; - DEOPT_IF(res == NULL, LOAD_ATTR); + attr = ep->me_value; + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; DECREF_INPUTS(); } - inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); @@ -1886,49 +1880,50 @@ dummy_func( if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = ep->me_value; + attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = ep->me_value; + attr = ep->me_value; } - DEOPT_IF(res == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; DECREF_INPUTS(); } - inst(LOAD_ATTR_SLOT, (unused/1, type_version/2, index/1, unused/5, owner -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_SLOT, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); char *addr = (char *)owner + index; - res = *(PyObject **)addr; - DEOPT_IF(res == NULL, LOAD_ATTR); + attr = *(PyObject **)addr; + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; DECREF_INPUTS(); } - inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, cls -- res2 if (oparg & 1), res)) { + inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, null if (oparg & 1))) { - DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); - DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, + DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); STAT_INC(LOAD_ATTR, hit); - res2 = NULL; - res = descr; - assert(res != NULL); - Py_INCREF(res); + null = NULL; + attr = descr; + assert(attr != NULL); + Py_INCREF(attr); DECREF_INPUTS(); } - inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused if (oparg & 1), unused)) { + inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused, unused if (0))) { + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -1945,16 +1940,15 @@ dummy_func( Py_INCREF(fget); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); // Manipulate stack directly because we exit with DISPATCH_INLINED(). - SET_TOP(NULL); - int shrink_stack = !(oparg & 1); - STACK_SHRINK(shrink_stack); + STACK_SHRINK(1); new_frame->localsplus[0] = owner; SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); } - inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused if (oparg & 1), unused)) { + inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused, unused if (0))) { + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -1972,9 +1966,7 @@ dummy_func( Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); // Manipulate stack directly because we exit with DISPATCH_INLINED(). - SET_TOP(NULL); - int shrink_stack = !(oparg & 1); - STACK_SHRINK(shrink_stack); + STACK_SHRINK(1); new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); @@ -2728,80 +2720,80 @@ dummy_func( exc_info->exc_value = Py_NewRef(new_exc); } - inst(LOAD_ATTR_METHOD_WITH_VALUES, (unused/1, type_version/2, keys_version/2, descr/4, self -- res2 if (1), res)) { + inst(LOAD_ATTR_METHOD_WITH_VALUES, (unused/1, type_version/2, keys_version/2, descr/4, owner -- attr, self if (1))) { assert(oparg & 1); /* Cached method object */ - PyTypeObject *self_cls = Py_TYPE(self); + PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls; - DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - res2 = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); - res = self; + attr = Py_NewRef(descr); + assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + self = owner; } - inst(LOAD_ATTR_METHOD_NO_DICT, (unused/1, type_version/2, unused/2, descr/4, self -- res2 if (1), res)) { + inst(LOAD_ATTR_METHOD_NO_DICT, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, self if (1))) { assert(oparg & 1); - PyTypeObject *self_cls = Py_TYPE(self); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_dictoffset == 0); + PyTypeObject *owner_cls = Py_TYPE(owner); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - res2 = Py_NewRef(descr); - res = self; + attr = Py_NewRef(descr); + self = owner; } - inst(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (unused/1, type_version/2, keys_version/2, descr/4, self -- res2 if (0), res)) { + inst(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, (unused/1, type_version/2, keys_version/2, descr/4, owner -- attr, unused if (0))) { assert((oparg & 1) == 0); - PyTypeObject *self_cls = Py_TYPE(self); + PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls; - DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); - res = Py_NewRef(descr); + attr = Py_NewRef(descr); } - inst(LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (unused/1, type_version/2, unused/2, descr/4, self -- res2 if (0), res)) { + inst(LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, unused if (0))) { assert((oparg & 1) == 0); - PyTypeObject *self_cls = Py_TYPE(self); + PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_dictoffset == 0); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); - res = Py_NewRef(descr); + attr = Py_NewRef(descr); } - inst(LOAD_ATTR_METHOD_LAZY_DICT, (unused/1, type_version/2, unused/2, descr/4, self -- res2 if (1), res)) { + inst(LOAD_ATTR_METHOD_LAZY_DICT, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, self if (1))) { assert(oparg & 1); - PyTypeObject *self_cls = Py_TYPE(self); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - Py_ssize_t dictoffset = self_cls->tp_dictoffset; + PyTypeObject *owner_cls = Py_TYPE(owner); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + Py_ssize_t dictoffset = owner_cls->tp_dictoffset; assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)self + dictoffset); + PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - res2 = Py_NewRef(descr); - res = self; + attr = Py_NewRef(descr); + self = owner; } inst(KW_NAMES, (--)) { @@ -2811,9 +2803,9 @@ dummy_func( } inst(INSTRUMENTED_CALL, ( -- )) { - int is_meth = PEEK(oparg+2) != NULL; + int is_meth = PEEK(oparg + 1) != NULL; int total_args = oparg + is_meth; - PyObject *function = PEEK(total_args + 1); + PyObject *function = PEEK(oparg + 2); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( @@ -2855,11 +2847,9 @@ dummy_func( // (Some args may be keywords, see KW_NAMES, which sets 'kwnames'.) // On exit, the stack is [result]. // When calling Python, inline the call using DISPATCH_INLINED(). - inst(CALL, (unused/1, unused/2, method, callable, args[oparg] -- res)) { - int is_meth = method != NULL; + inst(CALL, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -2873,13 +2863,12 @@ dummy_func( STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - if (!is_meth && Py_TYPE(callable) == &PyMethod_Type) { - is_meth = 1; // For consistenct; it's dead, though + if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { args--; total_args++; PyObject *self = ((PyMethodObject *)callable)->im_self; args[0] = Py_NewRef(self); - method = ((PyMethodObject *)callable)->im_func; + PyObject *method = ((PyMethodObject *)callable)->im_func; args[-1] = Py_NewRef(method); Py_DECREF(callable); callable = method; @@ -2915,7 +2904,7 @@ dummy_func( kwnames); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); + &_PyInstrumentation_MISSING : args[0]; if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -2943,25 +2932,23 @@ dummy_func( // Start out with [NULL, bound_method, arg1, arg2, ...] // Transform to [callable, self, arg1, arg2, ...] // Then fall through to CALL_PY_EXACT_ARGS - inst(CALL_BOUND_METHOD_EXACT_ARGS, (unused/1, unused/2, method, callable, unused[oparg] -- unused)) { - DEOPT_IF(method != NULL, CALL); + inst(CALL_BOUND_METHOD_EXACT_ARGS, (unused/1, unused/2, callable, null, unused[oparg] -- unused)) { + DEOPT_IF(null != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); PyObject *self = ((PyMethodObject *)callable)->im_self; - PEEK(oparg + 1) = Py_NewRef(self); // callable + PEEK(oparg + 1) = Py_NewRef(self); // self_or_null PyObject *meth = ((PyMethodObject *)callable)->im_func; - PEEK(oparg + 2) = Py_NewRef(meth); // method + PEEK(oparg + 2) = Py_NewRef(meth); // callable Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); } - inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, method, callable, args[oparg] -- unused)) { + inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); - int is_meth = method != NULL; int argcount = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; argcount++; } @@ -2983,13 +2970,11 @@ dummy_func( DISPATCH_INLINED(new_frame); } - inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, method, callable, args[oparg] -- unused)) { + inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); - int is_meth = method != NULL; int argcount = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; argcount++; } @@ -3021,7 +3006,7 @@ dummy_func( DISPATCH_INLINED(new_frame); } - inst(CALL_NO_KW_TYPE_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_TYPE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3033,7 +3018,7 @@ dummy_func( Py_DECREF(&PyType_Type); // I.e., callable } - inst(CALL_NO_KW_STR_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_STR_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3047,7 +3032,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_TUPLE_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_TUPLE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3061,7 +3046,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, null, callable, args[oparg] -- unused)) { + inst(CALL_NO_KW_ALLOC_AND_ENTER_INIT, (unused/1, unused/2, callable, null, args[oparg] -- unused)) { /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) @@ -3124,11 +3109,9 @@ dummy_func( } } - inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, method, callable, args[oparg] -- res)) { - int is_meth = method != NULL; + inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -3149,13 +3132,11 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_BUILTIN_O, (unused/1, unused/2, method, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_BUILTIN_O, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_O functions */ ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -3180,13 +3161,11 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_BUILTIN_FAST, (unused/1, unused/2, method, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_BUILTIN_FAST, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -3215,12 +3194,10 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, method, callable, args[oparg] -- res)) { + inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -3250,13 +3227,11 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_LEN, (unused/1, unused/2, method, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_LEN, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); /* len(o) */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -3277,13 +3252,11 @@ dummy_func( ERROR_IF(res == NULL, error); } - inst(CALL_NO_KW_ISINSTANCE, (unused/1, unused/2, method, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_ISINSTANCE, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); /* isinstance(o, o2) */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -3307,19 +3280,19 @@ dummy_func( } // This is secretly a super-instruction - inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, method, self, args[oparg] -- unused)) { + inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) { ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); - assert(method != NULL); + assert(self != NULL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(method != interp->callable_cache.list_append, CALL); + DEOPT_IF(callable != interp->callable_cache.list_append, CALL); DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { goto pop_1_error; // Since arg is DECREF'ed already } Py_DECREF(self); - Py_DECREF(method); + Py_DECREF(callable); STACK_SHRINK(3); // CALL + POP_TOP SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); @@ -3327,23 +3300,21 @@ dummy_func( DISPATCH(); } - inst(CALL_NO_KW_METHOD_DESCRIPTOR_O, (unused/1, unused/2, method, unused, args[oparg] -- res)) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_O, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_O, CALL); PyObject *arg = args[1]; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -3361,19 +3332,17 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (unused/1, unused/2, method, unused, args[oparg] -- res)) { - int is_meth = method != NULL; + inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = callable->d_common.d_type; + PyTypeObject *d_type = method->d_common.d_type; PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); @@ -3393,21 +3362,20 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, method, unused, args[oparg] -- res)) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); assert(oparg == 0 || oparg == 1); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -3425,22 +3393,20 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, method, unused, args[oparg] -- res)) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; @@ -3460,7 +3426,7 @@ dummy_func( GO_TO_INSTRUCTION(CALL_FUNCTION_EX); } - inst(CALL_FUNCTION_EX, (unused, func, callargs, kwargs if (oparg & 1) -- result)) { + inst(CALL_FUNCTION_EX, (func, unused, callargs, kwargs if (oparg & 1) -- result)) { // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -3523,7 +3489,7 @@ dummy_func( result = PyObject_Call(func, callargs, kwargs); } DECREF_INPUTS(); - assert(PEEK(3 + (oparg & 1)) == NULL); + assert(PEEK(2 + (oparg & 1)) == NULL); ERROR_IF(result == NULL, error); CHECK_EVAL_BREAKER(); } diff --git a/Python/compile.c b/Python/compile.c index b673e3ac6c1cc..68af5d61ac8de 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2361,11 +2361,6 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) int is_generic = asdl_seq_LEN(type_params) > 0; - if (is_generic) { - // Used by the CALL to the type parameters function. - ADDOP(c, loc, PUSH_NULL); - } - funcflags = compiler_default_arguments(c, loc, args); if (funcflags == -1) { return ERROR; @@ -2436,8 +2431,12 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) Py_DECREF(co); if (num_typeparam_args > 0) { ADDOP_I(c, loc, SWAP, num_typeparam_args + 1); + ADDOP_I(c, loc, CALL, num_typeparam_args - 1); + } + else { + ADDOP(c, loc, PUSH_NULL); + ADDOP_I(c, loc, CALL, 0); } - ADDOP_I(c, loc, CALL, num_typeparam_args); } RETURN_IF_ERROR(compiler_apply_decorators(c, decos)); @@ -2565,8 +2564,8 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) // these instructions should be attributed to the class line, // not a decorator line loc = LOC(s); - ADDOP(c, loc, PUSH_NULL); ADDOP(c, loc, LOAD_BUILD_CLASS); + ADDOP(c, loc, PUSH_NULL); /* 3. load a function (or closure) made from the code object */ if (compiler_make_closure(c, loc, co, 0) < 0) { @@ -2598,7 +2597,6 @@ compiler_class(struct compiler *c, stmt_ty s) int is_generic = asdl_seq_LEN(type_params) > 0; if (is_generic) { Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name)); - ADDOP(c, loc, PUSH_NULL); PyObject *type_params_name = PyUnicode_FromFormat("", s->v.ClassDef.name); if (!type_params_name) { @@ -2666,6 +2664,7 @@ compiler_class(struct compiler *c, stmt_ty s) return ERROR; } Py_DECREF(co); + ADDOP(c, loc, PUSH_NULL); ADDOP_I(c, loc, CALL, 0); } else { RETURN_IF_ERROR(compiler_call_helper(c, loc, 2, @@ -2716,7 +2715,6 @@ compiler_typealias(struct compiler *c, stmt_ty s) int is_generic = asdl_seq_LEN(type_params) > 0; PyObject *name = s->v.TypeAlias.name->v.Name.id; if (is_generic) { - ADDOP(c, loc, PUSH_NULL); PyObject *type_params_name = PyUnicode_FromFormat("", name); if (!type_params_name) { @@ -2756,6 +2754,7 @@ compiler_typealias(struct compiler *c, stmt_ty s) return ERROR; } Py_DECREF(co); + ADDOP(c, loc, PUSH_NULL); ADDOP_I(c, loc, CALL, 0); } RETURN_IF_ERROR(compiler_nameop(c, loc, name, Store)); @@ -4994,9 +4993,9 @@ compiler_call(struct compiler *c, expr_ty e) return SUCCESS; } RETURN_IF_ERROR(check_caller(c, e->v.Call.func)); + VISIT(c, expr, e->v.Call.func); location loc = LOC(e->v.Call.func); ADDOP(c, loc, PUSH_NULL); - VISIT(c, expr, e->v.Call.func); loc = LOC(e); return compiler_call_helper(c, loc, 0, e->v.Call.args, diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 27be8a383989e..d6d541a3b61ab 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1062,8 +1062,8 @@ case LOAD_GLOBAL: { static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyObject *res; PyObject *null = NULL; - PyObject *v; #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1079,10 +1079,10 @@ if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (v == NULL) { + res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ @@ -1091,17 +1091,17 @@ } if (true) goto error; } - Py_INCREF(v); + Py_INCREF(res); } else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0) goto error; - if (v == NULL) { + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; + if (res == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) goto error; - if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; + if (res == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1112,8 +1112,8 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } - stack_pointer[-1] = v; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } break; } @@ -1136,8 +1136,8 @@ } case _LOAD_GLOBAL_MODULE: { - PyObject *null = NULL; PyObject *res; + PyObject *null = NULL; uint16_t index = (uint16_t)operand; PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); @@ -1148,14 +1148,14 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } break; } case _LOAD_GLOBAL_BUILTINS: { - PyObject *null = NULL; PyObject *res; + PyObject *null = NULL; uint16_t index = (uint16_t)operand; PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); @@ -1166,8 +1166,8 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } break; } @@ -1445,8 +1445,9 @@ case DICT_UPDATE: { PyObject *update; + PyObject *dict; update = stack_pointer[-1]; - PyObject *dict = PEEK(oparg + 1); // update is still on the stack + dict = stack_pointer[-2 - (oparg - 1)]; if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -1463,11 +1464,13 @@ case DICT_MERGE: { PyObject *update; + PyObject *dict; + PyObject *callable; update = stack_pointer[-1]; - PyObject *dict = PEEK(oparg + 1); // update is still on the stack - + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, PEEK(3 + oparg), update); + _PyEval_FormatKwargsError(tstate, callable, update); Py_DECREF(update); if (true) goto pop_1_error; } @@ -1479,9 +1482,10 @@ case MAP_ADD: { PyObject *value; PyObject *key; + PyObject *dict; value = stack_pointer[-1]; key = stack_pointer[-2]; - PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack + dict = stack_pointer[-3 - (oparg - 1)]; assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references @@ -1494,8 +1498,7 @@ PyObject *self; PyObject *class; PyObject *global_super; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; @@ -1504,15 +1507,13 @@ DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); - if (res == NULL) goto pop_3_error; + if (attr == NULL) goto pop_3_error; STACK_SHRINK(2); - STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (0 ? 1 : 0)] = attr; break; } @@ -1520,8 +1521,8 @@ PyObject *self; PyObject *class; PyObject *global_super; - PyObject *res2; - PyObject *res; + PyObject *attr; + PyObject *self_or_null; self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; @@ -1532,32 +1533,31 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; - res2 = _PySuper_Lookup(cls, self, name, + attr = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); Py_DECREF(global_super); Py_DECREF(class); - if (res2 == NULL) { + if (attr == NULL) { Py_DECREF(self); if (true) goto pop_3_error; } if (method_found) { - res = self; // transfer ownership + self_or_null = self; // transfer ownership } else { Py_DECREF(self); - res = res2; - res2 = NULL; + self_or_null = NULL; } STACK_SHRINK(1); - stack_pointer[-2] = res2; - stack_pointer[-1] = res; + stack_pointer[-2] = attr; + stack_pointer[-1] = self_or_null; break; } case LOAD_ATTR: { static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *self_or_null = NULL; owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -1573,16 +1573,15 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - PyObject* meth = NULL; - if (_PyObject_GetMethod(owner, name, &meth)) { + attr = NULL; + if (_PyObject_GetMethod(owner, name, &attr)) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(meth != NULL); // No errors on this branch - res2 = meth; - res = owner; // Transfer ownership + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership } else { /* meth is not an unbound method (but a regular attr, or @@ -1593,20 +1592,19 @@ NULL | meth | arg1 | ... | argN */ Py_DECREF(owner); - if (meth == NULL) goto pop_1_error; - res2 = NULL; - res = meth; + if (attr == NULL) goto pop_1_error; + self_or_null = NULL; } } else { /* Classic, pushes one value. */ - res = PyObject_GetAttr(owner, name); + attr = PyObject_GetAttr(owner, name); Py_DECREF(owner); - if (res == NULL) goto pop_1_error; + if (attr == NULL) goto pop_1_error; } STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } break; } @@ -1632,20 +1630,20 @@ case _LOAD_ATTR_INSTANCE_VALUE: { PyObject *owner; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *null = NULL; owner = stack_pointer[-1]; uint16_t index = (uint16_t)operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - res = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(res == NULL, LOAD_ATTR); + attr = _PyDictOrValues_GetValues(dorv)->values[index]; + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } break; } @@ -2157,12 +2155,12 @@ case CALL_NO_KW_TYPE_1: { PyObject **args; - PyObject *callable; PyObject *null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - null = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2180,12 +2178,12 @@ case CALL_NO_KW_STR_1: { PyObject **args; - PyObject *callable; PyObject *null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - null = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2205,12 +2203,12 @@ case CALL_NO_KW_TUPLE_1: { PyObject **args; - PyObject *callable; PyObject *null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - null = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2244,18 +2242,16 @@ case CALL_NO_KW_BUILTIN_O: { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -2286,18 +2282,16 @@ case CALL_NO_KW_BUILTIN_FAST: { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -2332,18 +2326,16 @@ case CALL_NO_KW_LEN: { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* len(o) */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -2370,18 +2362,16 @@ case CALL_NO_KW_ISINSTANCE: { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* isinstance(o, o2) */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -2410,26 +2400,26 @@ case CALL_NO_KW_METHOD_DESCRIPTOR_O: { PyObject **args; - PyObject *method; + PyObject *self_or_null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_O, CALL); PyObject *arg = args[1]; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -2453,24 +2443,25 @@ case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: { PyObject **args; - PyObject *method; + PyObject *self_or_null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 0 || oparg == 1); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -2494,25 +2485,25 @@ case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: { PyObject **args; - PyObject *method; + PyObject *self_or_null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; diff --git a/Python/flowgraph.c b/Python/flowgraph.c index a72b85d45aa27..5b6b3f3cfefb0 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1536,10 +1536,10 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) break; case KW_NAMES: break; - case PUSH_NULL: - if (nextop == LOAD_GLOBAL && (inst[1].i_opcode & 1) == 0) { - INSTR_SET_OP0(inst, NOP); - inst[1].i_oparg |= 1; + case LOAD_GLOBAL: + if (nextop == PUSH_NULL && (oparg & 1) == 0) { + INSTR_SET_OP1(inst, LOAD_GLOBAL, oparg | 1); + INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); } break; case COMPARE_OP: diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d7db8b07005fd..cf20b869b8182 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1695,8 +1695,8 @@ TARGET(LOAD_GLOBAL) { PREDICTED(LOAD_GLOBAL); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + PyObject *res; PyObject *null = NULL; - PyObject *v; #if ENABLE_SPECIALIZATION _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1712,10 +1712,10 @@ if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (v == NULL) { + res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (res == NULL) { if (!_PyErr_Occurred(tstate)) { /* _PyDict_LoadGlobal() returns NULL without raising * an exception if the key doesn't exist */ @@ -1724,17 +1724,17 @@ } if (true) goto error; } - Py_INCREF(v); + Py_INCREF(res); } else { /* Slow-path if globals or builtins is not a dict */ /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &v) < 0) goto error; - if (v == NULL) { + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; + if (res == NULL) { /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) goto error; - if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; + if (res == NULL) { _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); @@ -1745,15 +1745,15 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } - stack_pointer[-1] = v; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 4; DISPATCH(); } TARGET(LOAD_GLOBAL_MODULE) { - PyObject *null = NULL; PyObject *res; + PyObject *null = NULL; // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&next_instr[1].cache); @@ -1775,15 +1775,15 @@ } STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 4; DISPATCH(); } TARGET(LOAD_GLOBAL_BUILTIN) { - PyObject *null = NULL; PyObject *res; + PyObject *null = NULL; // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&next_instr[1].cache); @@ -1813,8 +1813,8 @@ } STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = null; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 4; DISPATCH(); } @@ -2105,8 +2105,9 @@ TARGET(DICT_UPDATE) { PyObject *update; + PyObject *dict; update = stack_pointer[-1]; - PyObject *dict = PEEK(oparg + 1); // update is still on the stack + dict = stack_pointer[-2 - (oparg - 1)]; if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { _PyErr_Format(tstate, PyExc_TypeError, @@ -2123,11 +2124,13 @@ TARGET(DICT_MERGE) { PyObject *update; + PyObject *dict; + PyObject *callable; update = stack_pointer[-1]; - PyObject *dict = PEEK(oparg + 1); // update is still on the stack - + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; if (_PyDict_MergeEx(dict, update, 2) < 0) { - _PyEval_FormatKwargsError(tstate, PEEK(3 + oparg), update); + _PyEval_FormatKwargsError(tstate, callable, update); Py_DECREF(update); if (true) goto pop_1_error; } @@ -2139,9 +2142,10 @@ TARGET(MAP_ADD) { PyObject *value; PyObject *key; + PyObject *dict; value = stack_pointer[-1]; key = stack_pointer[-2]; - PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack + dict = stack_pointer[-3 - (oparg - 1)]; assert(PyDict_CheckExact(dict)); /* dict[key] = value */ // Do not DECREF INPUTS because the function steals the references @@ -2166,8 +2170,8 @@ PyObject *self; PyObject *class; PyObject *global_super; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *null = NULL; self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; @@ -2216,13 +2220,14 @@ Py_DECREF(class); Py_DECREF(self); if (super == NULL) goto pop_3_error; - res = PyObject_GetAttr(super, name); + attr = PyObject_GetAttr(super, name); Py_DECREF(super); - if (res == NULL) goto pop_3_error; + if (attr == NULL) goto pop_3_error; + null = NULL; STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 1; DISPATCH(); } @@ -2231,8 +2236,7 @@ PyObject *self; PyObject *class; PyObject *global_super; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; @@ -2241,15 +2245,13 @@ DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); Py_DECREF(global_super); Py_DECREF(class); Py_DECREF(self); - if (res == NULL) goto pop_3_error; + if (attr == NULL) goto pop_3_error; STACK_SHRINK(2); - STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (0 ? 1 : 0)] = attr; next_instr += 1; DISPATCH(); } @@ -2258,8 +2260,8 @@ PyObject *self; PyObject *class; PyObject *global_super; - PyObject *res2; - PyObject *res; + PyObject *attr; + PyObject *self_or_null; self = stack_pointer[-1]; class = stack_pointer[-2]; global_super = stack_pointer[-3]; @@ -2270,24 +2272,23 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; - res2 = _PySuper_Lookup(cls, self, name, + attr = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); Py_DECREF(global_super); Py_DECREF(class); - if (res2 == NULL) { + if (attr == NULL) { Py_DECREF(self); if (true) goto pop_3_error; } if (method_found) { - res = self; // transfer ownership + self_or_null = self; // transfer ownership } else { Py_DECREF(self); - res = res2; - res2 = NULL; + self_or_null = NULL; } STACK_SHRINK(1); - stack_pointer[-2] = res2; - stack_pointer[-1] = res; + stack_pointer[-2] = attr; + stack_pointer[-1] = self_or_null; next_instr += 1; DISPATCH(); } @@ -2296,8 +2297,8 @@ PREDICTED(LOAD_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *self_or_null = NULL; owner = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyAttrCache *cache = (_PyAttrCache *)next_instr; @@ -2313,16 +2314,15 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - PyObject* meth = NULL; - if (_PyObject_GetMethod(owner, name, &meth)) { + attr = NULL; + if (_PyObject_GetMethod(owner, name, &attr)) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. meth | self | arg1 | ... | argN */ - assert(meth != NULL); // No errors on this branch - res2 = meth; - res = owner; // Transfer ownership + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership } else { /* meth is not an unbound method (but a regular attr, or @@ -2333,28 +2333,27 @@ NULL | meth | arg1 | ... | argN */ Py_DECREF(owner); - if (meth == NULL) goto pop_1_error; - res2 = NULL; - res = meth; + if (attr == NULL) goto pop_1_error; + self_or_null = NULL; } } else { /* Classic, pushes one value. */ - res = PyObject_GetAttr(owner, name); + attr = PyObject_GetAttr(owner, name); Py_DECREF(owner); - if (res == NULL) goto pop_1_error; + if (attr == NULL) goto pop_1_error; } STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_INSTANCE_VALUE) { PyObject *owner; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *null = NULL; // _GUARD_TYPE_VERSION owner = stack_pointer[-1]; { @@ -2374,24 +2373,24 @@ { uint16_t index = read_u16(&next_instr[3].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - res = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(res == NULL, LOAD_ATTR); + attr = _PyDictOrValues_GetValues(dorv)->values[index]; + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; Py_DECREF(owner); } STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_MODULE) { PyObject *owner; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *null = NULL; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); @@ -2402,23 +2401,23 @@ assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; - res = ep->me_value; - DEOPT_IF(res == NULL, LOAD_ATTR); + attr = ep->me_value; + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_WITH_HINT) { PyObject *owner; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *null = NULL; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); @@ -2437,29 +2436,29 @@ if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = ep->me_value; + attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; DEOPT_IF(ep->me_key != name, LOAD_ATTR); - res = ep->me_value; + attr = ep->me_value; } - DEOPT_IF(res == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_SLOT) { PyObject *owner; - PyObject *res2 = NULL; - PyObject *res; + PyObject *attr; + PyObject *null = NULL; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); @@ -2467,41 +2466,41 @@ assert(type_version != 0); DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); char *addr = (char *)owner + index; - res = *(PyObject **)addr; - DEOPT_IF(res == NULL, LOAD_ATTR); + attr = *(PyObject **)addr; + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(res); - res2 = NULL; + Py_INCREF(attr); + null = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_CLASS) { - PyObject *cls; - PyObject *res2 = NULL; - PyObject *res; - cls = stack_pointer[-1]; + PyObject *owner; + PyObject *attr; + PyObject *null = NULL; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); - DEOPT_IF(!PyType_Check(cls), LOAD_ATTR); - DEOPT_IF(((PyTypeObject *)cls)->tp_version_tag != type_version, + DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); STAT_INC(LOAD_ATTR, hit); - res2 = NULL; - res = descr; - assert(res != NULL); - Py_INCREF(res); - Py_DECREF(cls); + null = NULL; + attr = descr; + assert(attr != NULL); + Py_INCREF(attr); + Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); - if (oparg & 1) { stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res2; } - stack_pointer[-1] = res; + stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; + if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = null; } next_instr += 9; DISPATCH(); } @@ -2512,6 +2511,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); @@ -2528,14 +2528,11 @@ Py_INCREF(fget); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); // Manipulate stack directly because we exit with DISPATCH_INLINED(). - SET_TOP(NULL); - int shrink_stack = !(oparg & 1); - STACK_SHRINK(shrink_stack); + STACK_SHRINK(1); new_frame->localsplus[0] = owner; SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - STACK_GROW(((oparg & 1) ? 1 : 0)); } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { @@ -2544,6 +2541,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); + assert((oparg & 1) == 0); DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); PyTypeObject *cls = Py_TYPE(owner); DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); @@ -2561,15 +2559,12 @@ Py_INCREF(f); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2); // Manipulate stack directly because we exit with DISPATCH_INLINED(). - SET_TOP(NULL); - int shrink_stack = !(oparg & 1); - STACK_SHRINK(shrink_stack); + STACK_SHRINK(1); new_frame->localsplus[0] = owner; new_frame->localsplus[1] = Py_NewRef(name); SKIP_OVER(INLINE_CACHE_ENTRIES_LOAD_ATTR); frame->return_offset = 0; DISPATCH_INLINED(new_frame); - STACK_GROW(((oparg & 1) ? 1 : 0)); } TARGET(STORE_ATTR_INSTANCE_VALUE) { @@ -3499,128 +3494,128 @@ } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + PyObject *owner; + PyObject *attr; PyObject *self; - PyObject *res2; - PyObject *res; - self = stack_pointer[-1]; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert(oparg & 1); /* Cached method object */ - PyTypeObject *self_cls = Py_TYPE(self); + PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls; - DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - res2 = Py_NewRef(descr); - assert(_PyType_HasFeature(Py_TYPE(res2), Py_TPFLAGS_METHOD_DESCRIPTOR)); - res = self; + attr = Py_NewRef(descr); + assert(_PyType_HasFeature(Py_TYPE(attr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + self = owner; STACK_GROW(1); - stack_pointer[-2] = res2; - stack_pointer[-1] = res; + stack_pointer[-2] = attr; + stack_pointer[-1] = self; next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_NO_DICT) { + PyObject *owner; + PyObject *attr; PyObject *self; - PyObject *res2; - PyObject *res; - self = stack_pointer[-1]; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert(oparg & 1); - PyTypeObject *self_cls = Py_TYPE(self); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_dictoffset == 0); + PyTypeObject *owner_cls = Py_TYPE(owner); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - res2 = Py_NewRef(descr); - res = self; + attr = Py_NewRef(descr); + self = owner; STACK_GROW(1); - stack_pointer[-2] = res2; - stack_pointer[-1] = res; + stack_pointer[-2] = attr; + stack_pointer[-1] = self; next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - PyObject *self; - PyObject *res; - self = stack_pointer[-1]; + PyObject *owner; + PyObject *attr; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t keys_version = read_u32(&next_instr[3].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert((oparg & 1) == 0); - PyTypeObject *self_cls = Py_TYPE(self); + PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); - PyHeapTypeObject *self_heap_type = (PyHeapTypeObject *)self_cls; - DEOPT_IF(self_heap_type->ht_cached_keys->dk_version != + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(self); - res = Py_NewRef(descr); - stack_pointer[-1] = res; + Py_DECREF(owner); + attr = Py_NewRef(descr); + stack_pointer[-1 - (0 ? 1 : 0)] = attr; next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - PyObject *self; - PyObject *res; - self = stack_pointer[-1]; + PyObject *owner; + PyObject *attr; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert((oparg & 1) == 0); - PyTypeObject *self_cls = Py_TYPE(self); + PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - assert(self_cls->tp_dictoffset == 0); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + assert(owner_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); - Py_DECREF(self); - res = Py_NewRef(descr); - stack_pointer[-1] = res; + Py_DECREF(owner); + attr = Py_NewRef(descr); + stack_pointer[-1 - (0 ? 1 : 0)] = attr; next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + PyObject *owner; + PyObject *attr; PyObject *self; - PyObject *res2; - PyObject *res; - self = stack_pointer[-1]; + owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); PyObject *descr = read_obj(&next_instr[5].cache); assert(oparg & 1); - PyTypeObject *self_cls = Py_TYPE(self); - DEOPT_IF(self_cls->tp_version_tag != type_version, LOAD_ATTR); - Py_ssize_t dictoffset = self_cls->tp_dictoffset; + PyTypeObject *owner_cls = Py_TYPE(owner); + DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + Py_ssize_t dictoffset = owner_cls->tp_dictoffset; assert(dictoffset > 0); - PyObject *dict = *(PyObject **)((char *)self + dictoffset); + PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ DEOPT_IF(dict != NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - res2 = Py_NewRef(descr); - res = self; + attr = Py_NewRef(descr); + self = owner; STACK_GROW(1); - stack_pointer[-2] = res2; - stack_pointer[-1] = res; + stack_pointer[-2] = attr; + stack_pointer[-1] = self; next_instr += 9; DISPATCH(); } @@ -3633,9 +3628,9 @@ } TARGET(INSTRUMENTED_CALL) { - int is_meth = PEEK(oparg+2) != NULL; + int is_meth = PEEK(oparg + 1) != NULL; int total_args = oparg + is_meth; - PyObject *function = PEEK(total_args + 1); + PyObject *function = PEEK(oparg + 2); PyObject *arg = total_args == 0 ? &_PyInstrumentation_MISSING : PEEK(total_args); int err = _Py_call_instrumentation_2args( @@ -3651,16 +3646,14 @@ PREDICTED(CALL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; - int is_meth = method != NULL; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -3674,13 +3667,12 @@ STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); #endif /* ENABLE_SPECIALIZATION */ - if (!is_meth && Py_TYPE(callable) == &PyMethod_Type) { - is_meth = 1; // For consistenct; it's dead, though + if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { args--; total_args++; PyObject *self = ((PyMethodObject *)callable)->im_self; args[0] = Py_NewRef(self); - method = ((PyMethodObject *)callable)->im_func; + PyObject *method = ((PyMethodObject *)callable)->im_func; args[-1] = Py_NewRef(method); Py_DECREF(callable); callable = method; @@ -3716,7 +3708,7 @@ kwnames); if (opcode == INSTRUMENTED_CALL) { PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PEEK(total_args); + &_PyInstrumentation_MISSING : args[0]; if (res == NULL) { _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, @@ -3747,17 +3739,17 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + PyObject *null; PyObject *callable; - PyObject *method; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; - DEOPT_IF(method != NULL, CALL); + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + DEOPT_IF(null != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); PyObject *self = ((PyMethodObject *)callable)->im_self; - PEEK(oparg + 1) = Py_NewRef(self); // callable + PEEK(oparg + 1) = Py_NewRef(self); // self_or_null PyObject *meth = ((PyMethodObject *)callable)->im_func; - PEEK(oparg + 2) = Py_NewRef(meth); // method + PEEK(oparg + 2) = Py_NewRef(meth); // callable Py_DECREF(callable); GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); STACK_SHRINK(oparg); @@ -3767,18 +3759,16 @@ TARGET(CALL_PY_EXACT_ARGS) { PREDICTED(CALL_PY_EXACT_ARGS); PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&next_instr[1].cache); ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); - int is_meth = method != NULL; int argcount = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; argcount++; } @@ -3804,18 +3794,16 @@ TARGET(CALL_PY_WITH_DEFAULTS) { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&next_instr[1].cache); ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); - int is_meth = method != NULL; int argcount = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; argcount++; } @@ -3851,12 +3839,12 @@ TARGET(CALL_NO_KW_TYPE_1) { PyObject **args; - PyObject *callable; PyObject *null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - null = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3875,12 +3863,12 @@ TARGET(CALL_NO_KW_STR_1) { PyObject **args; - PyObject *callable; PyObject *null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - null = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3901,12 +3889,12 @@ TARGET(CALL_NO_KW_TUPLE_1) { PyObject **args; - PyObject *callable; PyObject *null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - null = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3927,11 +3915,11 @@ TARGET(CALL_NO_KW_ALLOC_AND_ENTER_INIT) { PyObject **args; - PyObject *callable; PyObject *null; + PyObject *callable; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - null = stack_pointer[-2 - oparg]; + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; /* This instruction does the following: * 1. Creates the object (by calling ``object.__new__``) * 2. Pushes a shim frame to the frame stack (to cleanup after ``__init__``) @@ -4002,16 +3990,14 @@ TARGET(CALL_BUILTIN_CLASS) { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; - int is_meth = method != NULL; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -4039,18 +4025,16 @@ TARGET(CALL_NO_KW_BUILTIN_O) { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; /* Builtin METH_O functions */ ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -4082,18 +4066,16 @@ TARGET(CALL_NO_KW_BUILTIN_FAST) { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL functions, without keywords */ ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -4129,17 +4111,15 @@ TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -4176,18 +4156,16 @@ TARGET(CALL_NO_KW_LEN) { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* len(o) */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -4215,18 +4193,16 @@ TARGET(CALL_NO_KW_ISINSTANCE) { PyObject **args; + PyObject *self_or_null; PyObject *callable; - PyObject *method; PyObject *res; args = stack_pointer - oparg; - callable = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); /* isinstance(o, o2) */ - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { - callable = method; + if (self_or_null != NULL) { args--; total_args++; } @@ -4257,22 +4233,22 @@ TARGET(CALL_NO_KW_LIST_APPEND) { PyObject **args; PyObject *self; - PyObject *method; + PyObject *callable; args = stack_pointer - oparg; self = stack_pointer[-1 - oparg]; - method = stack_pointer[-2 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 1); - assert(method != NULL); + assert(self != NULL); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(method != interp->callable_cache.list_append, CALL); + DEOPT_IF(callable != interp->callable_cache.list_append, CALL); DEOPT_IF(!PyList_Check(self), CALL); STAT_INC(CALL, hit); if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { goto pop_1_error; // Since arg is DECREF'ed already } Py_DECREF(self); - Py_DECREF(method); + Py_DECREF(callable); STACK_SHRINK(3); // CALL + POP_TOP SKIP_OVER(INLINE_CACHE_ENTRIES_CALL + 1); @@ -4284,26 +4260,26 @@ TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args; - PyObject *method; + PyObject *self_or_null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_O, CALL); PyObject *arg = args[1]; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -4328,22 +4304,22 @@ TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { PyObject **args; - PyObject *method; + PyObject *self_or_null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - method = stack_pointer[-2 - oparg]; - int is_meth = method != NULL; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); - PyTypeObject *d_type = callable->d_common.d_type; + PyTypeObject *d_type = method->d_common.d_type; PyObject *self = args[0]; DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); STAT_INC(CALL, hit); @@ -4370,24 +4346,25 @@ TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { PyObject **args; - PyObject *method; + PyObject *self_or_null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); assert(oparg == 0 || oparg == 1); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } DEOPT_IF(total_args != 1, CALL); - PyMethodDescrObject *callable = (PyMethodDescrObject *)SECOND(); - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -4412,25 +4389,25 @@ TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { PyObject **args; - PyObject *method; + PyObject *self_or_null; + PyObject *callable; PyObject *res; args = stack_pointer - oparg; - method = stack_pointer[-2 - oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; ASSERT_KWNAMES_IS_NULL(); - int is_meth = method != NULL; int total_args = oparg; - if (is_meth) { + if (self_or_null != NULL) { args--; total_args++; } - PyMethodDescrObject *callable = - (PyMethodDescrObject *)PEEK(total_args + 1); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL); - PyMethodDef *meth = callable->d_method; + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + PyMethodDef *meth = method->d_method; DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, callable->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; @@ -4463,7 +4440,7 @@ PyObject *result; if (oparg & 1) { kwargs = stack_pointer[-(oparg & 1 ? 1 : 0)]; } callargs = stack_pointer[-1 - (oparg & 1 ? 1 : 0)]; - func = stack_pointer[-2 - (oparg & 1 ? 1 : 0)]; + func = stack_pointer[-3 - (oparg & 1 ? 1 : 0)]; // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. assert(kwargs == NULL || PyDict_CheckExact(kwargs)); @@ -4528,7 +4505,7 @@ Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - assert(PEEK(3 + (oparg & 1)) == NULL); + assert(PEEK(2 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); diff --git a/Python/specialize.c b/Python/specialize.c index 855252e066dea..98455ae3efc60 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -793,6 +793,10 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) if (!function_check_args(fget, 1, LOAD_ATTR)) { goto fail; } + if (instr->op.arg & 1) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); + goto fail; + } uint32_t version = function_get_version(fget, LOAD_ATTR); if (version == 0) { goto fail; @@ -859,6 +863,10 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) if (!function_check_args(descr, 2, LOAD_ATTR)) { goto fail; } + if (instr->op.arg & 1) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD); + goto fail; + } uint32_t version = function_get_version(descr, LOAD_ATTR); if (version == 0) { goto fail; From webhook-mailer at python.org Wed Aug 9 15:14:55 2023 From: webhook-mailer at python.org (brandtbucher) Date: Wed, 09 Aug 2023 19:14:55 -0000 Subject: [Python-checkins] GH-106485: Dematerialize instance dictionaries when possible (GH-106539) Message-ID: https://github.com/python/cpython/commit/326f0ba1c5dda1d9613dbba11ea2470654b0d9c8 commit: 326f0ba1c5dda1d9613dbba11ea2470654b0d9c8 branch: main author: Brandt Bucher committer: brandtbucher date: 2023-08-09T19:14:50Z summary: GH-106485: Dematerialize instance dictionaries when possible (GH-106539) files: A Misc/NEWS.d/next/Core and Builtins/2023-07-16-07-55-19.gh-issue-106485.wPb1bH.rst M Include/internal/pycore_dict.h M Include/pystats.h M Lib/test/test_opcache.py M Objects/dictobject.c M Objects/typeobject.c M Python/bytecodes.c M Python/executor_cases.c.h M Python/generated_cases.c.h M Python/specialize.c diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 26a913435ef9c..6ae81c033c43a 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -10,6 +10,7 @@ extern "C" { #endif #include "pycore_dict_state.h" +#include "pycore_object.h" #include "pycore_runtime.h" // _PyRuntime // Unsafe flavor of PyDict_GetItemWithError(): no error checking @@ -62,6 +63,8 @@ extern uint32_t _PyDictKeys_GetVersionForCurrentState( extern size_t _PyDict_KeysSize(PyDictKeysObject *keys); +extern void _PyDictKeys_DecRef(PyDictKeysObject *keys); + /* _Py_dict_lookup() returns index of entry which can be used like DK_ENTRIES(dk)[index]. * -1 when no entry found, -3 when compare raises error. */ @@ -196,6 +199,7 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, } extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); +extern int _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); extern PyObject *_PyDict_FromItems( PyObject *const *keys, Py_ssize_t keys_offset, PyObject *const *values, Py_ssize_t values_offset, diff --git a/Include/pystats.h b/Include/pystats.h index e24aef5fe8072..b1957596745f0 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -65,6 +65,7 @@ typedef struct _object_stats { uint64_t dict_materialized_new_key; uint64_t dict_materialized_too_big; uint64_t dict_materialized_str_subclass; + uint64_t dict_dematerialized; uint64_t type_cache_hits; uint64_t type_cache_misses; uint64_t type_cache_dunder_hits; diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 564dc4745ae64..7d317c012a304 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -865,8 +865,10 @@ class C: items = [] for _ in range(self.ITEMS): item = C() - item.__dict__ item.a = None + # Resize into a combined unicode dict: + for i in range(29): + setattr(item, f"_{i}", None) items.append(item) return items @@ -932,7 +934,9 @@ class C: items = [] for _ in range(self.ITEMS): item = C() - item.__dict__ + # Resize into a combined unicode dict: + for i in range(29): + setattr(item, f"_{i}", None) items.append(item) return items diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-16-07-55-19.gh-issue-106485.wPb1bH.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-16-07-55-19.gh-issue-106485.wPb1bH.rst new file mode 100644 index 0000000000000..1f80082821eda --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-16-07-55-19.gh-issue-106485.wPb1bH.rst @@ -0,0 +1,2 @@ +Reduce the number of materialized instances dictionaries by dematerializing +them when possible. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 41ae1fca90b46..931103cdc1a06 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5464,6 +5464,35 @@ _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) return make_dict_from_instance_attributes(interp, keys, values); } +// Return 1 if the dict was dematerialized, 0 otherwise. +int +_PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) +{ + assert(_PyObject_DictOrValuesPointer(obj) == dorv); + assert(!_PyDictOrValues_IsValues(*dorv)); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); + if (dict == NULL) { + return 0; + } + // It's likely that this dict still shares its keys (if it was materialized + // on request and not heavily modified): + assert(PyDict_CheckExact(dict)); + assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE)); + if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || Py_REFCNT(dict) != 1) { + return 0; + } + assert(dict->ma_values); + // We have an opportunity to do something *really* cool: dematerialize it! + _PyDictKeys_DecRef(dict->ma_keys); + _PyDictOrValues_SetValues(dorv, dict->ma_values); + OBJECT_STAT_INC(dict_dematerialized); + // Don't try this at home, kids: + dict->ma_keys = NULL; + dict->ma_values = NULL; + Py_DECREF(dict); + return 1; +} + int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value) @@ -5688,6 +5717,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) dict = _PyDictOrValues_GetDict(*dorv_ptr); if (dict == NULL) { dictkeys_incref(CACHED_KEYS(tp)); + OBJECT_STAT_INC(dict_materialized_on_request); dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); dorv_ptr->dict = dict; } @@ -5731,6 +5761,9 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, dict = *dictptr; if (dict == NULL) { dictkeys_incref(cached); + if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { + OBJECT_STAT_INC(dict_materialized_on_request); + } dict = new_dict_with_shared_keys(interp, cached); if (dict == NULL) return -1; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 76809494dd881..71e96f5af4e41 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4965,9 +4965,6 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value) return res; } -extern void -_PyDictKeys_DecRef(PyDictKeysObject *keys); - static void type_dealloc_common(PyTypeObject *type) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b2281abc6663d..5efa36fcf5c62 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1827,8 +1827,10 @@ dummy_func( op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); } op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { @@ -2727,8 +2729,10 @@ dummy_func( assert(type_version != 0); DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); @@ -2757,8 +2761,10 @@ dummy_func( assert(type_version != 0); DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d6d541a3b61ab..a7b5054417ed8 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1623,8 +1623,10 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index cf20b869b8182..ccf43c727b9e0 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2366,8 +2366,10 @@ { assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { @@ -3507,8 +3509,10 @@ assert(type_version != 0); DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); @@ -3559,8 +3563,10 @@ assert(type_version != 0); DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), + LOAD_ATTR); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); diff --git a/Python/specialize.c b/Python/specialize.c index 98455ae3efc60..2d514c0dc476d 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -192,6 +192,7 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key); fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big); fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); + fprintf(out, "Object dematerialize dict: %" PRIu64 "\n", stats->dict_dematerialized); fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); @@ -685,8 +686,10 @@ specialize_dict_access( return 0; } _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - if (_PyDictOrValues_IsValues(dorv)) { + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); + if (_PyDictOrValues_IsValues(*dorv) || + _PyObject_MakeInstanceAttributesFromDict(owner, dorv)) + { // Virtual dictionary PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; assert(PyUnicode_CheckExact(name)); @@ -704,12 +707,16 @@ specialize_dict_access( instr->op.code = values_op; } else { - PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); + PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); if (dict == NULL || !PyDict_CheckExact(dict)) { SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); return 0; } // We found an instance with a __dict__. + if (dict->ma_values) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT); + return 0; + } Py_ssize_t index = _PyDict_LookupIndex(dict, name); if (index != (uint16_t)index) { @@ -1100,9 +1107,11 @@ PyObject *descr, DescriptorClassification kind, bool is_method) assert(descr != NULL); assert((is_method && kind == METHOD) || (!is_method && kind == NON_DESCRIPTOR)); if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; - if (!_PyDictOrValues_IsValues(dorv)) { + if (!_PyDictOrValues_IsValues(*dorv) && + !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) + { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); return 0; } From webhook-mailer at python.org Wed Aug 9 18:42:20 2023 From: webhook-mailer at python.org (zooba) Date: Wed, 09 Aug 2023 22:42:20 -0000 Subject: [Python-checkins] gh-107814: Avoid output from Nuget installation in find_python.bat (GH-107815) Message-ID: https://github.com/python/cpython/commit/1e229e2c3d212accbd5fbe3a46cd42f8252b2868 commit: 1e229e2c3d212accbd5fbe3a46cd42f8252b2868 branch: main author: Max Bachmann committer: zooba date: 2023-08-09T23:42:16+01:00 summary: gh-107814: Avoid output from Nuget installation in find_python.bat (GH-107815) files: A Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst M PCbuild/find_python.bat diff --git a/Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst b/Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst new file mode 100644 index 0000000000000..d3723353470ce --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst @@ -0,0 +1 @@ +When calling ``find_python.bat`` with ``-q`` it did not properly silence the output of nuget. That is now fixed. diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index 7af5503d80a0f..d3f62c9386900 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -52,7 +52,7 @@ @if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe) @if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl) @if NOT exist "%_Py_NUGET%" ( - @echo Downloading nuget... + @if not "%_Py_Quiet%"=="1" @echo Downloading nuget... @rem NB: Must use single quotes around NUGET here, NOT double! @rem Otherwise, a space in the path would break things @rem If it fails, retry with any available copy of Python @@ -63,7 +63,11 @@ ) @if not "%_Py_Quiet%"=="1" @echo Installing Python via nuget... -@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" + at if not "%_Py_Quiet%"=="1" ( + @"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" +) else ( + @"%_Py_NUGET%" install pythonx86 -Verbosity quiet -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" +) @rem Quote it here; it's not quoted later because "py -x.y" wouldn't work @if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found From webhook-mailer at python.org Wed Aug 9 18:42:35 2023 From: webhook-mailer at python.org (carljm) Date: Wed, 09 Aug 2023 22:42:35 -0000 Subject: [Python-checkins] gh-91054: make code watcher tests resilient to other watchers (#107821) Message-ID: https://github.com/python/cpython/commit/2ec16fed14aae896e38dd5bd9e73e2eddc974439 commit: 2ec16fed14aae896e38dd5bd9e73e2eddc974439 branch: main author: Carl Meyer committer: carljm date: 2023-08-09T16:42:32-06:00 summary: gh-91054: make code watcher tests resilient to other watchers (#107821) files: M Modules/_testcapi/watchers.c M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 4cf567b331498..8a264bba4ed6e 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -295,6 +295,7 @@ _testcapi_unwatch_type_impl(PyObject *module, int watcher_id, PyObject *type) // Test code object watching #define NUM_CODE_WATCHERS 2 +static int code_watcher_ids[NUM_CODE_WATCHERS] = {-1, -1}; static int num_code_object_created_events[NUM_CODE_WATCHERS] = {0, 0}; static int num_code_object_destroyed_events[NUM_CODE_WATCHERS] = {0, 0}; @@ -345,11 +346,13 @@ add_code_watcher(PyObject *self, PyObject *which_watcher) long which_l = PyLong_AsLong(which_watcher); if (which_l == 0) { watcher_id = PyCode_AddWatcher(first_code_object_callback); + code_watcher_ids[0] = watcher_id; num_code_object_created_events[0] = 0; num_code_object_destroyed_events[0] = 0; } else if (which_l == 1) { watcher_id = PyCode_AddWatcher(second_code_object_callback); + code_watcher_ids[1] = watcher_id; num_code_object_created_events[1] = 0; num_code_object_destroyed_events[1] = 0; } @@ -375,9 +378,14 @@ clear_code_watcher(PyObject *self, PyObject *watcher_id) return NULL; } // reset static events counters - if (watcher_id_l >= 0 && watcher_id_l < NUM_CODE_WATCHERS) { - num_code_object_created_events[watcher_id_l] = 0; - num_code_object_destroyed_events[watcher_id_l] = 0; + if (watcher_id_l >= 0) { + for (int i = 0; i < NUM_CODE_WATCHERS; i++) { + if (watcher_id_l == code_watcher_ids[i]) { + code_watcher_ids[i] = -1; + num_code_object_created_events[i] = 0; + num_code_object_destroyed_events[i] = 0; + } + } } Py_RETURN_NONE; } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 099f20b5a1b43..66815c72ffbc6 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -429,6 +429,7 @@ Modules/_testcapi/watchers.c - g_dict_watch_events - Modules/_testcapi/watchers.c - g_dict_watchers_installed - Modules/_testcapi/watchers.c - g_type_modified_events - Modules/_testcapi/watchers.c - g_type_watchers_installed - +Modules/_testcapi/watchers.c - code_watcher_ids - Modules/_testcapi/watchers.c - num_code_object_created_events - Modules/_testcapi/watchers.c - num_code_object_destroyed_events - Modules/_testcapi/watchers.c - pyfunc_watchers - From webhook-mailer at python.org Wed Aug 9 19:15:47 2023 From: webhook-mailer at python.org (zooba) Date: Wed, 09 Aug 2023 23:15:47 -0000 Subject: [Python-checkins] [3.11] gh-107814: Avoid output from Nuget installation in find_python.bat (GH-107815) Message-ID: https://github.com/python/cpython/commit/eff2042fac6dc7d6e2f8aeec5a64a3a44a1a9648 commit: eff2042fac6dc7d6e2f8aeec5a64a3a44a1a9648 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: zooba date: 2023-08-09T23:15:43Z summary: [3.11] gh-107814: Avoid output from Nuget installation in find_python.bat (GH-107815) gh-107814: Avoid output from Nuget installation in find_python.bat (GH-107815) (cherry picked from commit 1e229e2c3d212accbd5fbe3a46cd42f8252b2868) Co-authored-by: Max Bachmann files: A Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst M PCbuild/find_python.bat diff --git a/Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst b/Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst new file mode 100644 index 0000000000000..d3723353470ce --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst @@ -0,0 +1 @@ +When calling ``find_python.bat`` with ``-q`` it did not properly silence the output of nuget. That is now fixed. diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index 11d6cba7a172c..31579e088e470 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -52,7 +52,7 @@ @if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe) @if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl) @if NOT exist "%_Py_NUGET%" ( - @echo Downloading nuget... + @if not "%_Py_Quiet%"=="1" @echo Downloading nuget... @rem NB: Must use single quotes around NUGET here, NOT double! @rem Otherwise, a space in the path would break things @rem If it fails, retry with any available copy of Python @@ -63,7 +63,11 @@ ) @if not "%_Py_Quiet%"=="1" @echo Installing Python via nuget... -@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" + at if not "%_Py_Quiet%"=="1" ( + @"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" +) else ( + @"%_Py_NUGET%" install pythonx86 -Verbosity quiet -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" +) @rem Quote it here; it's not quoted later because "py -x.y" wouldn't work @if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found From webhook-mailer at python.org Wed Aug 9 20:47:50 2023 From: webhook-mailer at python.org (corona10) Date: Thu, 10 Aug 2023 00:47:50 -0000 Subject: [Python-checkins] GH-107812: extend `socket`'s netlink support to FreeBSD (gh-107813) Message-ID: https://github.com/python/cpython/commit/f50c17243a87b02086000185f6ed1cad4b8c2376 commit: f50c17243a87b02086000185f6ed1cad4b8c2376 branch: main author: Mina Gali? committer: corona10 date: 2023-08-10T00:47:46Z summary: GH-107812: extend `socket`'s netlink support to FreeBSD (gh-107813) files: A Misc/NEWS.d/next/Library/2023-08-09-15-37-20.gh-issue-107812.CflAXa.rst M Modules/socketmodule.h M configure M configure.ac M pyconfig.h.in diff --git a/Misc/NEWS.d/next/Library/2023-08-09-15-37-20.gh-issue-107812.CflAXa.rst b/Misc/NEWS.d/next/Library/2023-08-09-15-37-20.gh-issue-107812.CflAXa.rst new file mode 100644 index 0000000000000..0aac44fb41883 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-09-15-37-20.gh-issue-107812.CflAXa.rst @@ -0,0 +1 @@ +Extend socket's netlink support to the FreeBSD platform. diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 663ae3d6e0dd6..47146a28e02c8 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -100,6 +100,8 @@ typedef int socklen_t; # include # endif # include +#elif defined(HAVE_NETLINK_NETLINK_H) +# include #else # undef AF_NETLINK #endif diff --git a/configure b/configure index 80b4a001c6d6d..aaacf8d2669c1 100755 --- a/configure +++ b/configure @@ -11153,6 +11153,7 @@ fi # On Linux, netlink.h requires asm/types.h +# On FreeBSD, netlink.h is located in netlink/netlink.h ac_fn_c_check_header_compile "$LINENO" "linux/netlink.h" "ac_cv_header_linux_netlink_h" " #ifdef HAVE_ASM_TYPES_H #include @@ -11167,6 +11168,20 @@ then : printf "%s\n" "#define HAVE_LINUX_NETLINK_H 1" >>confdefs.h fi +ac_fn_c_check_header_compile "$LINENO" "netlink/netlink.h" "ac_cv_header_netlink_netlink_h" " +#ifdef HAVE_ASM_TYPES_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif + +" +if test "x$ac_cv_header_netlink_netlink_h" = xyes +then : + printf "%s\n" "#define HAVE_NETLINK_NETLINK_H 1" >>confdefs.h + +fi # On Linux, qrtr.h requires asm/types.h diff --git a/configure.ac b/configure.ac index 8a84eaf6b370a..ddf6da0b9da12 100644 --- a/configure.ac +++ b/configure.ac @@ -2880,7 +2880,8 @@ AC_CHECK_HEADERS([net/if.h], [], [], ]) # On Linux, netlink.h requires asm/types.h -AC_CHECK_HEADERS([linux/netlink.h], [], [], [ +# On FreeBSD, netlink.h is located in netlink/netlink.h +AC_CHECK_HEADERS([linux/netlink.h netlink/netlink.h], [], [], [ #ifdef HAVE_ASM_TYPES_H #include #endif diff --git a/pyconfig.h.in b/pyconfig.h.in index dab8ebf64371f..181dc3d7d1137 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -841,6 +841,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_NETINET_IN_H +/* Define to 1 if you have the header file. */ +#undef HAVE_NETLINK_NETLINK_H + /* Define to 1 if you have the header file. */ #undef HAVE_NETPACKET_PACKET_H From webhook-mailer at python.org Wed Aug 9 21:05:55 2023 From: webhook-mailer at python.org (gvanrossum) Date: Thu, 10 Aug 2023 01:05:55 -0000 Subject: [Python-checkins] Update README for the cases generator (#107826) Message-ID: https://github.com/python/cpython/commit/4890bfe1f906202ef521ffd327cae36e1afa0873 commit: 4890bfe1f906202ef521ffd327cae36e1afa0873 branch: main author: Guido van Rossum committer: gvanrossum date: 2023-08-10T01:05:51Z summary: Update README for the cases generator (#107826) files: M Tools/cases_generator/README.md diff --git a/Tools/cases_generator/README.md b/Tools/cases_generator/README.md index fc9331656fe78..ed802e44f31ad 100644 --- a/Tools/cases_generator/README.md +++ b/Tools/cases_generator/README.md @@ -7,10 +7,14 @@ What's currently here: - `lexer.py`: lexer for C, originally written by Mark Shannon - `plexer.py`: OO interface on top of lexer.py; main class: `PLexer` -- `parser.py`: Parser for instruction definition DSL; main class `Parser` +- `parsing.py`: Parser for instruction definition DSL; main class `Parser` - `generate_cases.py`: driver script to read `Python/bytecodes.c` and write `Python/generated_cases.c.h` (and several other files) -- `test_generator.py`: tests, require manual running using `pytest` +- `analysis.py`: `Analyzer` class used to read the input files +- `flags.py`: abstractions related to metadata flags for instructions +- `formatting.py`: `Formatter` class used to write the output files +- `instructions.py`: classes to analyze and write instructions +- `stacking.py`: code to handle generalized stack effects Note that there is some dummy C code at the top and bottom of `Python/bytecodes.c` From webhook-mailer at python.org Thu Aug 10 02:39:19 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 10 Aug 2023 06:39:19 -0000 Subject: [Python-checkins] gh-107689: Add docstring to `ctypes.Array` (#107697) Message-ID: https://github.com/python/cpython/commit/0f2fb6efb4d5d8ca43a0e779b89a8e805880e09a commit: 0f2fb6efb4d5d8ca43a0e779b89a8e805880e09a branch: main author: Kostya Farber <73378227+kostyafarber at users.noreply.github.com> committer: kumaraditya303 date: 2023-08-10T06:39:14Z summary: gh-107689: Add docstring to `ctypes.Array` (#107697) files: M Modules/_ctypes/_ctypes.c diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 9aee37a9d954e..dc80291d3b810 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -4793,6 +4793,16 @@ static PyMappingMethods Array_as_mapping = { Array_ass_subscript, }; +PyDoc_STRVAR(array_doc, +"Abstract base class for arrays.\n" +"\n" +"The recommended way to create concrete array types is by multiplying any\n" +"ctypes data type with a non-negative integer. Alternatively, you can subclass\n" +"this type and define _length_ and _type_ class variables. Array elements can\n" +"be read and written using standard subscript and slice accesses for slice\n" +"reads, the resulting object is not itself an Array." +); + PyTypeObject PyCArray_Type = { PyVarObject_HEAD_INIT(NULL, 0) "_ctypes.Array", @@ -4813,8 +4823,8 @@ PyTypeObject PyCArray_Type = { 0, /* tp_getattro */ 0, /* tp_setattro */ &PyCData_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("XXX to be provided"), /* tp_doc */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + array_doc, /* tp_doc */ (traverseproc)PyCData_traverse, /* tp_traverse */ (inquiry)PyCData_clear, /* tp_clear */ 0, /* tp_richcompare */ From webhook-mailer at python.org Thu Aug 10 02:55:53 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 10 Aug 2023 06:55:53 -0000 Subject: [Python-checkins] gh-107409: set `__wrapped__` attribute in `reprlib.recursive_repr` (#107410) Message-ID: https://github.com/python/cpython/commit/4845b9712f2c187743344eca43fa1fb896bddfd6 commit: 4845b9712f2c187743344eca43fa1fb896bddfd6 branch: main author: denballakh <47365157+denballakh at users.noreply.github.com> committer: kumaraditya303 date: 2023-08-10T06:55:49Z summary: gh-107409: set `__wrapped__` attribute in `reprlib.recursive_repr` (#107410) Co-authored-by: Kumar Aditya files: A Misc/NEWS.d/next/Library/2023-07-29-02-36-50.gh-issue-107409.HG27Nu.rst M Lib/reprlib.py M Lib/test/test_reprlib.py diff --git a/Lib/reprlib.py b/Lib/reprlib.py index a92b3e3dbb613..840dd0e20132b 100644 --- a/Lib/reprlib.py +++ b/Lib/reprlib.py @@ -29,6 +29,7 @@ def wrapper(self): wrapper.__name__ = getattr(user_function, '__name__') wrapper.__qualname__ = getattr(user_function, '__qualname__') wrapper.__annotations__ = getattr(user_function, '__annotations__', {}) + wrapper.__wrapped__ = user_function return wrapper return decorating_function diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index e7216d427200c..502287b620d06 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -765,5 +765,14 @@ def test_assigned_attributes(self): for name in assigned: self.assertIs(getattr(wrapper, name), getattr(wrapped, name)) + def test__wrapped__(self): + class X: + def __repr__(self): + return 'X()' + f = __repr__ # save reference to check it later + __repr__ = recursive_repr()(__repr__) + + self.assertIs(X.f, X.__repr__.__wrapped__) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-07-29-02-36-50.gh-issue-107409.HG27Nu.rst b/Misc/NEWS.d/next/Library/2023-07-29-02-36-50.gh-issue-107409.HG27Nu.rst new file mode 100644 index 0000000000000..1ecc7207605c7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-29-02-36-50.gh-issue-107409.HG27Nu.rst @@ -0,0 +1 @@ +Set :attr:`!__wrapped__` attribute in :func:`reprlib.recursive_repr`. From webhook-mailer at python.org Thu Aug 10 03:19:09 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Thu, 10 Aug 2023 07:19:09 -0000 Subject: [Python-checkins] gh-95065: Argument Clinic: Add functional tests of deprecated positionals (#107768) Message-ID: https://github.com/python/cpython/commit/39ef93edb9802dccdb6555d4209ac2e60875a011 commit: 39ef93edb9802dccdb6555d4209ac2e60875a011 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-10T07:19:05Z summary: gh-95065: Argument Clinic: Add functional tests of deprecated positionals (#107768) Move the "deprecated positinal" tests from clinic.test.c to _testclinic.c. Mock PY_VERSION_HEX in order to prevent generated compiler warnings/errors to trigger. Put clinic code for deprecated positionals in Modules/clinic/_testclinic_depr_star.c.h for easy inspection of the generated code. Co-authored-by: Alex Waygood Co-authored-by: Serhiy Storchaka files: A Modules/clinic/_testclinic_depr_star.c.h M Lib/test/clinic.test.c M Lib/test/test_clinic.py M Modules/_testclinic.c M Tools/c-analyzer/cpython/globals-to-fix.tsv diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index c7063e1139b0a..019dc10073e98 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5467,1164 +5467,3 @@ docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py static PyObject * docstr_fallback_to_converter_default_impl(PyObject *module, str a) /*[clinic end generated code: output=ae24a9c6f60ee8a6 input=0cbe6a4d24bc2274]*/ - - -/*[clinic input] -test_deprecate_positional_pos1_len1_optional - a: object - * [from 3.14] - b: object = None -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos1_len1_optional__doc__, -"test_deprecate_positional_pos1_len1_optional($module, /, a, b=None)\n" -"--\n" -"\n" -"Note: Passing 2 positional arguments to\n" -"test_deprecate_positional_pos1_len1_optional() is deprecated.\n" -"Parameter \'b\' will become a keyword-only parameter in Python 3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS1_LEN1_OPTIONAL_METHODDEF \ - {"test_deprecate_positional_pos1_len1_optional", _PyCFunction_CAST(test_deprecate_positional_pos1_len1_optional), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len1_optional__doc__}, - -static PyObject * -test_deprecate_positional_pos1_len1_optional_impl(PyObject *module, - PyObject *a, PyObject *b); - -static PyObject * -test_deprecate_positional_pos1_len1_optional(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos1_len1_optional", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - PyObject *a; - PyObject *b = Py_None; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ - " 'test_deprecate_positional_pos1_len1_optional' to be " \ - "keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ - " 'test_deprecate_positional_pos1_len1_optional' to be " \ - "keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ - " 'test_deprecate_positional_pos1_len1_optional' to be " \ - "keyword-only." - # endif - #endif - if (nargs == 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing 2 positional arguments to " - "test_deprecate_positional_pos1_len1_optional() is deprecated. " - "Parameter 'b' will become a keyword-only parameter in Python " - "3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - if (!noptargs) { - goto skip_optional_pos; - } - b = args[1]; -skip_optional_pos: - return_value = test_deprecate_positional_pos1_len1_optional_impl(module, a, b); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos1_len1_optional_impl(PyObject *module, - PyObject *a, PyObject *b) -/*[clinic end generated code: output=144cbf1adc574dd9 input=89099f3dacd757da]*/ - - -/*[clinic input] -test_deprecate_positional_pos1_len1 - a: object - * [from 3.14] - b: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos1_len1__doc__, -"test_deprecate_positional_pos1_len1($module, /, a, b)\n" -"--\n" -"\n" -"Note: Passing 2 positional arguments to\n" -"test_deprecate_positional_pos1_len1() is deprecated. Parameter \'b\'\n" -"will become a keyword-only parameter in Python 3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS1_LEN1_METHODDEF \ - {"test_deprecate_positional_pos1_len1", _PyCFunction_CAST(test_deprecate_positional_pos1_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len1__doc__}, - -static PyObject * -test_deprecate_positional_pos1_len1_impl(PyObject *module, PyObject *a, - PyObject *b); - -static PyObject * -test_deprecate_positional_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos1_len1", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - PyObject *a; - PyObject *b; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ - " 'test_deprecate_positional_pos1_len1' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ - " 'test_deprecate_positional_pos1_len1' to be keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'b' in the clinic input of" \ - " 'test_deprecate_positional_pos1_len1' to be keyword-only." - # endif - #endif - if (nargs == 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing 2 positional arguments to " - "test_deprecate_positional_pos1_len1() is deprecated. Parameter " - "'b' will become a keyword-only parameter in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - return_value = test_deprecate_positional_pos1_len1_impl(module, a, b); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos1_len1_impl(PyObject *module, PyObject *a, - PyObject *b) -/*[clinic end generated code: output=994bd57c1c634709 input=1702bbab1e9b3b99]*/ - - -/*[clinic input] -test_deprecate_positional_pos1_len2_with_kwd - a: object - * [from 3.14] - b: object - c: object - * - d: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos1_len2_with_kwd__doc__, -"test_deprecate_positional_pos1_len2_with_kwd($module, /, a, b, c, *, d)\n" -"--\n" -"\n" -"Note: Passing more than 1 positional argument to\n" -"test_deprecate_positional_pos1_len2_with_kwd() is deprecated.\n" -"Parameters \'b\' and \'c\' will become keyword-only parameters in Python\n" -"3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS1_LEN2_WITH_KWD_METHODDEF \ - {"test_deprecate_positional_pos1_len2_with_kwd", _PyCFunction_CAST(test_deprecate_positional_pos1_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos1_len2_with_kwd__doc__}, - -static PyObject * -test_deprecate_positional_pos1_len2_with_kwd_impl(PyObject *module, - PyObject *a, PyObject *b, - PyObject *c, PyObject *d); - -static PyObject * -test_deprecate_positional_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 4 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos1_len2_with_kwd", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[4]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *d; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic " \ - "input of 'test_deprecate_positional_pos1_len2_with_kwd' to be " \ - "keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic " \ - "input of 'test_deprecate_positional_pos1_len2_with_kwd' to be " \ - "keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'b' and 'c' in the clinic " \ - "input of 'test_deprecate_positional_pos1_len2_with_kwd' to be " \ - "keyword-only." - # endif - #endif - if (nargs > 1 && nargs <= 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 1 positional argument to " - "test_deprecate_positional_pos1_len2_with_kwd() is deprecated. " - "Parameters 'b' and 'c' will become keyword-only parameters in " - "Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - d = args[3]; - return_value = test_deprecate_positional_pos1_len2_with_kwd_impl(module, a, b, c, d); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos1_len2_with_kwd_impl(PyObject *module, - PyObject *a, PyObject *b, - PyObject *c, PyObject *d) -/*[clinic end generated code: output=146c60ecbcdbf4b8 input=28cdb885f6c34eab]*/ - - -/*[clinic input] -test_deprecate_positional_pos0_len1 - * [from 3.14] - a: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos0_len1__doc__, -"test_deprecate_positional_pos0_len1($module, /, a)\n" -"--\n" -"\n" -"Note: Passing positional arguments to\n" -"test_deprecate_positional_pos0_len1() is deprecated. Parameter \'a\'\n" -"will become a keyword-only parameter in Python 3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS0_LEN1_METHODDEF \ - {"test_deprecate_positional_pos0_len1", _PyCFunction_CAST(test_deprecate_positional_pos0_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len1__doc__}, - -static PyObject * -test_deprecate_positional_pos0_len1_impl(PyObject *module, PyObject *a); - -static PyObject * -test_deprecate_positional_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos0_len1", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'test_deprecate_positional_pos0_len1' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'test_deprecate_positional_pos0_len1' to be keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'test_deprecate_positional_pos0_len1' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to " - "test_deprecate_positional_pos0_len1() is deprecated. Parameter " - "'a' will become a keyword-only parameter in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - return_value = test_deprecate_positional_pos0_len1_impl(module, a); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos0_len1_impl(PyObject *module, PyObject *a) -/*[clinic end generated code: output=dce99971a2494f9f input=678206db25c0652c]*/ - - -/*[clinic input] -test_deprecate_positional_pos0_len2 - * [from 3.14] - a: object - b: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos0_len2__doc__, -"test_deprecate_positional_pos0_len2($module, /, a, b)\n" -"--\n" -"\n" -"Note: Passing positional arguments to\n" -"test_deprecate_positional_pos0_len2() is deprecated. Parameters \'a\'\n" -"and \'b\' will become keyword-only parameters in Python 3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS0_LEN2_METHODDEF \ - {"test_deprecate_positional_pos0_len2", _PyCFunction_CAST(test_deprecate_positional_pos0_len2), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len2__doc__}, - -static PyObject * -test_deprecate_positional_pos0_len2_impl(PyObject *module, PyObject *a, - PyObject *b); - -static PyObject * -test_deprecate_positional_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos0_len2", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - PyObject *a; - PyObject *b; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic " \ - "input of 'test_deprecate_positional_pos0_len2' to be " \ - "keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic " \ - "input of 'test_deprecate_positional_pos0_len2' to be " \ - "keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'a' and 'b' in the clinic " \ - "input of 'test_deprecate_positional_pos0_len2' to be " \ - "keyword-only." - # endif - #endif - if (nargs > 0 && nargs <= 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to " - "test_deprecate_positional_pos0_len2() is deprecated. Parameters " - "'a' and 'b' will become keyword-only parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - return_value = test_deprecate_positional_pos0_len2_impl(module, a, b); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos0_len2_impl(PyObject *module, PyObject *a, - PyObject *b) -/*[clinic end generated code: output=06999692e0c8dac4 input=fae0d0b1d480c939]*/ - - -/*[clinic input] -test_deprecate_positional_pos0_len3_with_kwdonly - * [from 3.14] - a: object - b: object - c: object - * - e: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos0_len3_with_kwdonly__doc__, -"test_deprecate_positional_pos0_len3_with_kwdonly($module, /, a, b, c,\n" -" *, e)\n" -"--\n" -"\n" -"Note: Passing positional arguments to\n" -"test_deprecate_positional_pos0_len3_with_kwdonly() is deprecated.\n" -"Parameters \'a\', \'b\' and \'c\' will become keyword-only parameters in\n" -"Python 3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS0_LEN3_WITH_KWDONLY_METHODDEF \ - {"test_deprecate_positional_pos0_len3_with_kwdonly", _PyCFunction_CAST(test_deprecate_positional_pos0_len3_with_kwdonly), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos0_len3_with_kwdonly__doc__}, - -static PyObject * -test_deprecate_positional_pos0_len3_with_kwdonly_impl(PyObject *module, - PyObject *a, - PyObject *b, - PyObject *c, - PyObject *e); - -static PyObject * -test_deprecate_positional_pos0_len3_with_kwdonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 4 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(e), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "e", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos0_len3_with_kwdonly", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[4]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *e; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the " \ - "clinic input of " \ - "'test_deprecate_positional_pos0_len3_with_kwdonly' to be " \ - "keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the " \ - "clinic input of " \ - "'test_deprecate_positional_pos0_len3_with_kwdonly' to be " \ - "keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'a', 'b' and 'c' in the " \ - "clinic input of " \ - "'test_deprecate_positional_pos0_len3_with_kwdonly' to be " \ - "keyword-only." - # endif - #endif - if (nargs > 0 && nargs <= 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to " - "test_deprecate_positional_pos0_len3_with_kwdonly() is " - "deprecated. Parameters 'a', 'b' and 'c' will become keyword-only" - " parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - e = args[3]; - return_value = test_deprecate_positional_pos0_len3_with_kwdonly_impl(module, a, b, c, e); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos0_len3_with_kwdonly_impl(PyObject *module, - PyObject *a, - PyObject *b, - PyObject *c, - PyObject *e) -/*[clinic end generated code: output=a553e33101dc42b2 input=1b0121770c0c52e0]*/ - - -/*[clinic input] -test_deprecate_positional_pos2_len1 - a: object - b: object - * [from 3.14] - c: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos2_len1__doc__, -"test_deprecate_positional_pos2_len1($module, /, a, b, c)\n" -"--\n" -"\n" -"Note: Passing 3 positional arguments to\n" -"test_deprecate_positional_pos2_len1() is deprecated. Parameter \'c\'\n" -"will become a keyword-only parameter in Python 3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS2_LEN1_METHODDEF \ - {"test_deprecate_positional_pos2_len1", _PyCFunction_CAST(test_deprecate_positional_pos2_len1), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len1__doc__}, - -static PyObject * -test_deprecate_positional_pos2_len1_impl(PyObject *module, PyObject *a, - PyObject *b, PyObject *c); - -static PyObject * -test_deprecate_positional_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 3 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos2_len1", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[3]; - PyObject *a; - PyObject *b; - PyObject *c; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'c' in the clinic input of" \ - " 'test_deprecate_positional_pos2_len1' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'c' in the clinic input of" \ - " 'test_deprecate_positional_pos2_len1' to be keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'c' in the clinic input of" \ - " 'test_deprecate_positional_pos2_len1' to be keyword-only." - # endif - #endif - if (nargs == 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing 3 positional arguments to " - "test_deprecate_positional_pos2_len1() is deprecated. Parameter " - "'c' will become a keyword-only parameter in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - return_value = test_deprecate_positional_pos2_len1_impl(module, a, b, c); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos2_len1_impl(PyObject *module, PyObject *a, - PyObject *b, PyObject *c) -/*[clinic end generated code: output=f96454a4970b443c input=e1d129689e69ec7c]*/ - - -/*[clinic input] -test_deprecate_positional_pos2_len2 - a: object - b: object - * [from 3.14] - c: object - d: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos2_len2__doc__, -"test_deprecate_positional_pos2_len2($module, /, a, b, c, d)\n" -"--\n" -"\n" -"Note: Passing more than 2 positional arguments to\n" -"test_deprecate_positional_pos2_len2() is deprecated. Parameters \'c\'\n" -"and \'d\' will become keyword-only parameters in Python 3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS2_LEN2_METHODDEF \ - {"test_deprecate_positional_pos2_len2", _PyCFunction_CAST(test_deprecate_positional_pos2_len2), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len2__doc__}, - -static PyObject * -test_deprecate_positional_pos2_len2_impl(PyObject *module, PyObject *a, - PyObject *b, PyObject *c, - PyObject *d); - -static PyObject * -test_deprecate_positional_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 4 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos2_len2", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[4]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *d; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'test_deprecate_positional_pos2_len2' to be " \ - "keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'test_deprecate_positional_pos2_len2' to be " \ - "keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'test_deprecate_positional_pos2_len2' to be " \ - "keyword-only." - # endif - #endif - if (nargs > 2 && nargs <= 4) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 2 positional arguments to " - "test_deprecate_positional_pos2_len2() is deprecated. Parameters " - "'c' and 'd' will become keyword-only parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - d = args[3]; - return_value = test_deprecate_positional_pos2_len2_impl(module, a, b, c, d); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos2_len2_impl(PyObject *module, PyObject *a, - PyObject *b, PyObject *c, - PyObject *d) -/*[clinic end generated code: output=5e648e887da0a804 input=0d53533463a12792]*/ - - -/*[clinic input] -test_deprecate_positional_pos2_len3_with_kwdonly - a: object - b: object - * [from 3.14] - c: object - d: object - * - e: object -[clinic start generated code]*/ - -PyDoc_STRVAR(test_deprecate_positional_pos2_len3_with_kwdonly__doc__, -"test_deprecate_positional_pos2_len3_with_kwdonly($module, /, a, b, c,\n" -" d, *, e)\n" -"--\n" -"\n" -"Note: Passing more than 2 positional arguments to\n" -"test_deprecate_positional_pos2_len3_with_kwdonly() is deprecated.\n" -"Parameters \'c\' and \'d\' will become keyword-only parameters in Python\n" -"3.14.\n" -""); - -#define TEST_DEPRECATE_POSITIONAL_POS2_LEN3_WITH_KWDONLY_METHODDEF \ - {"test_deprecate_positional_pos2_len3_with_kwdonly", _PyCFunction_CAST(test_deprecate_positional_pos2_len3_with_kwdonly), METH_FASTCALL|METH_KEYWORDS, test_deprecate_positional_pos2_len3_with_kwdonly__doc__}, - -static PyObject * -test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, - PyObject *a, - PyObject *b, - PyObject *c, - PyObject *d, - PyObject *e); - -static PyObject * -test_deprecate_positional_pos2_len3_with_kwdonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 5 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "d", "e", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "test_deprecate_positional_pos2_len3_with_kwdonly", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[5]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *d; - PyObject *e; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to " \ - "be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to " \ - "be keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'test_deprecate_positional_pos2_len3_with_kwdonly' to " \ - "be keyword-only." - # endif - #endif - if (nargs > 2 && nargs <= 4) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 2 positional arguments to " - "test_deprecate_positional_pos2_len3_with_kwdonly() is " - "deprecated. Parameters 'c' and 'd' will become keyword-only " - "parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 1, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - d = args[3]; - e = args[4]; - return_value = test_deprecate_positional_pos2_len3_with_kwdonly_impl(module, a, b, c, d, e); - -exit: - return return_value; -} - -static PyObject * -test_deprecate_positional_pos2_len3_with_kwdonly_impl(PyObject *module, - PyObject *a, - PyObject *b, - PyObject *c, - PyObject *d, - PyObject *e) -/*[clinic end generated code: output=383d56b03f7c2dcb input=154fd450448d8935]*/ - - -/*[clinic input] - at classmethod -Test.__new__ - * [from 3.14] - a: object -The deprecation message should use the class name instead of __new__. -[clinic start generated code]*/ - -PyDoc_STRVAR(Test__doc__, -"Test(a)\n" -"--\n" -"\n" -"The deprecation message should use the class name instead of __new__.\n" -"\n" -"Note: Passing positional arguments to Test() is deprecated. Parameter\n" -"\'a\' will become a keyword-only parameter in Python 3.14.\n" -""); - -static PyObject * -Test_impl(PyTypeObject *type, PyObject *a); - -static PyObject * -Test(PyTypeObject *type, PyObject *args, PyObject *kwargs) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "Test", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject * const *fastargs; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'Test.__new__' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'Test.__new__' to be keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'Test.__new__' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to Test() is deprecated. Parameter " - "'a' will become a keyword-only parameter in Python 3.14.", 1)) - { - goto exit; - } - } - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); - if (!fastargs) { - goto exit; - } - a = fastargs[0]; - return_value = Test_impl(type, a); - -exit: - return return_value; -} - -static PyObject * -Test_impl(PyTypeObject *type, PyObject *a) -/*[clinic end generated code: output=d15a69ea37ec6502 input=f133dc077aef49ec]*/ - - -/*[clinic input] -m.T.__init__ - * [from 3.14] - a: object -The deprecation message should use the class name instead of __init__. -[clinic start generated code]*/ - -PyDoc_STRVAR(m_T___init____doc__, -"T(a)\n" -"--\n" -"\n" -"The deprecation message should use the class name instead of __init__.\n" -"\n" -"Note: Passing positional arguments to m.T() is deprecated. Parameter\n" -"\'a\' will become a keyword-only parameter in Python 3.14.\n" -""); - -static int -m_T___init___impl(TestObj *self, PyObject *a); - -static int -m_T___init__(PyObject *self, PyObject *args, PyObject *kwargs) -{ - int return_value = -1; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "T", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject * const *fastargs; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'm.T.__init__' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'm.T.__init__' to be keyword-only.") - # else - # warning \ - "In clinic.test.c, update parameter(s) 'a' in the clinic input of" \ - " 'm.T.__init__' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to m.T() is deprecated. Parameter " - "'a' will become a keyword-only parameter in Python 3.14.", 1)) - { - goto exit; - } - } - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); - if (!fastargs) { - goto exit; - } - a = fastargs[0]; - return_value = m_T___init___impl((TestObj *)self, a); - -exit: - return return_value; -} - -static int -m_T___init___impl(TestObj *self, PyObject *a) -/*[clinic end generated code: output=ef43c425816a549f input=f71b51dbe19fa657]*/ diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 3921523af067f..a649b5fe2201c 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2,6 +2,7 @@ # Copyright 2012-2013 by Larry Hastings. # Licensed to the PSF under a contributor agreement. +from functools import partial from test import support, test_tools from test.support import os_helper from test.support.os_helper import TESTFN, unlink @@ -2431,6 +2432,19 @@ class ClinicFunctionalTest(unittest.TestCase): locals().update((name, getattr(ac_tester, name)) for name in dir(ac_tester) if name.startswith('test_')) + def check_depr_star(self, pnames, fn, *args, **kwds): + regex = ( + fr"Passing( more than)?( [0-9]+)? positional argument(s)? to " + fr"{fn.__name__}\(\) is deprecated. Parameter(s)? {pnames} will " + fr"become( a)? keyword-only parameter(s)? in Python 3\.14" + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + # Record the line number, so we're sure we've got the correct stack + # level on the deprecation warning. + _, lineno = fn(*args, **kwds), sys._getframe().f_lineno + self.assertEqual(cm.filename, __file__) + self.assertEqual(cm.lineno, lineno) + def test_objects_converter(self): with self.assertRaises(TypeError): ac_tester.objects_converter() @@ -2895,6 +2909,95 @@ def test_cloned_func_with_converter_exception_message(self): func = getattr(ac_tester, name) self.assertEqual(func(), name) + def test_depr_star_new(self): + regex = re.escape( + "Passing positional arguments to _testclinic.DeprStarNew() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14." + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + ac_tester.DeprStarNew(None) + self.assertEqual(cm.filename, __file__) + + def test_depr_star_init(self): + regex = re.escape( + "Passing positional arguments to _testclinic.DeprStarInit() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14." + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + ac_tester.DeprStarInit(None) + self.assertEqual(cm.filename, __file__) + + def test_depr_star_pos0_len1(self): + fn = ac_tester.depr_star_pos0_len1 + fn(a=None) + self.check_depr_star("'a'", fn, "a") + + def test_depr_star_pos0_len2(self): + fn = ac_tester.depr_star_pos0_len2 + fn(a=0, b=0) + check = partial(self.check_depr_star, "'a' and 'b'", fn) + check("a", b=0) + check("a", "b") + + def test_depr_star_pos0_len3_with_kwd(self): + fn = ac_tester.depr_star_pos0_len3_with_kwd + fn(a=0, b=0, c=0, d=0) + check = partial(self.check_depr_star, "'a', 'b' and 'c'", fn) + check("a", b=0, c=0, d=0) + check("a", "b", c=0, d=0) + check("a", "b", "c", d=0) + + def test_depr_star_pos1_len1_opt(self): + fn = ac_tester.depr_star_pos1_len1_opt + fn(a=0, b=0) + fn("a", b=0) + fn(a=0) # b is optional + check = partial(self.check_depr_star, "'b'", fn) + check("a", "b") + + def test_depr_star_pos1_len1(self): + fn = ac_tester.depr_star_pos1_len1 + fn(a=0, b=0) + fn("a", b=0) + check = partial(self.check_depr_star, "'b'", fn) + check("a", "b") + + def test_depr_star_pos1_len2_with_kwd(self): + fn = ac_tester.depr_star_pos1_len2_with_kwd + fn(a=0, b=0, c=0, d=0), + fn("a", b=0, c=0, d=0), + check = partial(self.check_depr_star, "'b' and 'c'", fn) + check("a", "b", c=0, d=0), + check("a", "b", "c", d=0), + + def test_depr_star_pos2_len1(self): + fn = ac_tester.depr_star_pos2_len1 + fn(a=0, b=0, c=0) + fn("a", b=0, c=0) + fn("a", "b", c=0) + check = partial(self.check_depr_star, "'c'", fn) + check("a", "b", "c") + + def test_depr_star_pos2_len2(self): + fn = ac_tester.depr_star_pos2_len2 + fn(a=0, b=0, c=0, d=0) + fn("a", b=0, c=0, d=0) + fn("a", "b", c=0, d=0) + check = partial(self.check_depr_star, "'c' and 'd'", fn) + check("a", "b", "c", d=0) + check("a", "b", "c", "d") + + def test_depr_star_pos2_len2_with_kwd(self): + fn = ac_tester.depr_star_pos2_len2_with_kwd + fn(a=0, b=0, c=0, d=0, e=0) + fn("a", b=0, c=0, d=0, e=0) + fn("a", "b", c=0, d=0, e=0) + check = partial(self.check_depr_star, "'c' and 'd'", fn) + check("a", "b", "c", d=0, e=0) + check("a", "b", "c", "d", e=0) + class PermutationTests(unittest.TestCase): """Test permutation support functions.""" diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 8ba8f8ecadec7..8fa3cc83d871b 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1193,6 +1193,240 @@ clone_with_conv_f2_impl(PyObject *module, custom_t path) } +/*[clinic input] +output push +destination deprstar new file '{dirname}/clinic/_testclinic_depr_star.c.h' +output everything deprstar +#output methoddef_ifndef buffer 1 +output docstring_prototype suppress +output parser_prototype suppress +output impl_definition block +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=f88f37038e00fb0a]*/ + + +// Mock Python version 3.8 +#define _SAVED_PY_VERSION PY_VERSION_HEX +#undef PY_VERSION_HEX +#define PY_VERSION_HEX 0x03080000 + + +#include "clinic/_testclinic_depr_star.c.h" + + +/*[clinic input] +class _testclinic.DeprStarNew "PyObject *" "PyObject" + at classmethod +_testclinic.DeprStarNew.__new__ as depr_star_new + * [from 3.14] + a: object +The deprecation message should use the class name instead of __new__. +[clinic start generated code]*/ + +static PyObject * +depr_star_new_impl(PyTypeObject *type, PyObject *a) +/*[clinic end generated code: output=bdbb36244f90cf46 input=f4ae7dafbc23c378]*/ +{ + return type->tp_alloc(type, 0); +} + +static PyTypeObject DeprStarNew = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.DeprStarNew", + .tp_basicsize = sizeof(PyObject), + .tp_new = depr_star_new, + .tp_flags = Py_TPFLAGS_DEFAULT, +}; + + +/*[clinic input] +class _testclinic.DeprStarInit "PyObject *" "PyObject" +_testclinic.DeprStarInit.__init__ as depr_star_init + * [from 3.14] + a: object +The deprecation message should use the class name instead of __init__. +[clinic start generated code]*/ + +static int +depr_star_init_impl(PyObject *self, PyObject *a) +/*[clinic end generated code: output=8d27b43c286d3ecc input=659ebc748d87fa86]*/ +{ + return 0; +} + +static PyTypeObject DeprStarInit = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.DeprStarInit", + .tp_basicsize = sizeof(PyObject), + .tp_new = PyType_GenericNew, + .tp_init = depr_star_init, + .tp_flags = Py_TPFLAGS_DEFAULT, +}; + + +/*[clinic input] +depr_star_pos0_len1 + * [from 3.14] + a: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos0_len1_impl(PyObject *module, PyObject *a) +/*[clinic end generated code: output=e1c6c2b423129499 input=089b9aee25381b69]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos0_len2 + * [from 3.14] + a: object + b: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=96df9be39859c7e4 input=65c83a32e01495c6]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos0_len3_with_kwd + * [from 3.14] + a: object + b: object + c: object + * + d: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos0_len3_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=7f2531eda837052f input=b33f620f57d9270f]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos1_len1_opt + a: object + * [from 3.14] + b: object = None +[clinic start generated code]*/ + +static PyObject * +depr_star_pos1_len1_opt_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=b5b4e326ee3b216f input=4a4b8ff72eae9ff7]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos1_len1 + a: object + * [from 3.14] + b: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=eab92e37d5b0a480 input=1e7787a9fe5f62a0]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos1_len2_with_kwd + a: object + * [from 3.14] + b: object + c: object + * + d: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=3bccab672b7cfbb8 input=6bc7bd742fa8be15]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos2_len1 + a: object + b: object + * [from 3.14] + c: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=20f5b230e9beeb70 input=5fc3e1790dec00d5]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos2_len2 + a: object + b: object + * [from 3.14] + c: object + d: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=9f90ed8fbce27d7a input=9cc8003b89d38779]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_star_pos2_len2_with_kwd + a: object + b: object + * [from 3.14] + c: object + d: object + * + e: object +[clinic start generated code]*/ + +static PyObject * +depr_star_pos2_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d, PyObject *e) +/*[clinic end generated code: output=05432c4f20527215 input=831832d90534da91]*/ +{ + Py_RETURN_NONE; +} + + +// Reset PY_VERSION_HEX +#undef PY_VERSION_HEX +#define PY_VERSION_HEX _SAVED_PY_VERSION +#undef _SAVED_PY_VERSION + + +/*[clinic input] +output pop +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e7c7c42daced52b0]*/ + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF OBJECTS_CONVERTER_METHODDEF @@ -1248,6 +1482,16 @@ static PyMethodDef tester_methods[] = { CLONE_F2_METHODDEF CLONE_WITH_CONV_F1_METHODDEF CLONE_WITH_CONV_F2_METHODDEF + + DEPR_STAR_POS0_LEN1_METHODDEF + DEPR_STAR_POS0_LEN2_METHODDEF + DEPR_STAR_POS0_LEN3_WITH_KWD_METHODDEF + DEPR_STAR_POS1_LEN1_OPT_METHODDEF + DEPR_STAR_POS1_LEN1_METHODDEF + DEPR_STAR_POS1_LEN2_WITH_KWD_METHODDEF + DEPR_STAR_POS2_LEN1_METHODDEF + DEPR_STAR_POS2_LEN2_METHODDEF + DEPR_STAR_POS2_LEN2_WITH_KWD_METHODDEF {NULL, NULL} }; @@ -1261,7 +1505,21 @@ static struct PyModuleDef _testclinic_module = { PyMODINIT_FUNC PyInit__testclinic(void) { - return PyModule_Create(&_testclinic_module); + PyObject *m = PyModule_Create(&_testclinic_module); + if (m == NULL) { + return NULL; + } + if (PyModule_AddType(m, &DeprStarNew) < 0) { + goto error; + } + if (PyModule_AddType(m, &DeprStarInit) < 0) { + goto error; + } + return m; + +error: + Py_DECREF(m); + return NULL; } #undef RETURN_PACKED_ARGS diff --git a/Modules/clinic/_testclinic_depr_star.c.h b/Modules/clinic/_testclinic_depr_star.c.h new file mode 100644 index 0000000000000..0c2fa088268c6 --- /dev/null +++ b/Modules/clinic/_testclinic_depr_star.c.h @@ -0,0 +1,974 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(depr_star_new__doc__, +"DeprStarNew(a)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __new__.\n" +"\n" +"Note: Passing positional arguments to _testclinic.DeprStarNew() is\n" +"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +static PyObject * +depr_star_new_impl(PyTypeObject *type, PyObject *a); + +static PyObject * +depr_star_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "DeprStarNew", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarNew.__new__' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarNew.__new__' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarNew.__new__' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to _testclinic.DeprStarNew() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + return_value = depr_star_new_impl(type, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_init__doc__, +"DeprStarInit(a)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __init__.\n" +"\n" +"Note: Passing positional arguments to _testclinic.DeprStarInit() is\n" +"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +static int +depr_star_init_impl(PyObject *self, PyObject *a); + +static int +depr_star_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "DeprStarInit", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarInit.__init__' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarInit.__init__' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarInit.__init__' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to _testclinic.DeprStarInit() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + a = fastargs[0]; + return_value = depr_star_init_impl(self, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos0_len1__doc__, +"depr_star_pos0_len1($module, /, a)\n" +"--\n" +"\n" +"Note: Passing positional arguments to depr_star_pos0_len1() is\n" +"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS0_LEN1_METHODDEF \ + {"depr_star_pos0_len1", _PyCFunction_CAST(depr_star_pos0_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len1__doc__}, + +static PyObject * +depr_star_pos0_len1_impl(PyObject *module, PyObject *a); + +static PyObject * +depr_star_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos0_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *a; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " 'depr_star_pos0_len1' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " 'depr_star_pos0_len1' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " 'depr_star_pos0_len1' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to depr_star_pos0_len1() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + return_value = depr_star_pos0_len1_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos0_len2__doc__, +"depr_star_pos0_len2($module, /, a, b)\n" +"--\n" +"\n" +"Note: Passing positional arguments to depr_star_pos0_len2() is\n" +"deprecated. Parameters \'a\' and \'b\' will become keyword-only parameters\n" +"in Python 3.14.\n" +""); + +#define DEPR_STAR_POS0_LEN2_METHODDEF \ + {"depr_star_pos0_len2", _PyCFunction_CAST(depr_star_pos0_len2), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len2__doc__}, + +static PyObject * +depr_star_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +depr_star_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos0_len2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'a' and 'b' in the clinic " \ + "input of 'depr_star_pos0_len2' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'a' and 'b' in the clinic " \ + "input of 'depr_star_pos0_len2' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'a' and 'b' in the clinic " \ + "input of 'depr_star_pos0_len2' to be keyword-only." + # endif + #endif + if (nargs > 0 && nargs <= 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to depr_star_pos0_len2() is " + "deprecated. Parameters 'a' and 'b' will become keyword-only " + "parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = depr_star_pos0_len2_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos0_len3_with_kwd__doc__, +"depr_star_pos0_len3_with_kwd($module, /, a, b, c, *, d)\n" +"--\n" +"\n" +"Note: Passing positional arguments to depr_star_pos0_len3_with_kwd()\n" +"is deprecated. Parameters \'a\', \'b\' and \'c\' will become keyword-only\n" +"parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS0_LEN3_WITH_KWD_METHODDEF \ + {"depr_star_pos0_len3_with_kwd", _PyCFunction_CAST(depr_star_pos0_len3_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len3_with_kwd__doc__}, + +static PyObject * +depr_star_pos0_len3_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +depr_star_pos0_len3_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos0_len3_with_kwd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'a', 'b' and 'c' in the " \ + "clinic input of 'depr_star_pos0_len3_with_kwd' to be " \ + "keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'a', 'b' and 'c' in the " \ + "clinic input of 'depr_star_pos0_len3_with_kwd' to be " \ + "keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'a', 'b' and 'c' in the " \ + "clinic input of 'depr_star_pos0_len3_with_kwd' to be " \ + "keyword-only." + # endif + #endif + if (nargs > 0 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to depr_star_pos0_len3_with_kwd() " + "is deprecated. Parameters 'a', 'b' and 'c' will become " + "keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = depr_star_pos0_len3_with_kwd_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos1_len1_opt__doc__, +"depr_star_pos1_len1_opt($module, /, a, b=None)\n" +"--\n" +"\n" +"Note: Passing 2 positional arguments to depr_star_pos1_len1_opt() is\n" +"deprecated. Parameter \'b\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS1_LEN1_OPT_METHODDEF \ + {"depr_star_pos1_len1_opt", _PyCFunction_CAST(depr_star_pos1_len1_opt), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len1_opt__doc__}, + +static PyObject * +depr_star_pos1_len1_opt_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +depr_star_pos1_len1_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos1_len1_opt", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ + " 'depr_star_pos1_len1_opt' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ + " 'depr_star_pos1_len1_opt' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ + " 'depr_star_pos1_len1_opt' to be keyword-only." + # endif + #endif + if (nargs == 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 2 positional arguments to depr_star_pos1_len1_opt() is " + "deprecated. Parameter 'b' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + b = args[1]; +skip_optional_pos: + return_value = depr_star_pos1_len1_opt_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos1_len1__doc__, +"depr_star_pos1_len1($module, /, a, b)\n" +"--\n" +"\n" +"Note: Passing 2 positional arguments to depr_star_pos1_len1() is\n" +"deprecated. Parameter \'b\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS1_LEN1_METHODDEF \ + {"depr_star_pos1_len1", _PyCFunction_CAST(depr_star_pos1_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len1__doc__}, + +static PyObject * +depr_star_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b); + +static PyObject * +depr_star_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos1_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ + " 'depr_star_pos1_len1' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ + " 'depr_star_pos1_len1' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ + " 'depr_star_pos1_len1' to be keyword-only." + # endif + #endif + if (nargs == 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 2 positional arguments to depr_star_pos1_len1() is " + "deprecated. Parameter 'b' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = depr_star_pos1_len1_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos1_len2_with_kwd__doc__, +"depr_star_pos1_len2_with_kwd($module, /, a, b, c, *, d)\n" +"--\n" +"\n" +"Note: Passing more than 1 positional argument to\n" +"depr_star_pos1_len2_with_kwd() is deprecated. Parameters \'b\' and \'c\'\n" +"will become keyword-only parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS1_LEN2_WITH_KWD_METHODDEF \ + {"depr_star_pos1_len2_with_kwd", _PyCFunction_CAST(depr_star_pos1_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len2_with_kwd__doc__}, + +static PyObject * +depr_star_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +depr_star_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos1_len2_with_kwd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'b' and 'c' in the clinic " \ + "input of 'depr_star_pos1_len2_with_kwd' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'b' and 'c' in the clinic " \ + "input of 'depr_star_pos1_len2_with_kwd' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'b' and 'c' in the clinic " \ + "input of 'depr_star_pos1_len2_with_kwd' to be keyword-only." + # endif + #endif + if (nargs > 1 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to " + "depr_star_pos1_len2_with_kwd() is deprecated. Parameters 'b' and" + " 'c' will become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = depr_star_pos1_len2_with_kwd_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos2_len1__doc__, +"depr_star_pos2_len1($module, /, a, b, c)\n" +"--\n" +"\n" +"Note: Passing 3 positional arguments to depr_star_pos2_len1() is\n" +"deprecated. Parameter \'c\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS2_LEN1_METHODDEF \ + {"depr_star_pos2_len1", _PyCFunction_CAST(depr_star_pos2_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len1__doc__}, + +static PyObject * +depr_star_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +static PyObject * +depr_star_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos2_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *c; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'c' in the clinic input of" \ + " 'depr_star_pos2_len1' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'c' in the clinic input of" \ + " 'depr_star_pos2_len1' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'c' in the clinic input of" \ + " 'depr_star_pos2_len1' to be keyword-only." + # endif + #endif + if (nargs == 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 3 positional arguments to depr_star_pos2_len1() is " + "deprecated. Parameter 'c' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + return_value = depr_star_pos2_len1_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos2_len2__doc__, +"depr_star_pos2_len2($module, /, a, b, c, d)\n" +"--\n" +"\n" +"Note: Passing more than 2 positional arguments to\n" +"depr_star_pos2_len2() is deprecated. Parameters \'c\' and \'d\' will\n" +"become keyword-only parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS2_LEN2_METHODDEF \ + {"depr_star_pos2_len2", _PyCFunction_CAST(depr_star_pos2_len2), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len2__doc__}, + +static PyObject * +depr_star_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +static PyObject * +depr_star_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos2_len2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'depr_star_pos2_len2' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'depr_star_pos2_len2' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'depr_star_pos2_len2' to be keyword-only." + # endif + #endif + if (nargs > 2 && nargs <= 4) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 2 positional arguments to " + "depr_star_pos2_len2() is deprecated. Parameters 'c' and 'd' will" + " become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = depr_star_pos2_len2_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos2_len2_with_kwd__doc__, +"depr_star_pos2_len2_with_kwd($module, /, a, b, c, d, *, e)\n" +"--\n" +"\n" +"Note: Passing more than 2 positional arguments to\n" +"depr_star_pos2_len2_with_kwd() is deprecated. Parameters \'c\' and \'d\'\n" +"will become keyword-only parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS2_LEN2_WITH_KWD_METHODDEF \ + {"depr_star_pos2_len2_with_kwd", _PyCFunction_CAST(depr_star_pos2_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len2_with_kwd__doc__}, + +static PyObject * +depr_star_pos2_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d, PyObject *e); + +static PyObject * +depr_star_pos2_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 5 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", "e", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos2_len2_with_kwd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[5]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + PyObject *e; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'depr_star_pos2_len2_with_kwd' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'depr_star_pos2_len2_with_kwd' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ + "input of 'depr_star_pos2_len2_with_kwd' to be keyword-only." + # endif + #endif + if (nargs > 2 && nargs <= 4) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 2 positional arguments to " + "depr_star_pos2_len2_with_kwd() is deprecated. Parameters 'c' and" + " 'd' will become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + e = args[4]; + return_value = depr_star_pos2_len2_with_kwd_impl(module, a, b, c, d, e); + +exit: + return return_value; +} +/*[clinic end generated code: output=18ab056f6cc06d7e input=a9049054013a1b77]*/ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index dedcb3141457c..28bd2a4430d14 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -322,6 +322,8 @@ Modules/_testcapi/vectorcall.c - MethodDescriptorBase_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorDerived_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorNopGet_Type - Modules/_testcapi/vectorcall.c - MethodDescriptor2_Type - +Modules/_testclinic.c - DeprStarInit - +Modules/_testclinic.c - DeprStarNew - ################################## From webhook-mailer at python.org Thu Aug 10 05:19:18 2023 From: webhook-mailer at python.org (Yhg1s) Date: Thu, 10 Aug 2023 09:19:18 -0000 Subject: [Python-checkins] [3.12] gh-107814: Avoid output from Nuget installation in find_python.bat (GH-107815) (#107823) Message-ID: https://github.com/python/cpython/commit/a71500cd2446ac460610c8d57951d02ea37b0bb6 commit: a71500cd2446ac460610c8d57951d02ea37b0bb6 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-10T11:19:14+02:00 summary: [3.12] gh-107814: Avoid output from Nuget installation in find_python.bat (GH-107815) (#107823) gh-107814: Avoid output from Nuget installation in find_python.bat (GH-107815) (cherry picked from commit 1e229e2c3d212accbd5fbe3a46cd42f8252b2868) Co-authored-by: Max Bachmann files: A Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst M PCbuild/find_python.bat diff --git a/Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst b/Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst new file mode 100644 index 0000000000000..d3723353470ce --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-08-09-17-05-33.gh-issue-107814.c0Oapq.rst @@ -0,0 +1 @@ +When calling ``find_python.bat`` with ``-q`` it did not properly silence the output of nuget. That is now fixed. diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index 7af5503d80a0f..d3f62c9386900 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -52,7 +52,7 @@ @if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe) @if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl) @if NOT exist "%_Py_NUGET%" ( - @echo Downloading nuget... + @if not "%_Py_Quiet%"=="1" @echo Downloading nuget... @rem NB: Must use single quotes around NUGET here, NOT double! @rem Otherwise, a space in the path would break things @rem If it fails, retry with any available copy of Python @@ -63,7 +63,11 @@ ) @if not "%_Py_Quiet%"=="1" @echo Installing Python via nuget... -@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" + at if not "%_Py_Quiet%"=="1" ( + @"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" +) else ( + @"%_Py_NUGET%" install pythonx86 -Verbosity quiet -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" +) @rem Quote it here; it's not quoted later because "py -x.y" wouldn't work @if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found From webhook-mailer at python.org Thu Aug 10 05:24:26 2023 From: webhook-mailer at python.org (Yhg1s) Date: Thu, 10 Aug 2023 09:24:26 -0000 Subject: [Python-checkins] [3.12] GH-106684: Close `asyncio.StreamWriter` when `asyncio.StreamWriter` is not closed by application (GH-107650) (#107656) Message-ID: https://github.com/python/cpython/commit/7853c769067699c79c0d4fe4967e9d8f8b8b0a5e commit: 7853c769067699c79c0d4fe4967e9d8f8b8b0a5e branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-10T11:24:22+02:00 summary: [3.12] GH-106684: Close `asyncio.StreamWriter` when `asyncio.StreamWriter` is not closed by application (GH-107650) (#107656) GH-106684: raise `ResourceWarning` when `asyncio.StreamWriter` is not closed (GH-107650) (cherry picked from commit 41178e41995992bbe417f94bce158de93f9e3188) Co-authored-by: Kumar Aditya Co-authored-by: Kumar Aditya <59607654+kumaraditya303 at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst M Lib/asyncio/streams.py diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index bf15f517e50dc..14861dffce3a8 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -392,6 +392,10 @@ async def start_tls(self, sslcontext, *, self._transport = new_transport protocol._replace_writer(self) + def __del__(self): + if not self._transport.is_closing(): + self.close() + class StreamReader: diff --git a/Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst b/Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst new file mode 100644 index 0000000000000..85bce76229853 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst @@ -0,0 +1 @@ +Close :class:`asyncio.StreamWriter` when it is not closed by application leading to memory leaks. Patch by Kumar Aditya. From webhook-mailer at python.org Thu Aug 10 06:44:35 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Thu, 10 Aug 2023 10:44:35 -0000 Subject: [Python-checkins] [3.11] [3.12] GH-106684: Close `asyncio.StreamWriter` when `asyncio.StreamWriter` is not closed by application (GH-107650) (GH-107656) (#107836) Message-ID: https://github.com/python/cpython/commit/fb08b7905eaf67902a89eeec0f8c1aa5ee0c7ea5 commit: fb08b7905eaf67902a89eeec0f8c1aa5ee0c7ea5 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: kumaraditya303 date: 2023-08-10T16:14:31+05:30 summary: [3.11] [3.12] GH-106684: Close `asyncio.StreamWriter` when `asyncio.StreamWriter` is not closed by application (GH-107650) (GH-107656) (#107836) [3.12] GH-106684: Close `asyncio.StreamWriter` when `asyncio.StreamWriter` is not closed by application (GH-107650) (GH-107656) GH-106684: raise `ResourceWarning` when `asyncio.StreamWriter` is not closed (GH-107650) (cherry picked from commit 41178e41995992bbe417f94bce158de93f9e3188) (cherry picked from commit 7853c769067699c79c0d4fe4967e9d8f8b8b0a5e) Co-authored-by: Kumar Aditya files: A Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst M Lib/asyncio/streams.py diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 560ad8a8510e5..3d577f127041a 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -391,6 +391,10 @@ async def start_tls(self, sslcontext, *, self._transport = new_transport protocol._replace_writer(self) + def __del__(self): + if not self._transport.is_closing(): + self.close() + class StreamReader: diff --git a/Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst b/Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst new file mode 100644 index 0000000000000..85bce76229853 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-05-05-10-41.gh-issue-106684.P9zRXb.rst @@ -0,0 +1 @@ +Close :class:`asyncio.StreamWriter` when it is not closed by application leading to memory leaks. Patch by Kumar Aditya. From webhook-mailer at python.org Thu Aug 10 07:29:10 2023 From: webhook-mailer at python.org (markshannon) Date: Thu, 10 Aug 2023 11:29:10 -0000 Subject: [Python-checkins] GH-107774: Add missing audit event for PEP 669 (GH-107775) Message-ID: https://github.com/python/cpython/commit/494e3d4436774a5ac1a569a635b8c5c881ef1c0c commit: 494e3d4436774a5ac1a569a635b8c5c881ef1c0c branch: main author: Mark Shannon committer: markshannon date: 2023-08-10T12:29:06+01:00 summary: GH-107774: Add missing audit event for PEP 669 (GH-107775) files: A Misc/NEWS.d/next/Security/2023-08-05-03-51-05.gh-issue-107774.VPjaTR.rst M Lib/test/audit-tests.py M Lib/test/test_audit.py M Python/instrumentation.c diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index 0edc9d9c47276..9504829e96f00 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -514,6 +514,17 @@ def test_not_in_gc(): assert hook not in o +def test_sys_monitoring_register_callback(): + import sys + + def hook(event, args): + if event.startswith("sys.monitoring"): + print(event, args) + + sys.addaudithook(hook) + sys.monitoring.register_callback(1, 1, None) + + if __name__ == "__main__": from test.support import suppress_msvcrt_asserts diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 0b69864751d83..b12ffa5d872e8 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -257,5 +257,18 @@ def test_not_in_gc(self): self.fail(stderr) + def test_sys_monitoring_register_callback(self): + returncode, events, stderr = self.run_python("test_sys_monitoring_register_callback") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("sys.monitoring.register_callback", "(None,)")] + + self.assertEqual(actual, expected) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Security/2023-08-05-03-51-05.gh-issue-107774.VPjaTR.rst b/Misc/NEWS.d/next/Security/2023-08-05-03-51-05.gh-issue-107774.VPjaTR.rst new file mode 100644 index 0000000000000..b89b50c79f7e2 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-08-05-03-51-05.gh-issue-107774.VPjaTR.rst @@ -0,0 +1,3 @@ +PEP 669 specifies that ``sys.monitoring.register_callback`` will generate an +audit event. Pre-releases of Python 3.12 did not generate the audit event. +This is now fixed. diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 65ea7902a80a6..64684ad522f68 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1851,6 +1851,9 @@ monitoring_register_callback_impl(PyObject *module, int tool_id, int event, PyErr_Format(PyExc_ValueError, "invalid event %d", event); return NULL; } + if (PySys_Audit("sys.monitoring.register_callback", "O", func) < 0) { + return NULL; + } if (func == Py_None) { func = NULL; } From webhook-mailer at python.org Thu Aug 10 08:03:51 2023 From: webhook-mailer at python.org (iritkatriel) Date: Thu, 10 Aug 2023 12:03:51 -0000 Subject: [Python-checkins] gh-106149: move CFG and basicblock definitions into flowgraph.c, use them as opaque types in compile.c (#107639) Message-ID: https://github.com/python/cpython/commit/bafedfbebd0f21291a7824d54e39ec79f142428b commit: bafedfbebd0f21291a7824d54e39ec79f142428b branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-10T13:03:47+01:00 summary: gh-106149: move CFG and basicblock definitions into flowgraph.c, use them as opaque types in compile.c (#107639) files: M Include/internal/pycore_compile.h M Include/internal/pycore_flowgraph.h M Python/compile.c M Python/flowgraph.c diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index fa2f64021e73c..ad657c0f0fced 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -54,6 +54,11 @@ typedef struct { int s_next_free_label; /* next free label id */ } _PyCompile_InstructionSequence; +int _PyCompile_InstructionSequence_UseLabel(_PyCompile_InstructionSequence *seq, int lbl); +int _PyCompile_InstructionSequence_Addop(_PyCompile_InstructionSequence *seq, + int opcode, int oparg, + _PyCompilerSrcLocation loc); + typedef struct { PyObject *u_name; PyObject *u_qualname; /* dot-separated qualified name (lazy) */ diff --git a/Include/internal/pycore_flowgraph.h b/Include/internal/pycore_flowgraph.h index cb55f8712e7ad..58fed46886ea4 100644 --- a/Include/internal/pycore_flowgraph.h +++ b/Include/internal/pycore_flowgraph.h @@ -11,88 +11,26 @@ extern "C" { #include "pycore_opcode_utils.h" #include "pycore_compile.h" - -typedef struct { - int i_opcode; - int i_oparg; - _PyCompilerSrcLocation i_loc; - struct _PyCfgBasicblock_ *i_target; /* target block (if jump instruction) */ - struct _PyCfgBasicblock_ *i_except; /* target block when exception is raised */ -} _PyCfgInstruction; - typedef struct { int id; } _PyCfgJumpTargetLabel; +struct _PyCfgBuilder; -typedef struct { - struct _PyCfgBasicblock_ *handlers[CO_MAXBLOCKS+1]; - int depth; -} _PyCfgExceptStack; - -typedef struct _PyCfgBasicblock_ { - /* Each basicblock in a compilation unit is linked via b_list in the - reverse order that the block are allocated. b_list points to the next - block in this list, not to be confused with b_next, which is next by - control flow. */ - struct _PyCfgBasicblock_ *b_list; - /* The label of this block if it is a jump target, -1 otherwise */ - _PyCfgJumpTargetLabel b_label; - /* Exception stack at start of block, used by assembler to create the exception handling table */ - _PyCfgExceptStack *b_exceptstack; - /* pointer to an array of instructions, initially NULL */ - _PyCfgInstruction *b_instr; - /* If b_next is non-NULL, it is a pointer to the next - block reached by normal control flow. */ - struct _PyCfgBasicblock_ *b_next; - /* number of instructions used */ - int b_iused; - /* length of instruction array (b_instr) */ - int b_ialloc; - /* Used by add_checks_for_loads_of_unknown_variables */ - uint64_t b_unsafe_locals_mask; - /* Number of predecessors that a block has. */ - int b_predecessors; - /* depth of stack upon entry of block, computed by stackdepth() */ - int b_startdepth; - /* Basic block is an exception handler that preserves lasti */ - unsigned b_preserve_lasti : 1; - /* Used by compiler passes to mark whether they have visited a basic block. */ - unsigned b_visited : 1; - /* b_except_handler is used by the cold-detection algorithm to mark exception targets */ - unsigned b_except_handler : 1; - /* b_cold is true if this block is not perf critical (like an exception handler) */ - unsigned b_cold : 1; - /* b_warm is used by the cold-detection algorithm to mark blocks which are definitely not cold */ - unsigned b_warm : 1; -} _PyCfgBasicblock; +int _PyCfgBuilder_UseLabel(struct _PyCfgBuilder *g, _PyCfgJumpTargetLabel lbl); +int _PyCfgBuilder_Addop(struct _PyCfgBuilder *g, int opcode, int oparg, _PyCompilerSrcLocation loc); -int _PyBasicblock_InsertInstruction(_PyCfgBasicblock *block, int pos, _PyCfgInstruction *instr); +struct _PyCfgBuilder* _PyCfgBuilder_New(void); +void _PyCfgBuilder_Free(struct _PyCfgBuilder *g); +int _PyCfgBuilder_CheckSize(struct _PyCfgBuilder* g); -typedef struct cfg_builder_ { - /* The entryblock, at which control flow begins. All blocks of the - CFG are reachable through the b_next links */ - _PyCfgBasicblock *g_entryblock; - /* Pointer to the most recently allocated block. By following - b_list links, you can reach all allocated blocks. */ - _PyCfgBasicblock *g_block_list; - /* pointer to the block currently being constructed */ - _PyCfgBasicblock *g_curblock; - /* label for the next instruction to be placed */ - _PyCfgJumpTargetLabel g_current_label; -} _PyCfgBuilder; - -int _PyCfgBuilder_UseLabel(_PyCfgBuilder *g, _PyCfgJumpTargetLabel lbl); -int _PyCfgBuilder_Addop(_PyCfgBuilder *g, int opcode, int oparg, _PyCompilerSrcLocation loc); - -int _PyCfgBuilder_Init(_PyCfgBuilder *g); -void _PyCfgBuilder_Fini(_PyCfgBuilder *g); - -int _PyCfg_OptimizeCodeUnit(_PyCfgBuilder *g, PyObject *consts, PyObject *const_cache, +int _PyCfg_OptimizeCodeUnit(struct _PyCfgBuilder *g, PyObject *consts, PyObject *const_cache, int nlocals, int nparams, int firstlineno); -int _PyCfg_Stackdepth(_PyCfgBuilder *g); -void _PyCfg_ConvertPseudoOps(_PyCfgBasicblock *entryblock); -int _PyCfg_ResolveJumps(_PyCfgBuilder *g); + +int _PyCfg_ToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_InstructionSequence *seq); +int _PyCfg_OptimizedCfgToInstructionSequence(struct _PyCfgBuilder *g, _PyCompile_CodeUnitMetadata *umd, + int code_flags, int *stackdepth, int *nlocalsplus, + _PyCompile_InstructionSequence *seq); PyCodeObject * _PyAssemble_MakeCodeObject(_PyCompile_CodeUnitMetadata *u, PyObject *const_cache, diff --git a/Python/compile.c b/Python/compile.c index 68af5d61ac8de..83cf45550e258 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -70,9 +70,7 @@ && ((C)->u->u_ste->ste_type == ModuleBlock)) typedef _PyCompilerSrcLocation location; -typedef _PyCfgInstruction cfg_instr; -typedef _PyCfgBasicblock basicblock; -typedef _PyCfgBuilder cfg_builder; +typedef struct _PyCfgBuilder cfg_builder; #define LOCATION(LNO, END_LNO, COL, END_COL) \ ((const _PyCompilerSrcLocation){(LNO), (END_LNO), (COL), (END_COL)}) @@ -101,7 +99,7 @@ static jump_target_label NO_LABEL = {-1}; } #define USE_LABEL(C, LBL) \ - RETURN_IF_ERROR(instr_sequence_use_label(INSTR_SEQUENCE(C), (LBL).id)) + RETURN_IF_ERROR(_PyCompile_InstructionSequence_UseLabel(INSTR_SEQUENCE(C), (LBL).id)) /* fblockinfo tracks the current frame block. @@ -218,8 +216,9 @@ instr_sequence_new_label(instr_sequence *seq) return lbl; } -static int -instr_sequence_use_label(instr_sequence *seq, int lbl) { +int +_PyCompile_InstructionSequence_UseLabel(instr_sequence *seq, int lbl) +{ int old_size = seq->s_labelmap_size; RETURN_IF_ERROR( _PyCompile_EnsureArrayLargeEnough(lbl, @@ -238,8 +237,9 @@ instr_sequence_use_label(instr_sequence *seq, int lbl) { #define MAX_OPCODE 511 -static int -instr_sequence_addop(instr_sequence *seq, int opcode, int oparg, location loc) +int +_PyCompile_InstructionSequence_Addop(instr_sequence *seq, int opcode, int oparg, + location loc) { assert(0 <= opcode && opcode <= MAX_OPCODE); assert(IS_WITHIN_OPCODE_RANGE(opcode)); @@ -288,10 +288,12 @@ instr_sequence_fini(instr_sequence *seq) { seq->s_instrs = NULL; } -static int -instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) { - memset(g, 0, sizeof(cfg_builder)); - RETURN_IF_ERROR(_PyCfgBuilder_Init(g)); +static cfg_builder* +instr_sequence_to_cfg(instr_sequence *seq) { + cfg_builder *g = _PyCfgBuilder_New(); + if (g == NULL) { + return NULL; + } /* There can be more than one label for the same offset. The * offset2lbl maping selects one of them which we use consistently. @@ -300,7 +302,7 @@ instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) { int *offset2lbl = PyMem_Malloc(seq->s_used * sizeof(int)); if (offset2lbl == NULL) { PyErr_NoMemory(); - return ERROR; + goto error; } for (int i = 0; i < seq->s_used; i++) { offset2lbl[i] = -1; @@ -336,23 +338,17 @@ instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) { goto error; } } - PyMem_Free(offset2lbl); - - int nblocks = 0; - for (basicblock *b = g->g_block_list; b != NULL; b = b->b_list) { - nblocks++; - } - if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) { - PyErr_NoMemory(); - return ERROR; + if (_PyCfgBuilder_CheckSize(g) < 0) { + goto error; } - return SUCCESS; + PyMem_Free(offset2lbl); + return g; error: + _PyCfgBuilder_Free(g); PyMem_Free(offset2lbl); - return ERROR; + return NULL; } - /* The following items change on entry and exit of code blocks. They must be saved and restored when returning to a block. */ @@ -920,7 +916,7 @@ codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) { assert(!OPCODE_HAS_ARG(opcode)); assert(!IS_ASSEMBLER_OPCODE(opcode)); - return instr_sequence_addop(seq, opcode, 0, loc); + return _PyCompile_InstructionSequence_Addop(seq, opcode, 0, loc); } static Py_ssize_t @@ -1153,7 +1149,7 @@ codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) int oparg_ = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int); assert(!IS_ASSEMBLER_OPCODE(opcode)); - return instr_sequence_addop(seq, opcode, oparg_, loc); + return _PyCompile_InstructionSequence_Addop(seq, opcode, oparg_, loc); } static int @@ -1163,7 +1159,7 @@ codegen_addop_j(instr_sequence *seq, location loc, assert(IS_LABEL(target)); assert(OPCODE_HAS_JUMP(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)); assert(!IS_ASSEMBLER_OPCODE(opcode)); - return instr_sequence_addop(seq, opcode, target.id, loc); + return _PyCompile_InstructionSequence_Addop(seq, opcode, target.id, loc); } #define RETURN_IF_ERROR_IN_SCOPE(C, CALL) { \ @@ -7492,201 +7488,6 @@ _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj) return SUCCESS; } - -static int * -build_cellfixedoffsets(_PyCompile_CodeUnitMetadata *umd) -{ - int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); - - int noffsets = ncellvars + nfreevars; - int *fixed = PyMem_New(int, noffsets); - if (fixed == NULL) { - PyErr_NoMemory(); - return NULL; - } - for (int i = 0; i < noffsets; i++) { - fixed[i] = nlocals + i; - } - - PyObject *varname, *cellindex; - Py_ssize_t pos = 0; - while (PyDict_Next(umd->u_cellvars, &pos, &varname, &cellindex)) { - PyObject *varindex = PyDict_GetItem(umd->u_varnames, varname); - if (varindex != NULL) { - assert(PyLong_AS_LONG(cellindex) < INT_MAX); - assert(PyLong_AS_LONG(varindex) < INT_MAX); - int oldindex = (int)PyLong_AS_LONG(cellindex); - int argoffset = (int)PyLong_AS_LONG(varindex); - fixed[oldindex] = argoffset; - } - } - - return fixed; -} - -#define IS_GENERATOR(CF) \ - ((CF) & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) - -static int -insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, - int *fixed, int nfreevars, int code_flags) -{ - assert(umd->u_firstlineno > 0); - - /* Add the generator prefix instructions. */ - if (IS_GENERATOR(code_flags)) { - /* Note that RETURN_GENERATOR + POP_TOP have a net stack effect - * of 0. This is because RETURN_GENERATOR pushes an element - * with _PyFrame_StackPush before switching stacks. - */ - cfg_instr make_gen = { - .i_opcode = RETURN_GENERATOR, - .i_oparg = 0, - .i_loc = LOCATION(umd->u_firstlineno, umd->u_firstlineno, -1, -1), - .i_target = NULL, - }; - RETURN_IF_ERROR(_PyBasicblock_InsertInstruction(entryblock, 0, &make_gen)); - cfg_instr pop_top = { - .i_opcode = POP_TOP, - .i_oparg = 0, - .i_loc = NO_LOCATION, - .i_target = NULL, - }; - RETURN_IF_ERROR(_PyBasicblock_InsertInstruction(entryblock, 1, &pop_top)); - } - - /* Set up cells for any variable that escapes, to be put in a closure. */ - const int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); - if (ncellvars) { - // umd->u_cellvars has the cells out of order so we sort them - // before adding the MAKE_CELL instructions. Note that we - // adjust for arg cells, which come first. - const int nvars = ncellvars + (int)PyDict_GET_SIZE(umd->u_varnames); - int *sorted = PyMem_RawCalloc(nvars, sizeof(int)); - if (sorted == NULL) { - PyErr_NoMemory(); - return ERROR; - } - for (int i = 0; i < ncellvars; i++) { - sorted[fixed[i]] = i + 1; - } - for (int i = 0, ncellsused = 0; ncellsused < ncellvars; i++) { - int oldindex = sorted[i] - 1; - if (oldindex == -1) { - continue; - } - cfg_instr make_cell = { - .i_opcode = MAKE_CELL, - // This will get fixed in offset_derefs(). - .i_oparg = oldindex, - .i_loc = NO_LOCATION, - .i_target = NULL, - }; - if (_PyBasicblock_InsertInstruction(entryblock, ncellsused, &make_cell) < 0) { - PyMem_RawFree(sorted); - return ERROR; - } - ncellsused += 1; - } - PyMem_RawFree(sorted); - } - - if (nfreevars) { - cfg_instr copy_frees = { - .i_opcode = COPY_FREE_VARS, - .i_oparg = nfreevars, - .i_loc = NO_LOCATION, - .i_target = NULL, - }; - RETURN_IF_ERROR(_PyBasicblock_InsertInstruction(entryblock, 0, ©_frees)); - } - - return SUCCESS; -} - -static int -fix_cell_offsets(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, int *fixedmap) -{ - int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); - int noffsets = ncellvars + nfreevars; - - // First deal with duplicates (arg cells). - int numdropped = 0; - for (int i = 0; i < noffsets ; i++) { - if (fixedmap[i] == i + nlocals) { - fixedmap[i] -= numdropped; - } - else { - // It was a duplicate (cell/arg). - numdropped += 1; - } - } - - // Then update offsets, either relative to locals or by cell2arg. - for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - for (int i = 0; i < b->b_iused; i++) { - cfg_instr *inst = &b->b_instr[i]; - // This is called before extended args are generated. - assert(inst->i_opcode != EXTENDED_ARG); - int oldoffset = inst->i_oparg; - switch(inst->i_opcode) { - case MAKE_CELL: - case LOAD_CLOSURE: - case LOAD_DEREF: - case STORE_DEREF: - case DELETE_DEREF: - case LOAD_FROM_DICT_OR_DEREF: - assert(oldoffset >= 0); - assert(oldoffset < noffsets); - assert(fixedmap[oldoffset] >= 0); - inst->i_oparg = fixedmap[oldoffset]; - } - } - } - - return numdropped; -} - - -static int -prepare_localsplus(_PyCompile_CodeUnitMetadata *umd, cfg_builder *g, int code_flags) -{ - assert(PyDict_GET_SIZE(umd->u_varnames) < INT_MAX); - assert(PyDict_GET_SIZE(umd->u_cellvars) < INT_MAX); - assert(PyDict_GET_SIZE(umd->u_freevars) < INT_MAX); - int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); - assert(INT_MAX - nlocals - ncellvars > 0); - assert(INT_MAX - nlocals - ncellvars - nfreevars > 0); - int nlocalsplus = nlocals + ncellvars + nfreevars; - int* cellfixedoffsets = build_cellfixedoffsets(umd); - if (cellfixedoffsets == NULL) { - return ERROR; - } - - - // This must be called before fix_cell_offsets(). - if (insert_prefix_instructions(umd, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { - PyMem_Free(cellfixedoffsets); - return ERROR; - } - - int numdropped = fix_cell_offsets(umd, g->g_entryblock, cellfixedoffsets); - PyMem_Free(cellfixedoffsets); // At this point we're done with it. - cellfixedoffsets = NULL; - if (numdropped < 0) { - return ERROR; - } - - nlocalsplus -= numdropped; - return nlocalsplus; -} - static int add_return_at_end(struct compiler *c, int addNone) { @@ -7700,12 +7501,11 @@ add_return_at_end(struct compiler *c, int addNone) return SUCCESS; } -static int cfg_to_instr_sequence(cfg_builder *g, instr_sequence *seq); - static PyCodeObject * optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, int code_flags, PyObject *filename) { + cfg_builder *g = NULL; instr_sequence optimized_instrs; memset(&optimized_instrs, 0, sizeof(instr_sequence)); @@ -7714,51 +7514,28 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, if (consts == NULL) { goto error; } - cfg_builder g; - if (instr_sequence_to_cfg(&u->u_instr_sequence, &g) < 0) { + g = instr_sequence_to_cfg(&u->u_instr_sequence); + if (g == NULL) { goto error; } - int nparams = (int)PyList_GET_SIZE(u->u_ste->ste_varnames); int nlocals = (int)PyDict_GET_SIZE(u->u_metadata.u_varnames); + int nparams = (int)PyList_GET_SIZE(u->u_ste->ste_varnames); assert(u->u_metadata.u_firstlineno); - if (_PyCfg_OptimizeCodeUnit(&g, consts, const_cache, nlocals, + + if (_PyCfg_OptimizeCodeUnit(g, consts, const_cache, nlocals, nparams, u->u_metadata.u_firstlineno) < 0) { goto error; } - int stackdepth = _PyCfg_Stackdepth(&g); - if (stackdepth < 0) { + int stackdepth; + int nlocalsplus; + if (_PyCfg_OptimizedCfgToInstructionSequence(g, &u->u_metadata, code_flags, + &stackdepth, &nlocalsplus, + &optimized_instrs) < 0) { goto error; } - /* prepare_localsplus adds instructions for generators that push - * and pop an item on the stack. This assertion makes sure there - * is space on the stack for that. - * It should always be true, because a generator must have at - * least one expression or call to INTRINSIC_STOPITERATION_ERROR, - * which requires stackspace. - */ - assert(!(IS_GENERATOR(code_flags) && stackdepth == 0)); - /** Assembly **/ - int nlocalsplus = prepare_localsplus(&u->u_metadata, &g, code_flags); - if (nlocalsplus < 0) { - goto error; - } - - _PyCfg_ConvertPseudoOps(g.g_entryblock); - - /* Order of basic blocks must have been determined by now */ - - if (_PyCfg_ResolveJumps(&g) < 0) { - goto error; - } - - /* Can't modify the bytecode after computing jump offsets. */ - - if (cfg_to_instr_sequence(&g, &optimized_instrs) < 0) { - goto error; - } co = _PyAssemble_MakeCodeObject(&u->u_metadata, const_cache, consts, stackdepth, &optimized_instrs, nlocalsplus, @@ -7767,7 +7544,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, error: Py_XDECREF(consts); instr_sequence_fini(&optimized_instrs); - _PyCfgBuilder_Fini(&g); + _PyCfgBuilder_Free(g); return co; } @@ -7790,39 +7567,6 @@ optimize_and_assemble(struct compiler *c, int addNone) return optimize_and_assemble_code_unit(u, const_cache, code_flags, filename); } -static int -cfg_to_instr_sequence(cfg_builder *g, instr_sequence *seq) -{ - int lbl = 0; - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - b->b_label = (jump_target_label){lbl}; - lbl += b->b_iused; - } - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(instr_sequence_use_label(seq, b->b_label.id)); - for (int i = 0; i < b->b_iused; i++) { - cfg_instr *instr = &b->b_instr[i]; - if (OPCODE_HAS_JUMP(instr->i_opcode)) { - instr->i_oparg = instr->i_target->b_label.id; - } - RETURN_IF_ERROR( - instr_sequence_addop(seq, instr->i_opcode, instr->i_oparg, instr->i_loc)); - - _PyCompile_ExceptHandlerInfo *hi = &seq->s_instrs[seq->s_used-1].i_except_handler_info; - if (instr->i_except != NULL) { - hi->h_label = instr->i_except->b_label.id; - hi->h_startdepth = instr->i_except->b_startdepth; - hi->h_preserve_lasti = instr->i_except->b_preserve_lasti; - } - else { - hi->h_label = -1; - } - } - } - return SUCCESS; -} - - /* Access to compiler optimizations for unit tests. * * _PyCompile_CodeGen takes and AST, applies code-gen and @@ -7872,7 +7616,7 @@ instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq) for (int i = 0; i < num_insts; i++) { if (is_target[i]) { - if (instr_sequence_use_label(seq, i) < 0) { + if (_PyCompile_InstructionSequence_UseLabel(seq, i) < 0) { goto error; } } @@ -7912,7 +7656,7 @@ instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq) if (PyErr_Occurred()) { goto error; } - if (instr_sequence_addop(seq, opcode, oparg, loc) < 0) { + if (_PyCompile_InstructionSequence_Addop(seq, opcode, oparg, loc) < 0) { goto error; } } @@ -7923,23 +7667,26 @@ instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq) return ERROR; } -static int -instructions_to_cfg(PyObject *instructions, cfg_builder *g) +static cfg_builder* +instructions_to_cfg(PyObject *instructions) { + cfg_builder *g = NULL; instr_sequence seq; memset(&seq, 0, sizeof(instr_sequence)); if (instructions_to_instr_sequence(instructions, &seq) < 0) { goto error; } - if (instr_sequence_to_cfg(&seq, g) < 0) { + g = instr_sequence_to_cfg(&seq); + if (g == NULL) { goto error; } instr_sequence_fini(&seq); - return SUCCESS; + return g; error: + _PyCfgBuilder_Free(g); instr_sequence_fini(&seq); - return ERROR; + return NULL; } static PyObject * @@ -7978,42 +7725,14 @@ instr_sequence_to_instructions(instr_sequence *seq) static PyObject * cfg_to_instructions(cfg_builder *g) { - PyObject *instructions = PyList_New(0); - if (instructions == NULL) { + instr_sequence seq; + memset(&seq, 0, sizeof(seq)); + if (_PyCfg_ToInstructionSequence(g, &seq) < 0) { return NULL; } - int lbl = 0; - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - b->b_label = (jump_target_label){lbl}; - lbl += b->b_iused; - } - for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - for (int i = 0; i < b->b_iused; i++) { - cfg_instr *instr = &b->b_instr[i]; - location loc = instr->i_loc; - int arg = HAS_TARGET(instr->i_opcode) ? - instr->i_target->b_label.id : instr->i_oparg; - - PyObject *inst_tuple = Py_BuildValue( - "(iiiiii)", instr->i_opcode, arg, - loc.lineno, loc.end_lineno, - loc.col_offset, loc.end_col_offset); - if (inst_tuple == NULL) { - goto error; - } - - if (PyList_Append(instructions, inst_tuple) != 0) { - Py_DECREF(inst_tuple); - goto error; - } - Py_DECREF(inst_tuple); - } - } - - return instructions; -error: - Py_DECREF(instructions); - return NULL; + PyObject *res = instr_sequence_to_instructions(&seq); + instr_sequence_fini(&seq); + return res; } // C implementation of inspect.cleandoc() @@ -8195,34 +7914,36 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, PyObject * _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts, int nlocals) { + cfg_builder *g = NULL; PyObject *res = NULL; PyObject *const_cache = PyDict_New(); if (const_cache == NULL) { return NULL; } - cfg_builder g; - if (instructions_to_cfg(instructions, &g) < 0) { + g = instructions_to_cfg(instructions); + if (g == NULL) { goto error; } int nparams = 0, firstlineno = 1; - if (_PyCfg_OptimizeCodeUnit(&g, consts, const_cache, nlocals, + if (_PyCfg_OptimizeCodeUnit(g, consts, const_cache, nlocals, nparams, firstlineno) < 0) { goto error; } - res = cfg_to_instructions(&g); + res = cfg_to_instructions(g); error: Py_DECREF(const_cache); - _PyCfgBuilder_Fini(&g); + _PyCfgBuilder_Free(g); return res; } -int _PyCfg_JumpLabelsToTargets(basicblock *entryblock); +int _PyCfg_JumpLabelsToTargets(cfg_builder *g); PyCodeObject * _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, PyObject *instructions) { + cfg_builder *g = NULL; PyCodeObject *co = NULL; instr_sequence optimized_instrs; memset(&optimized_instrs, 0, sizeof(instr_sequence)); @@ -8232,37 +7953,20 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, return NULL; } - cfg_builder g; - if (instructions_to_cfg(instructions, &g) < 0) { + g = instructions_to_cfg(instructions); + if (g == NULL) { goto error; } - if (_PyCfg_JumpLabelsToTargets(g.g_entryblock) < 0) { - goto error; - } - - int stackdepth = _PyCfg_Stackdepth(&g); - if (stackdepth < 0) { + if (_PyCfg_JumpLabelsToTargets(g) < 0) { goto error; } int code_flags = 0; - int nlocalsplus = prepare_localsplus(umd, &g, code_flags); - if (nlocalsplus < 0) { - goto error; - } - - _PyCfg_ConvertPseudoOps(g.g_entryblock); - - /* Order of basic blocks must have been determined by now */ - - if (_PyCfg_ResolveJumps(&g) < 0) { - goto error; - } - - /* Can't modify the bytecode after computing jump offsets. */ - - if (cfg_to_instr_sequence(&g, &optimized_instrs) < 0) { + int stackdepth, nlocalsplus; + if (_PyCfg_OptimizedCfgToInstructionSequence(g, umd, code_flags, + &stackdepth, &nlocalsplus, + &optimized_instrs) < 0) { goto error; } @@ -8277,7 +7981,7 @@ _PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, error: Py_DECREF(const_cache); - _PyCfgBuilder_Fini(&g); + _PyCfgBuilder_Free(g); instr_sequence_fini(&optimized_instrs); return co; } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 5b6b3f3cfefb0..9d7865661a803 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -24,15 +24,75 @@ typedef _PyCompilerSrcLocation location; typedef _PyCfgJumpTargetLabel jump_target_label; -typedef _PyCfgBasicblock basicblock; -typedef _PyCfgBuilder cfg_builder; -typedef _PyCfgInstruction cfg_instr; + +typedef struct _PyCfgInstruction { + int i_opcode; + int i_oparg; + _PyCompilerSrcLocation i_loc; + struct _PyCfgBasicblock *i_target; /* target block (if jump instruction) */ + struct _PyCfgBasicblock *i_except; /* target block when exception is raised */ +} cfg_instr; + +typedef struct _PyCfgBasicblock { + /* Each basicblock in a compilation unit is linked via b_list in the + reverse order that the block are allocated. b_list points to the next + block in this list, not to be confused with b_next, which is next by + control flow. */ + struct _PyCfgBasicblock *b_list; + /* The label of this block if it is a jump target, -1 otherwise */ + _PyCfgJumpTargetLabel b_label; + /* Exception stack at start of block, used by assembler to create the exception handling table */ + struct _PyCfgExceptStack *b_exceptstack; + /* pointer to an array of instructions, initially NULL */ + cfg_instr *b_instr; + /* If b_next is non-NULL, it is a pointer to the next + block reached by normal control flow. */ + struct _PyCfgBasicblock *b_next; + /* number of instructions used */ + int b_iused; + /* length of instruction array (b_instr) */ + int b_ialloc; + /* Used by add_checks_for_loads_of_unknown_variables */ + uint64_t b_unsafe_locals_mask; + /* Number of predecessors that a block has. */ + int b_predecessors; + /* depth of stack upon entry of block, computed by stackdepth() */ + int b_startdepth; + /* Basic block is an exception handler that preserves lasti */ + unsigned b_preserve_lasti : 1; + /* Used by compiler passes to mark whether they have visited a basic block. */ + unsigned b_visited : 1; + /* b_except_handler is used by the cold-detection algorithm to mark exception targets */ + unsigned b_except_handler : 1; + /* b_cold is true if this block is not perf critical (like an exception handler) */ + unsigned b_cold : 1; + /* b_warm is used by the cold-detection algorithm to mark blocks which are definitely not cold */ + unsigned b_warm : 1; +} basicblock; + + +struct _PyCfgBuilder { + /* The entryblock, at which control flow begins. All blocks of the + CFG are reachable through the b_next links */ + struct _PyCfgBasicblock *g_entryblock; + /* Pointer to the most recently allocated block. By following + b_list links, you can reach all allocated blocks. */ + struct _PyCfgBasicblock *g_block_list; + /* pointer to the block currently being constructed */ + struct _PyCfgBasicblock *g_curblock; + /* label for the next instruction to be placed */ + _PyCfgJumpTargetLabel g_current_label; +}; + +typedef struct _PyCfgBuilder cfg_builder; static const jump_target_label NO_LABEL = {-1}; #define SAME_LABEL(L1, L2) ((L1).id == (L2).id) #define IS_LABEL(L) (!SAME_LABEL((L), (NO_LABEL))) +#define LOCATION(LNO, END_LNO, COL, END_COL) \ + ((const _PyCompilerSrcLocation){(LNO), (END_LNO), (COL), (END_COL)}) static inline int is_block_push(cfg_instr *i) @@ -50,7 +110,7 @@ is_jump(cfg_instr *i) #define INSTR_SET_OP1(I, OP, ARG) \ do { \ assert(OPCODE_HAS_ARG(OP)); \ - _PyCfgInstruction *_instr__ptr_ = (I); \ + cfg_instr *_instr__ptr_ = (I); \ _instr__ptr_->i_opcode = (OP); \ _instr__ptr_->i_oparg = (ARG); \ } while (0); @@ -59,7 +119,7 @@ is_jump(cfg_instr *i) #define INSTR_SET_OP0(I, OP) \ do { \ assert(!OPCODE_HAS_ARG(OP)); \ - _PyCfgInstruction *_instr__ptr_ = (I); \ + cfg_instr *_instr__ptr_ = (I); \ _instr__ptr_->i_opcode = (OP); \ _instr__ptr_->i_oparg = 0; \ } while (0); @@ -148,8 +208,8 @@ basicblock_last_instr(const basicblock *b) { } static inline int -basicblock_nofallthrough(const _PyCfgBasicblock *b) { - _PyCfgInstruction *last = basicblock_last_instr(b); +basicblock_nofallthrough(const basicblock *b) { + cfg_instr *last = basicblock_last_instr(b); return (last && (IS_SCOPE_EXIT_OPCODE(last->i_opcode) || IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode))); @@ -175,8 +235,8 @@ copy_basicblock(cfg_builder *g, basicblock *block) return result; } -int -_PyBasicblock_InsertInstruction(basicblock *block, int pos, cfg_instr *instr) { +static int +basicblock_insert_instruction(basicblock *block, int pos, cfg_instr *instr) { RETURN_IF_ERROR(basicblock_next_instr(block)); for (int i = block->b_iused - 1; i > pos; i--) { block->b_instr[i] = block->b_instr[i-1]; @@ -311,8 +371,8 @@ cfg_builder_check(cfg_builder *g) } #endif -int -_PyCfgBuilder_Init(cfg_builder *g) +static int +init_cfg_builder(cfg_builder *g) { g->g_block_list = NULL; basicblock *block = cfg_builder_new_block(g); @@ -324,9 +384,28 @@ _PyCfgBuilder_Init(cfg_builder *g) return SUCCESS; } +cfg_builder * +_PyCfgBuilder_New(void) +{ + cfg_builder *g = PyMem_Malloc(sizeof(cfg_builder)); + if (g == NULL) { + PyErr_NoMemory(); + return NULL; + } + memset(g, 0, sizeof(cfg_builder)); + if (init_cfg_builder(g) < 0) { + PyMem_Free(g); + return NULL; + } + return g; +} + void -_PyCfgBuilder_Fini(cfg_builder* g) +_PyCfgBuilder_Free(cfg_builder *g) { + if (g == NULL) { + return; + } assert(cfg_builder_check(g)); basicblock *b = g->g_block_list; while (b != NULL) { @@ -337,6 +416,21 @@ _PyCfgBuilder_Fini(cfg_builder* g) PyObject_Free((void *)b); b = next; } + PyMem_Free(g); +} + +int +_PyCfgBuilder_CheckSize(cfg_builder *g) +{ + int nblocks = 0; + for (basicblock *b = g->g_block_list; b != NULL; b = b->b_list) { + nblocks++; + } + if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) { + PyErr_NoMemory(); + return ERROR; + } + return SUCCESS; } int @@ -450,7 +544,7 @@ normalize_jumps_in_block(cfg_builder *g, basicblock *b) { static int -normalize_jumps(_PyCfgBuilder *g) +normalize_jumps(cfg_builder *g) { basicblock *entryblock = g->g_entryblock; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { @@ -463,14 +557,6 @@ normalize_jumps(_PyCfgBuilder *g) return SUCCESS; } -int -_PyCfg_ResolveJumps(_PyCfgBuilder *g) -{ - RETURN_IF_ERROR(normalize_jumps(g)); - assert(no_redundant_jumps(g)); - return SUCCESS; -} - static int check_cfg(cfg_builder *g) { for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { @@ -529,9 +615,9 @@ translate_jump_labels_to_targets(basicblock *entryblock) } int -_PyCfg_JumpLabelsToTargets(basicblock *entryblock) +_PyCfg_JumpLabelsToTargets(cfg_builder *g) { - return translate_jump_labels_to_targets(entryblock); + return translate_jump_labels_to_targets(g->g_entryblock); } static int @@ -553,10 +639,14 @@ mark_except_handlers(basicblock *entryblock) { } -typedef _PyCfgExceptStack ExceptStack; +struct _PyCfgExceptStack { + basicblock *handlers[CO_MAXBLOCKS+1]; + int depth; +}; + static basicblock * -push_except_block(ExceptStack *stack, cfg_instr *setup) { +push_except_block(struct _PyCfgExceptStack *stack, cfg_instr *setup) { assert(is_block_push(setup)); int opcode = setup->i_opcode; basicblock * target = setup->i_target; @@ -568,19 +658,19 @@ push_except_block(ExceptStack *stack, cfg_instr *setup) { } static basicblock * -pop_except_block(ExceptStack *stack) { +pop_except_block(struct _PyCfgExceptStack *stack) { assert(stack->depth > 0); return stack->handlers[--stack->depth]; } static basicblock * -except_stack_top(ExceptStack *stack) { +except_stack_top(struct _PyCfgExceptStack *stack) { return stack->handlers[stack->depth]; } -static ExceptStack * +static struct _PyCfgExceptStack * make_except_stack(void) { - ExceptStack *new = PyMem_Malloc(sizeof(ExceptStack)); + struct _PyCfgExceptStack *new = PyMem_Malloc(sizeof(struct _PyCfgExceptStack)); if (new == NULL) { PyErr_NoMemory(); return NULL; @@ -590,14 +680,14 @@ make_except_stack(void) { return new; } -static ExceptStack * -copy_except_stack(ExceptStack *stack) { - ExceptStack *copy = PyMem_Malloc(sizeof(ExceptStack)); +static struct _PyCfgExceptStack * +copy_except_stack(struct _PyCfgExceptStack *stack) { + struct _PyCfgExceptStack *copy = PyMem_Malloc(sizeof(struct _PyCfgExceptStack)); if (copy == NULL) { PyErr_NoMemory(); return NULL; } - memcpy(copy, stack, sizeof(ExceptStack)); + memcpy(copy, stack, sizeof(struct _PyCfgExceptStack)); return copy; } @@ -633,8 +723,8 @@ stackdepth_push(basicblock ***sp, basicblock *b, int depth) /* Find the flow path that needs the largest stack. We assume that * cycles in the flow graph have no net effect on the stack depth. */ -int -_PyCfg_Stackdepth(cfg_builder *g) +static int +calculate_stackdepth(cfg_builder *g) { basicblock *entryblock = g->g_entryblock; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { @@ -723,7 +813,7 @@ label_exception_targets(basicblock *entryblock) { if (todo_stack == NULL) { return ERROR; } - ExceptStack *except_stack = make_except_stack(); + struct _PyCfgExceptStack *except_stack = make_except_stack(); if (except_stack == NULL) { PyMem_Free(todo_stack); PyErr_NoMemory(); @@ -747,7 +837,7 @@ label_exception_targets(basicblock *entryblock) { cfg_instr *instr = &b->b_instr[i]; if (is_block_push(instr)) { if (!instr->i_target->b_visited) { - ExceptStack *copy = copy_except_stack(except_stack); + struct _PyCfgExceptStack *copy = copy_except_stack(except_stack); if (copy == NULL) { goto error; } @@ -766,7 +856,7 @@ label_exception_targets(basicblock *entryblock) { assert(i == b->b_iused -1); if (!instr->i_target->b_visited) { if (BB_HAS_FALLTHROUGH(b)) { - ExceptStack *copy = copy_except_stack(except_stack); + struct _PyCfgExceptStack *copy = copy_except_stack(except_stack); if (copy == NULL) { goto error; } @@ -2103,8 +2193,8 @@ push_cold_blocks_to_end(cfg_builder *g) { return SUCCESS; } -void -_PyCfg_ConvertPseudoOps(basicblock *entryblock) +static void +convert_pseudo_ops(basicblock *entryblock) { for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { @@ -2293,3 +2383,270 @@ _PyCfg_OptimizeCodeUnit(cfg_builder *g, PyObject *consts, PyObject *const_cache, RETURN_IF_ERROR(resolve_line_numbers(g, firstlineno)); return SUCCESS; } + +static int * +build_cellfixedoffsets(_PyCompile_CodeUnitMetadata *umd) +{ + int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); + + int noffsets = ncellvars + nfreevars; + int *fixed = PyMem_New(int, noffsets); + if (fixed == NULL) { + PyErr_NoMemory(); + return NULL; + } + for (int i = 0; i < noffsets; i++) { + fixed[i] = nlocals + i; + } + + PyObject *varname, *cellindex; + Py_ssize_t pos = 0; + while (PyDict_Next(umd->u_cellvars, &pos, &varname, &cellindex)) { + PyObject *varindex = PyDict_GetItem(umd->u_varnames, varname); + if (varindex != NULL) { + assert(PyLong_AS_LONG(cellindex) < INT_MAX); + assert(PyLong_AS_LONG(varindex) < INT_MAX); + int oldindex = (int)PyLong_AS_LONG(cellindex); + int argoffset = (int)PyLong_AS_LONG(varindex); + fixed[oldindex] = argoffset; + } + } + + return fixed; +} + +#define IS_GENERATOR(CF) \ + ((CF) & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) + +static int +insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, + int *fixed, int nfreevars, int code_flags) +{ + assert(umd->u_firstlineno > 0); + + /* Add the generator prefix instructions. */ + if (IS_GENERATOR(code_flags)) { + /* Note that RETURN_GENERATOR + POP_TOP have a net stack effect + * of 0. This is because RETURN_GENERATOR pushes an element + * with _PyFrame_StackPush before switching stacks. + */ + cfg_instr make_gen = { + .i_opcode = RETURN_GENERATOR, + .i_oparg = 0, + .i_loc = LOCATION(umd->u_firstlineno, umd->u_firstlineno, -1, -1), + .i_target = NULL, + }; + RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 0, &make_gen)); + cfg_instr pop_top = { + .i_opcode = POP_TOP, + .i_oparg = 0, + .i_loc = NO_LOCATION, + .i_target = NULL, + }; + RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 1, &pop_top)); + } + + /* Set up cells for any variable that escapes, to be put in a closure. */ + const int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + if (ncellvars) { + // umd->u_cellvars has the cells out of order so we sort them + // before adding the MAKE_CELL instructions. Note that we + // adjust for arg cells, which come first. + const int nvars = ncellvars + (int)PyDict_GET_SIZE(umd->u_varnames); + int *sorted = PyMem_RawCalloc(nvars, sizeof(int)); + if (sorted == NULL) { + PyErr_NoMemory(); + return ERROR; + } + for (int i = 0; i < ncellvars; i++) { + sorted[fixed[i]] = i + 1; + } + for (int i = 0, ncellsused = 0; ncellsused < ncellvars; i++) { + int oldindex = sorted[i] - 1; + if (oldindex == -1) { + continue; + } + cfg_instr make_cell = { + .i_opcode = MAKE_CELL, + // This will get fixed in offset_derefs(). + .i_oparg = oldindex, + .i_loc = NO_LOCATION, + .i_target = NULL, + }; + if (basicblock_insert_instruction(entryblock, ncellsused, &make_cell) < 0) { + PyMem_RawFree(sorted); + return ERROR; + } + ncellsused += 1; + } + PyMem_RawFree(sorted); + } + + if (nfreevars) { + cfg_instr copy_frees = { + .i_opcode = COPY_FREE_VARS, + .i_oparg = nfreevars, + .i_loc = NO_LOCATION, + .i_target = NULL, + }; + RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 0, ©_frees)); + } + + return SUCCESS; +} + +static int +fix_cell_offsets(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, int *fixedmap) +{ + int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); + int noffsets = ncellvars + nfreevars; + + // First deal with duplicates (arg cells). + int numdropped = 0; + for (int i = 0; i < noffsets ; i++) { + if (fixedmap[i] == i + nlocals) { + fixedmap[i] -= numdropped; + } + else { + // It was a duplicate (cell/arg). + numdropped += 1; + } + } + + // Then update offsets, either relative to locals or by cell2arg. + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + for (int i = 0; i < b->b_iused; i++) { + cfg_instr *inst = &b->b_instr[i]; + // This is called before extended args are generated. + assert(inst->i_opcode != EXTENDED_ARG); + int oldoffset = inst->i_oparg; + switch(inst->i_opcode) { + case MAKE_CELL: + case LOAD_CLOSURE: + case LOAD_DEREF: + case STORE_DEREF: + case DELETE_DEREF: + case LOAD_FROM_DICT_OR_DEREF: + assert(oldoffset >= 0); + assert(oldoffset < noffsets); + assert(fixedmap[oldoffset] >= 0); + inst->i_oparg = fixedmap[oldoffset]; + } + } + } + + return numdropped; +} + +static int +prepare_localsplus(_PyCompile_CodeUnitMetadata *umd, cfg_builder *g, int code_flags) +{ + assert(PyDict_GET_SIZE(umd->u_varnames) < INT_MAX); + assert(PyDict_GET_SIZE(umd->u_cellvars) < INT_MAX); + assert(PyDict_GET_SIZE(umd->u_freevars) < INT_MAX); + int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); + assert(INT_MAX - nlocals - ncellvars > 0); + assert(INT_MAX - nlocals - ncellvars - nfreevars > 0); + int nlocalsplus = nlocals + ncellvars + nfreevars; + int* cellfixedoffsets = build_cellfixedoffsets(umd); + if (cellfixedoffsets == NULL) { + return ERROR; + } + + // This must be called before fix_cell_offsets(). + if (insert_prefix_instructions(umd, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { + PyMem_Free(cellfixedoffsets); + return ERROR; + } + + int numdropped = fix_cell_offsets(umd, g->g_entryblock, cellfixedoffsets); + PyMem_Free(cellfixedoffsets); // At this point we're done with it. + cellfixedoffsets = NULL; + if (numdropped < 0) { + return ERROR; + } + + nlocalsplus -= numdropped; + return nlocalsplus; +} + +int +_PyCfg_ToInstructionSequence(cfg_builder *g, _PyCompile_InstructionSequence *seq) +{ + int lbl = 0; + for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { + b->b_label = (jump_target_label){lbl}; + lbl += b->b_iused; + } + for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { + RETURN_IF_ERROR(_PyCompile_InstructionSequence_UseLabel(seq, b->b_label.id)); + for (int i = 0; i < b->b_iused; i++) { + cfg_instr *instr = &b->b_instr[i]; + if (OPCODE_HAS_JUMP(instr->i_opcode)) { + instr->i_oparg = instr->i_target->b_label.id; + } + RETURN_IF_ERROR( + _PyCompile_InstructionSequence_Addop( + seq, instr->i_opcode, instr->i_oparg, instr->i_loc)); + + _PyCompile_ExceptHandlerInfo *hi = &seq->s_instrs[seq->s_used-1].i_except_handler_info; + if (instr->i_except != NULL) { + hi->h_label = instr->i_except->b_label.id; + hi->h_startdepth = instr->i_except->b_startdepth; + hi->h_preserve_lasti = instr->i_except->b_preserve_lasti; + } + else { + hi->h_label = -1; + } + } + } + return SUCCESS; +} + + +int +_PyCfg_OptimizedCfgToInstructionSequence(cfg_builder *g, + _PyCompile_CodeUnitMetadata *umd, int code_flags, + int *stackdepth, int *nlocalsplus, + _PyCompile_InstructionSequence *seq) +{ + *stackdepth = calculate_stackdepth(g); + if (*stackdepth < 0) { + return ERROR; + } + + /* prepare_localsplus adds instructions for generators that push + * and pop an item on the stack. This assertion makes sure there + * is space on the stack for that. + * It should always be true, because a generator must have at + * least one expression or call to INTRINSIC_STOPITERATION_ERROR, + * which requires stackspace. + */ + assert(!(IS_GENERATOR(code_flags) && *stackdepth == 0)); + + *nlocalsplus = prepare_localsplus(umd, g, code_flags); + if (*nlocalsplus < 0) { + return ERROR; + } + + convert_pseudo_ops(g->g_entryblock); + + /* Order of basic blocks must have been determined by now */ + + RETURN_IF_ERROR(normalize_jumps(g)); + assert(no_redundant_jumps(g)); + + /* Can't modify the bytecode after computing jump offsets. */ + if (_PyCfg_ToInstructionSequence(g, seq) < 0) { + return ERROR; + } + + return SUCCESS; +} + From webhook-mailer at python.org Thu Aug 10 08:34:05 2023 From: webhook-mailer at python.org (markshannon) Date: Thu, 10 Aug 2023 12:34:05 -0000 Subject: [Python-checkins] GH-106485: Handle dict subclasses correctly when dematerializing `__dict__` (GH-107837) Message-ID: https://github.com/python/cpython/commit/1d976b2da26cd62b5d0e860e3055df6fb893577f commit: 1d976b2da26cd62b5d0e860e3055df6fb893577f branch: main author: Mark Shannon committer: markshannon date: 2023-08-10T13:34:00+01:00 summary: GH-106485: Handle dict subclasses correctly when dematerializing `__dict__` (GH-107837) files: M Include/internal/pycore_dict.h M Include/internal/pycore_global_objects_fini_generated.h M Include/internal/pycore_global_strings.h M Include/internal/pycore_runtime_init_generated.h M Lib/test/test_opcache.py M Modules/_testinternalcapi.c M Objects/dictobject.c diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 6ae81c033c43a..2ad6ef0f7c04d 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -199,7 +199,7 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, } extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); -extern int _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); +extern bool _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv); extern PyObject *_PyDict_FromItems( PyObject *const *keys, Py_ssize_t keys_offset, PyObject *const *values, Py_ssize_t values_offset, diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 6d50ffd0a02f1..ee9010583ff8b 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -547,6 +547,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_lambda)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_listcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_module)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_null)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_setcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_string)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_unknown)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index bb1fb13f342fc..b081c0e023fa4 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -33,6 +33,7 @@ struct _Py_global_strings { STRUCT_FOR_STR(anon_lambda, "") STRUCT_FOR_STR(anon_listcomp, "") STRUCT_FOR_STR(anon_module, "") + STRUCT_FOR_STR(anon_null, "") STRUCT_FOR_STR(anon_setcomp, "") STRUCT_FOR_STR(anon_string, "") STRUCT_FOR_STR(anon_unknown, "") diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 2d66647438b19..8c9c7f753d857 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -539,6 +539,7 @@ extern "C" { INIT_STR(anon_lambda, ""), \ INIT_STR(anon_listcomp, ""), \ INIT_STR(anon_module, ""), \ + INIT_STR(anon_null, ""), \ INIT_STR(anon_setcomp, ""), \ INIT_STR(anon_string, ""), \ INIT_STR(anon_unknown, ""), \ diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 7d317c012a304..1baa10cbdfdef 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1,8 +1,11 @@ +import copy +import pickle import dis import threading import types import unittest from test.support import threading_helper +import _testinternalcapi class TestLoadSuperAttrCache(unittest.TestCase): @@ -997,6 +1000,124 @@ def write(items): opname = "UNPACK_SEQUENCE_LIST" self.assert_races_do_not_crash(opname, get_items, read, write) +class C: + pass + +class TestInstanceDict(unittest.TestCase): + + def setUp(self): + c = C() + c.a, c.b, c.c = 0,0,0 + + def test_values_on_instance(self): + c = C() + c.a = 1 + C().b = 2 + c.c = 3 + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, '', 3) + ) + + def test_dict_materialization(self): + c = C() + c.a = 1 + c.b = 2 + c.__dict__ + self.assertIs( + _testinternalcapi.get_object_dict_values(c), + None + ) + + def test_dict_dematerialization(self): + c = C() + c.a = 1 + c.b = 2 + c.__dict__ + self.assertIs( + _testinternalcapi.get_object_dict_values(c), + None + ) + for _ in range(100): + c.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '') + ) + + def test_dict_dematerialization_multiple_refs(self): + c = C() + c.a = 1 + c.b = 2 + d = c.__dict__ + for _ in range(100): + c.a + self.assertIs( + _testinternalcapi.get_object_dict_values(c), + None + ) + self.assertIs(c.__dict__, d) + + def test_dict_dematerialization_copy(self): + c = C() + c.a = 1 + c.b = 2 + c2 = copy.copy(c) + for _ in range(100): + c.a + c2.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '') + ) + self.assertEqual( + _testinternalcapi.get_object_dict_values(c2), + (1, 2, '') + ) + c3 = copy.deepcopy(c) + for _ in range(100): + c.a + c3.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '') + ) + #NOTE -- c3.__dict__ does not de-materialize + + def test_dict_dematerialization_pickle(self): + c = C() + c.a = 1 + c.b = 2 + c2 = pickle.loads(pickle.dumps(c)) + for _ in range(100): + c.a + c2.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '') + ) + self.assertEqual( + _testinternalcapi.get_object_dict_values(c2), + (1, 2, '') + ) + + def test_dict_dematerialization_subclass(self): + class D(dict): pass + c = C() + c.a = 1 + c.b = 2 + c.__dict__ = D(c.__dict__) + for _ in range(100): + c.a + self.assertIs( + _testinternalcapi.get_object_dict_values(c), + None + ) + self.assertEqual( + c.__dict__, + {'a':1, 'b':2} + ) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index f28b46dd9846c..d1082c7dae8ae 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -15,6 +15,7 @@ #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc #include "pycore_ceval.h" // _PyEval_AddPendingCall +#include "pycore_dict.h" // _PyDictOrValues_GetValues #include "pycore_fileutils.h" // _Py_normpath #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // PyGC_Head @@ -1530,6 +1531,40 @@ test_pymem_getallocatorsname(PyObject *self, PyObject *args) return PyUnicode_FromString(name); } +static PyObject * +get_object_dict_values(PyObject *self, PyObject *obj) +{ + PyTypeObject *type = Py_TYPE(obj); + if (!_PyType_HasFeature(type, Py_TPFLAGS_MANAGED_DICT)) { + Py_RETURN_NONE; + } + PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj); + if (!_PyDictOrValues_IsValues(dorv)) { + Py_RETURN_NONE; + } + PyDictValues *values = _PyDictOrValues_GetValues(dorv); + PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; + assert(keys != NULL); + int size = (int)keys->dk_nentries; + assert(size >= 0); + PyObject *res = PyTuple_New(size); + if (res == NULL) { + return NULL; + } + _Py_DECLARE_STR(anon_null, ""); + for(int i = 0; i < size; i++) { + PyObject *item = values->values[i]; + if (item == NULL) { + item = &_Py_STR(anon_null); + } + else { + Py_INCREF(item); + } + PyTuple_SET_ITEM(res, i, item); + } + return res; +} + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, @@ -1594,6 +1629,7 @@ static PyMethodDef module_functions[] = { {"check_pyobject_uninitialized_is_freed", check_pyobject_uninitialized_is_freed, METH_NOARGS}, {"pymem_getallocatorsname", test_pymem_getallocatorsname, METH_NOARGS}, + {"get_object_dict_values", get_object_dict_values, METH_O}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 931103cdc1a06..36375f50646fd 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5464,22 +5464,24 @@ _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) return make_dict_from_instance_attributes(interp, keys, values); } -// Return 1 if the dict was dematerialized, 0 otherwise. -int +// Return true if the dict was dematerialized, false otherwise. +bool _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) { assert(_PyObject_DictOrValuesPointer(obj) == dorv); assert(!_PyDictOrValues_IsValues(*dorv)); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv); if (dict == NULL) { - return 0; + return false; } // It's likely that this dict still shares its keys (if it was materialized // on request and not heavily modified): - assert(PyDict_CheckExact(dict)); + if (!PyDict_CheckExact(dict)) { + return false; + } assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE)); if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) || Py_REFCNT(dict) != 1) { - return 0; + return false; } assert(dict->ma_values); // We have an opportunity to do something *really* cool: dematerialize it! @@ -5490,7 +5492,7 @@ _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv) dict->ma_keys = NULL; dict->ma_values = NULL; Py_DECREF(dict); - return 1; + return true; } int From webhook-mailer at python.org Thu Aug 10 08:35:06 2023 From: webhook-mailer at python.org (markshannon) Date: Thu, 10 Aug 2023 12:35:06 -0000 Subject: [Python-checkins] GH-107674: Avoid allocating boxed ints for `sys.settrace` line events (GH-107780) Message-ID: https://github.com/python/cpython/commit/37d8b904f8b5b660f556597b21c0933b841d18de commit: 37d8b904f8b5b660f556597b21c0933b841d18de branch: main author: Mark Shannon committer: markshannon date: 2023-08-10T13:35:02+01:00 summary: GH-107674: Avoid allocating boxed ints for `sys.settrace` line events (GH-107780) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-05-04-47-18.gh-issue-107674.0sYhR2.rst M Python/instrumentation.c diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-05-04-47-18.gh-issue-107674.0sYhR2.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-05-04-47-18.gh-issue-107674.0sYhR2.rst new file mode 100644 index 0000000000000..acfbf1fa2adf2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-05-04-47-18.gh-issue-107674.0sYhR2.rst @@ -0,0 +1 @@ +Fixed performance regression in ``sys.settrace``. diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 64684ad522f68..6d11649c07fbe 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -961,7 +961,7 @@ call_instrumentation_vector( /* Offset visible to user should be the offset in bytes, as that is the * convention for APIs involving code offsets. */ int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT); - PyObject *offset_obj = PyLong_FromSsize_t(bytes_offset); + PyObject *offset_obj = PyLong_FromLong(bytes_offset); if (offset_obj == NULL) { return -1; } @@ -1141,14 +1141,46 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, (interp->monitors.tools[PY_MONITORING_EVENT_LINE] | code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_LINE] ); - PyObject *line_obj = PyLong_FromSsize_t(line); + /* Special case sys.settrace to avoid boxing the line number, + * only to immediately unbox it. */ + if (tools & (1 << PY_MONITORING_SYS_TRACE_ID)) { + if (tstate->c_tracefunc != NULL && line >= 0) { + PyFrameObject *frame_obj = _PyFrame_GetFrameObject(frame); + if (frame_obj == NULL) { + return -1; + } + if (frame_obj->f_trace_lines) { + /* Need to set tracing and what_event as if using + * the instrumentation call. */ + int old_what = tstate->what_event; + tstate->what_event = PY_MONITORING_EVENT_LINE; + tstate->tracing++; + /* Call c_tracefunc directly, having set the line number. */ + Py_INCREF(frame_obj); + frame_obj->f_lineno = line; + int err = tstate->c_tracefunc(tstate->c_traceobj, frame_obj, PyTrace_LINE, Py_None); + frame_obj->f_lineno = 0; + tstate->tracing--; + tstate->what_event = old_what; + Py_DECREF(frame_obj); + if (err) { + return -1; + } + } + } + tools &= (255 - (1 << PY_MONITORING_SYS_TRACE_ID)); + } + if (tools == 0) { + goto done; + } + PyObject *line_obj = PyLong_FromLong(line); if (line_obj == NULL) { return -1; } PyObject *args[3] = { NULL, (PyObject *)code, line_obj }; - while (tools) { + do { int tool = most_significant_bit(tools); - assert(tool >= 0 && tool < 8); + assert(tool >= 0 && tool < PY_MONITORING_SYS_PROFILE_ID); assert(tools & (1 << tool)); tools &= ~(1 << tool); int res = call_one_instrument(interp, tstate, &args[1], @@ -1166,7 +1198,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, /* DISABLE */ remove_line_tools(code, i, 1 << tool); } - } + } while (tools); Py_DECREF(line_obj); done: assert(original_opcode != 0); @@ -1195,7 +1227,7 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* code->_co_monitoring->local_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION] ); int bytes_offset = offset * (int)sizeof(_Py_CODEUNIT); - PyObject *offset_obj = PyLong_FromSsize_t(bytes_offset); + PyObject *offset_obj = PyLong_FromLong(bytes_offset); if (offset_obj == NULL) { return -1; } From webhook-mailer at python.org Thu Aug 10 10:39:17 2023 From: webhook-mailer at python.org (ericvsmith) Date: Thu, 10 Aug 2023 14:39:17 -0000 Subject: [Python-checkins] gh-107838: In dataclasses, improve error message when a non-default field follows a default field. (gh-107842) Message-ID: https://github.com/python/cpython/commit/e4275f4df36a7cdd58cd4daa7d65b1947a2593d3 commit: e4275f4df36a7cdd58cd4daa7d65b1947a2593d3 branch: main author: Eric V. Smith committer: ericvsmith date: 2023-08-10T10:39:13-04:00 summary: gh-107838: In dataclasses, improve error message when a non-default field follows a default field. (gh-107842) Add the name of the previous default argument field in an error message. files: M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 3c72c289e4388..21f3fa5c213f1 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -575,15 +575,15 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, # message, and future-proofs us in case we build up the function # using ast. - seen_default = False + seen_default = None for f in std_fields: # Only consider the non-kw-only fields in the __init__ call. if f.init: if not (f.default is MISSING and f.default_factory is MISSING): - seen_default = True + seen_default = f elif seen_default: raise TypeError(f'non-default argument {f.name!r} ' - 'follows default argument') + f'follows default argument {seen_default.name!r}') locals = {f'__dataclass_type_{f.name}__': f.type for f in fields} locals.update({ diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 6669f1c57e2e7..bd8d82438414e 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -134,7 +134,7 @@ class C: # Non-defaults following defaults. with self.assertRaisesRegex(TypeError, "non-default argument 'y' follows " - "default argument"): + "default argument 'x'"): @dataclass class C: x: int = 0 @@ -143,7 +143,7 @@ class C: # A derived class adds a non-default field after a default one. with self.assertRaisesRegex(TypeError, "non-default argument 'y' follows " - "default argument"): + "default argument 'x'"): @dataclass class B: x: int = 0 @@ -156,7 +156,7 @@ class C(B): # a field which didn't use to have a default. with self.assertRaisesRegex(TypeError, "non-default argument 'y' follows " - "default argument"): + "default argument 'x'"): @dataclass class B: x: int @@ -4521,7 +4521,7 @@ class A: # Make sure we still check for non-kwarg non-defaults not following # defaults. - err_regex = "non-default argument 'z' follows default argument" + err_regex = "non-default argument 'z' follows default argument 'a'" with self.assertRaisesRegex(TypeError, err_regex): @dataclass class A: From webhook-mailer at python.org Thu Aug 10 11:55:52 2023 From: webhook-mailer at python.org (encukou) Date: Thu, 10 Aug 2023 15:55:52 -0000 Subject: [Python-checkins] gh-107810: Improve DeprecationWarning for metaclasses with custom tp_new (GH-107834) Message-ID: https://github.com/python/cpython/commit/16dcce21768ba381996a88ac8c255bf1490b3680 commit: 16dcce21768ba381996a88ac8c255bf1490b3680 branch: main author: Marc Mueller <30130371+cdce8p at users.noreply.github.com> committer: encukou date: 2023-08-10T15:55:47Z summary: gh-107810: Improve DeprecationWarning for metaclasses with custom tp_new (GH-107834) Co-authored-by: Kirill Podoprigora files: A Misc/NEWS.d/next/C API/2023-08-10-11-12-25.gh-issue-107810.oJ40Qx.rst M Lib/test/test_capi/test_misc.py M Objects/typeobject.c diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 001d37de8e0eb..c81212202d9ef 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -611,7 +611,7 @@ class Base(metaclass=metaclass): # Class creation from C with warnings_helper.check_warnings( - ('.*custom tp_new.*in Python 3.14.*', DeprecationWarning), + ('.* _testcapi.Subclass .* custom tp_new.*in Python 3.14.*', DeprecationWarning), ): sub = _testcapi.make_type_with_base(Base) self.assertTrue(issubclass(sub, Base)) diff --git a/Misc/NEWS.d/next/C API/2023-08-10-11-12-25.gh-issue-107810.oJ40Qx.rst b/Misc/NEWS.d/next/C API/2023-08-10-11-12-25.gh-issue-107810.oJ40Qx.rst new file mode 100644 index 0000000000000..c8a1f6d122b61 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-10-11-12-25.gh-issue-107810.oJ40Qx.rst @@ -0,0 +1 @@ +Improve :exc:`DeprecationWarning` for uses of :c:type:`PyType_Spec` with metaclasses that have custom ``tp_new``. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 71e96f5af4e41..aca14e79a1997 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4264,9 +4264,9 @@ _PyType_FromMetaclass_impl( if (_allow_tp_new) { if (PyErr_WarnFormat( PyExc_DeprecationWarning, 1, - "Using PyType_Spec with metaclasses that have custom " - "tp_new is deprecated and will no longer be allowed in " - "Python 3.14.") < 0) { + "Type %s uses PyType_Spec with a metaclass that has custom " + "tp_new. This is deprecated and will no longer be allowed in " + "Python 3.14.", spec->name) < 0) { goto finally; } } From webhook-mailer at python.org Thu Aug 10 16:43:17 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Thu, 10 Aug 2023 20:43:17 -0000 Subject: [Python-checkins] Fix the long64 reader in umarshal.py (GH-107828) Message-ID: https://github.com/python/cpython/commit/50bbc56009ae7303d2482f28eb62f2603664b58f commit: 50bbc56009ae7303d2482f28eb62f2603664b58f branch: main author: Martin DeMello committer: serhiy-storchaka date: 2023-08-10T23:43:14+03:00 summary: Fix the long64 reader in umarshal.py (GH-107828) files: M Tools/build/umarshal.py diff --git a/Tools/build/umarshal.py b/Tools/build/umarshal.py index f61570cbaff75..e05d93cf23c92 100644 --- a/Tools/build/umarshal.py +++ b/Tools/build/umarshal.py @@ -125,10 +125,10 @@ def r_long64(self) -> int: x |= buf[1] << 8 x |= buf[2] << 16 x |= buf[3] << 24 - x |= buf[1] << 32 - x |= buf[1] << 40 - x |= buf[1] << 48 - x |= buf[1] << 56 + x |= buf[4] << 32 + x |= buf[5] << 40 + x |= buf[6] << 48 + x |= buf[7] << 56 x |= -(x & (1<<63)) # Sign-extend return x From webhook-mailer at python.org Thu Aug 10 17:10:50 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Thu, 10 Aug 2023 21:10:50 -0000 Subject: [Python-checkins] [3.11] Fix the long64 reader in umarshal.py (GH-107828) (GH-107850) Message-ID: https://github.com/python/cpython/commit/202efe1a3bcd499f3bf17bd953c6d36d47747e78 commit: 202efe1a3bcd499f3bf17bd953c6d36d47747e78 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: serhiy-storchaka date: 2023-08-10T21:10:46Z summary: [3.11] Fix the long64 reader in umarshal.py (GH-107828) (GH-107850) (cherry picked from commit 50bbc56009ae7303d2482f28eb62f2603664b58f) Co-authored-by: Martin DeMello files: M Tools/scripts/umarshal.py diff --git a/Tools/scripts/umarshal.py b/Tools/scripts/umarshal.py index f61570cbaff75..e05d93cf23c92 100644 --- a/Tools/scripts/umarshal.py +++ b/Tools/scripts/umarshal.py @@ -125,10 +125,10 @@ def r_long64(self) -> int: x |= buf[1] << 8 x |= buf[2] << 16 x |= buf[3] << 24 - x |= buf[1] << 32 - x |= buf[1] << 40 - x |= buf[1] << 48 - x |= buf[1] << 56 + x |= buf[4] << 32 + x |= buf[5] << 40 + x |= buf[6] << 48 + x |= buf[7] << 56 x |= -(x & (1<<63)) # Sign-extend return x From webhook-mailer at python.org Fri Aug 11 00:43:16 2023 From: webhook-mailer at python.org (rhettinger) Date: Fri, 11 Aug 2023 04:43:16 -0000 Subject: [Python-checkins] gh-107421: Clarify `OrderedDict` Examples and Recipes (#107613) Message-ID: https://github.com/python/cpython/commit/23a6db98f21cba3af69a921f01613bd5f602bf6d commit: 23a6db98f21cba3af69a921f01613bd5f602bf6d branch: main author: shailshouryya <42100758+shailshouryya at users.noreply.github.com> committer: rhettinger date: 2023-08-10T23:43:13-05:00 summary: gh-107421: Clarify `OrderedDict` Examples and Recipes (#107613) files: M Doc/library/collections.rst diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index bb46782c06e1c..b8b231bb15b1b 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -1224,7 +1224,7 @@ variants of :func:`functools.lru_cache`: result = self.func(*args) self.cache[args] = time(), result if len(self.cache) > self.maxsize: - self.cache.popitem(0) + self.cache.popitem(last=False) return result @@ -1256,12 +1256,12 @@ variants of :func:`functools.lru_cache`: if self.requests[args] <= self.cache_after: self.requests.move_to_end(args) if len(self.requests) > self.maxrequests: - self.requests.popitem(0) + self.requests.popitem(last=False) else: self.requests.pop(args, None) self.cache[args] = result if len(self.cache) > self.maxsize: - self.cache.popitem(0) + self.cache.popitem(last=False) return result .. doctest:: From webhook-mailer at python.org Fri Aug 11 05:58:31 2023 From: webhook-mailer at python.org (Yhg1s) Date: Fri, 11 Aug 2023 09:58:31 -0000 Subject: [Python-checkins] [3.12] GH-107724: Fix the signature of `PY_THROW` callback functions. (GH-107725) (#107802) Message-ID: https://github.com/python/cpython/commit/ddca26188d7828ecb762213f4b9c15d44b6048cc commit: ddca26188d7828ecb762213f4b9c15d44b6048cc branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-11T11:58:27+02:00 summary: [3.12] GH-107724: Fix the signature of `PY_THROW` callback functions. (GH-107725) (#107802) GH-107724: Fix the signature of `PY_THROW` callback functions. (GH-107725) (cherry picked from commit 52fbcf61b5a70993c2d32332ff0ad9f369d968d3) Co-authored-by: Mark Shannon files: A Misc/NEWS.d/next/Core and Builtins/2023-08-04-21-25-26.gh-issue-107724.EbBXMr.rst M Include/internal/pycore_instruments.h M Lib/test/test_monitoring.py M Python/ceval.c M Python/instrumentation.c M Python/legacy_tracing.c diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index ccccd54a2f70a..56de9f8717148 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -90,10 +90,6 @@ extern int _Py_call_instrumentation_2args(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); -extern void -_Py_call_instrumentation_exc0(PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); - extern void _Py_call_instrumentation_exc2(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 9d0ad6fa834bc..4c7438992f88e 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -743,6 +743,13 @@ class ExceptionHandledRecorder(ExceptionRecorder): def __call__(self, code, offset, exc): self.events.append(("handled", type(exc))) +class ThrowRecorder(ExceptionRecorder): + + event_type = E.PY_THROW + + def __call__(self, code, offset, exc): + self.events.append(("throw", type(exc))) + class ExceptionMonitoringTest(CheckEvents): @@ -888,6 +895,31 @@ async def async_loop(): func, recorders = self.exception_recorders) + def test_throw(self): + + def gen(): + yield 1 + yield 2 + + def func(): + try: + g = gen() + next(g) + g.throw(IndexError) + except IndexError: + pass + + self.check_balanced( + func, + recorders = self.exception_recorders) + + events = self.get_events( + func, + TEST_TOOL, + self.exception_recorders + (ThrowRecorder,) + ) + self.assertEqual(events[0], ("throw", IndexError)) + class LineRecorder: event_type = E.LINE diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-04-21-25-26.gh-issue-107724.EbBXMr.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-04-21-25-26.gh-issue-107724.EbBXMr.rst new file mode 100644 index 0000000000000..6e853cf72a334 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-04-21-25-26.gh-issue-107724.EbBXMr.rst @@ -0,0 +1,3 @@ +In pre-release versions of 3.12, up to rc1, the sys.monitoring callback +function for the ``PY_THROW`` event was missing the third, exception +argument. That is now fixed. diff --git a/Python/ceval.c b/Python/ceval.c index 88d6362bf48a1..c6f54dd24052e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2099,7 +2099,7 @@ monitor_throw(PyThreadState *tstate, if (no_tools_for_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) { return; } - _Py_call_instrumentation_exc0(tstate, PY_MONITORING_EVENT_PY_THROW, frame, instr); + do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW); } void diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 05a53d04bf3cc..8714324e72c32 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1070,16 +1070,6 @@ call_instrumentation_vector_protected( assert(_PyErr_Occurred(tstate)); } -void -_Py_call_instrumentation_exc0( - PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr) -{ - assert(_PyErr_Occurred(tstate)); - PyObject *args[3] = { NULL, NULL, NULL }; - call_instrumentation_vector_protected(tstate, event, frame, instr, 2, args); -} - void _Py_call_instrumentation_exc2( PyThreadState *tstate, int event, diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index c1c70f667ccd2..b0136d2ebc755 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -163,7 +163,7 @@ sys_trace_func2( } static PyObject * -sys_trace_unwind( +sys_trace_func3( _PyLegacyEventHandler *self, PyObject *const *args, size_t nargsf, PyObject *kwnames ) { @@ -446,7 +446,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - (vectorcallfunc)sys_trace_func2, PyTrace_CALL, + (vectorcallfunc)sys_trace_func3, PyTrace_CALL, PY_MONITORING_EVENT_PY_THROW, -1)) { return -1; } @@ -471,7 +471,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - (vectorcallfunc)sys_trace_unwind, PyTrace_RETURN, + (vectorcallfunc)sys_trace_func3, PyTrace_RETURN, PY_MONITORING_EVENT_PY_UNWIND, -1)) { return -1; } From webhook-mailer at python.org Fri Aug 11 05:59:02 2023 From: webhook-mailer at python.org (Yhg1s) Date: Fri, 11 Aug 2023 09:59:02 -0000 Subject: [Python-checkins] [3.12] GH-107774: Add missing audit event for PEP 669 (GH-107775) (#107839) Message-ID: https://github.com/python/cpython/commit/81d3afae1ae89a302866fb0cd12dc9f722e8aa4c commit: 81d3afae1ae89a302866fb0cd12dc9f722e8aa4c branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-11T11:58:58+02:00 summary: [3.12] GH-107774: Add missing audit event for PEP 669 (GH-107775) (#107839) GH-107774: Add missing audit event for PEP 669 (GH-107775) (cherry picked from commit 494e3d4436774a5ac1a569a635b8c5c881ef1c0c) Co-authored-by: Mark Shannon files: A Misc/NEWS.d/next/Security/2023-08-05-03-51-05.gh-issue-107774.VPjaTR.rst M Lib/test/audit-tests.py M Lib/test/test_audit.py M Python/instrumentation.c diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index 0edc9d9c47276..9504829e96f00 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -514,6 +514,17 @@ def test_not_in_gc(): assert hook not in o +def test_sys_monitoring_register_callback(): + import sys + + def hook(event, args): + if event.startswith("sys.monitoring"): + print(event, args) + + sys.addaudithook(hook) + sys.monitoring.register_callback(1, 1, None) + + if __name__ == "__main__": from test.support import suppress_msvcrt_asserts diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 0b69864751d83..b12ffa5d872e8 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -257,5 +257,18 @@ def test_not_in_gc(self): self.fail(stderr) + def test_sys_monitoring_register_callback(self): + returncode, events, stderr = self.run_python("test_sys_monitoring_register_callback") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + actual = [(ev[0], ev[2]) for ev in events] + expected = [("sys.monitoring.register_callback", "(None,)")] + + self.assertEqual(actual, expected) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Security/2023-08-05-03-51-05.gh-issue-107774.VPjaTR.rst b/Misc/NEWS.d/next/Security/2023-08-05-03-51-05.gh-issue-107774.VPjaTR.rst new file mode 100644 index 0000000000000..b89b50c79f7e2 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-08-05-03-51-05.gh-issue-107774.VPjaTR.rst @@ -0,0 +1,3 @@ +PEP 669 specifies that ``sys.monitoring.register_callback`` will generate an +audit event. Pre-releases of Python 3.12 did not generate the audit event. +This is now fixed. diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 8714324e72c32..a5b10ae63e2e6 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1858,6 +1858,9 @@ monitoring_register_callback_impl(PyObject *module, int tool_id, int event, PyErr_Format(PyExc_ValueError, "invalid event %d", event); return NULL; } + if (PySys_Audit("sys.monitoring.register_callback", "O", func) < 0) { + return NULL; + } if (func == Py_None) { func = NULL; } From webhook-mailer at python.org Fri Aug 11 05:59:49 2023 From: webhook-mailer at python.org (Yhg1s) Date: Fri, 11 Aug 2023 09:59:49 -0000 Subject: [Python-checkins] [3.12] Fix the long64 reader in umarshal.py (GH-107828) (#107849) Message-ID: https://github.com/python/cpython/commit/585b4cc3bd57480f8bd04ddbc6f098e55d311582 commit: 585b4cc3bd57480f8bd04ddbc6f098e55d311582 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-11T11:59:45+02:00 summary: [3.12] Fix the long64 reader in umarshal.py (GH-107828) (#107849) Fix the long64 reader in umarshal.py (GH-107828) (cherry picked from commit 50bbc56009ae7303d2482f28eb62f2603664b58f) Co-authored-by: Martin DeMello files: M Tools/build/umarshal.py diff --git a/Tools/build/umarshal.py b/Tools/build/umarshal.py index f61570cbaff75..e05d93cf23c92 100644 --- a/Tools/build/umarshal.py +++ b/Tools/build/umarshal.py @@ -125,10 +125,10 @@ def r_long64(self) -> int: x |= buf[1] << 8 x |= buf[2] << 16 x |= buf[3] << 24 - x |= buf[1] << 32 - x |= buf[1] << 40 - x |= buf[1] << 48 - x |= buf[1] << 56 + x |= buf[4] << 32 + x |= buf[5] << 40 + x |= buf[6] << 48 + x |= buf[7] << 56 x |= -(x & (1<<63)) # Sign-extend return x From webhook-mailer at python.org Fri Aug 11 09:42:00 2023 From: webhook-mailer at python.org (pablogsal) Date: Fri, 11 Aug 2023 13:42:00 -0000 Subject: [Python-checkins] [3.10] gh-98154: Clarify Usage of "Reference Count" In the Docs (#107754) Message-ID: https://github.com/python/cpython/commit/917439d4d9bebdb5d2792bb5bba095b821fdf003 commit: 917439d4d9bebdb5d2792bb5bba095b821fdf003 branch: 3.10 author: Eric Snow committer: pablogsal date: 2023-08-11T13:41:56Z summary: [3.10] gh-98154: Clarify Usage of "Reference Count" In the Docs (#107754) files: M Doc/c-api/allocation.rst M Doc/c-api/arg.rst M Doc/c-api/buffer.rst M Doc/c-api/bytes.rst M Doc/c-api/exceptions.rst M Doc/c-api/intro.rst M Doc/c-api/module.rst M Doc/c-api/object.rst M Doc/c-api/refcounting.rst M Doc/c-api/sys.rst M Doc/c-api/typeobj.rst M Doc/c-api/unicode.rst M Doc/glossary.rst M Doc/library/sys.rst diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 0a8fcc5ae5fcd..231086cb38632 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -31,9 +31,11 @@ Allocating Objects on the Heap Allocate a new Python object using the C structure type *TYPE* and the Python type object *type*. Fields not defined by the Python object header - are not initialized; the object's reference count will be one. The size of - the memory allocation is determined from the :c:member:`~PyTypeObject.tp_basicsize` field of - the type object. + are not initialized. + The caller will own the only reference to the object + (i.e. its reference count will be one). + The size of the memory allocation is determined from the + :c:member:`~PyTypeObject.tp_basicsize` field of the type object. .. c:function:: TYPE* PyObject_NewVar(TYPE, PyTypeObject *type, Py_ssize_t size) diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 6a53c79bd3bec..0c199f29a87a7 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -330,8 +330,10 @@ Other objects ``O`` (object) [PyObject \*] Store a Python object (without any conversion) in a C object pointer. The C - program thus receives the actual object that was passed. The object's reference - count is not increased. The pointer stored is not ``NULL``. + program thus receives the actual object that was passed. A new + :term:`strong reference` to the object is not created + (i.e. its reference count is not increased). + The pointer stored is not ``NULL``. ``O!`` (object) [*typeobject*, PyObject \*] Store a Python object in a C object pointer. This is similar to ``O``, but @@ -415,7 +417,8 @@ inside nested parentheses. They are: mutually exclude each other. Note that any Python object references which are provided to the caller are -*borrowed* references; do not decrement their reference count! +*borrowed* references; do not release them +(i.e. do not decrement their reference count)! Additional arguments passed to these functions must be addresses of variables whose type is determined by the format string; these are used to store values @@ -650,8 +653,10 @@ Building values Convert a C :c:type:`Py_complex` structure to a Python complex number. ``O`` (object) [PyObject \*] - Pass a Python object untouched (except for its reference count, which is - incremented by one). If the object passed in is a ``NULL`` pointer, it is assumed + Pass a Python object untouched but create a new + :term:`strong reference` to it + (i.e. its reference count is incremented by one). + If the object passed in is a ``NULL`` pointer, it is assumed that this was caused because the call producing the argument found an error and set an exception. Therefore, :c:func:`Py_BuildValue` will return ``NULL`` but won't raise an exception. If no exception has been raised yet, :exc:`SystemError` is @@ -661,7 +666,7 @@ Building values Same as ``O``. ``N`` (object) [PyObject \*] - Same as ``O``, except it doesn't increment the reference count on the object. + Same as ``O``, except it doesn't create a new :term:`strong reference`. Useful when the object is created by a call to an object constructor in the argument list. diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index e3b49472991a8..5786d96753321 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -102,7 +102,9 @@ a buffer, see :c:func:`PyObject_GetBuffer`. .. c:member:: PyObject *obj A new reference to the exporting object. The reference is owned by - the consumer and automatically decremented and set to ``NULL`` by + the consumer and automatically released + (i.e. reference count decremented) + and set to ``NULL`` by :c:func:`PyBuffer_Release`. The field is the equivalent of the return value of any standard C-API function. @@ -454,7 +456,8 @@ Buffer-related functions .. c:function:: void PyBuffer_Release(Py_buffer *view) - Release the buffer *view* and decrement the reference count for + Release the buffer *view* and release the :term:`strong reference` + (i.e. decrement the reference count) to the view's supporting object, ``view->obj``. This function MUST be called when the buffer is no longer being used, otherwise reference leaks may occur. diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index ffbfb5a61b44b..fb9b2c9de4a72 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -187,8 +187,8 @@ called with a non-bytes parameter. .. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) Create a new bytes object in *\*bytes* containing the contents of *newpart* - appended to *bytes*. This version decrements the reference count of - *newpart*. + appended to *bytes*. This version releases the :term:`strong reference` + to *newpart* (i.e. decrements its reference count). .. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 37c938ae3d7ee..00e3e8d4b8d65 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -99,7 +99,8 @@ For convenience, some of these functions will always return a This is the most common way to set the error indicator. The first argument specifies the exception type; it is normally one of the standard exceptions, - e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. + e.g. :c:data:`PyExc_RuntimeError`. You need not create a new + :term:`strong reference` to it (e.g. with :c:func:`Py_INCREF`). The second argument is an error message; it is decoded from ``'utf-8'``. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 7591537fe69f7..ae3bcd4f9ec2e 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -252,52 +252,58 @@ true if (and only if) the object pointed to by *a* is a Python list. Reference Counts ---------------- -The reference count is important because today's computers have a finite (and -often severely limited) memory size; it counts how many different places there -are that have a reference to an object. Such a place could be another object, -or a global (or static) C variable, or a local variable in some C function. -When an object's reference count becomes zero, the object is deallocated. If -it contains references to other objects, their reference count is decremented. -Those other objects may be deallocated in turn, if this decrement makes their -reference count become zero, and so on. (There's an obvious problem with -objects that reference each other here; for now, the solution is "don't do -that.") +The reference count is important because today's computers have a finite +(and often severely limited) memory size; it counts how many different +places there are that have a :term:`strong reference` to an object. +Such a place could be another object, or a global (or static) C variable, +or a local variable in some C function. +When the last :term:`strong reference` to an object is released +(i.e. its reference count becomes zero), the object is deallocated. +If it contains references to other objects, those references are released. +Those other objects may be deallocated in turn, if there are no more +references to them, and so on. (There's an obvious problem with +objects that reference each other here; for now, the solution +is "don't do that.") .. index:: single: Py_INCREF() single: Py_DECREF() -Reference counts are always manipulated explicitly. The normal way is to use -the macro :c:func:`Py_INCREF` to increment an object's reference count by one, -and :c:func:`Py_DECREF` to decrement it by one. The :c:func:`Py_DECREF` macro +Reference counts are always manipulated explicitly. The normal way is +to use the macro :c:func:`Py_INCREF` to take a new reference to an +object (i.e. increment its reference count by one), +and :c:func:`Py_DECREF` to release that reference (i.e. decrement the +reference count by one). The :c:func:`Py_DECREF` macro is considerably more complex than the incref one, since it must check whether the reference count becomes zero and then cause the object's deallocator to be -called. The deallocator is a function pointer contained in the object's type -structure. The type-specific deallocator takes care of decrementing the -reference counts for other objects contained in the object if this is a compound +called. The deallocator is a function pointer contained in the object's type +structure. The type-specific deallocator takes care of releasing references +for other objects contained in the object if this is a compound object type, such as a list, as well as performing any additional finalization that's needed. There's no chance that the reference count can overflow; at least as many bits are used to hold the reference count as there are distinct memory locations in virtual memory (assuming ``sizeof(Py_ssize_t) >= sizeof(void*)``). Thus, the reference count increment is a simple operation. -It is not necessary to increment an object's reference count for every local -variable that contains a pointer to an object. In theory, the object's +It is not necessary to hold a :term:`strong reference` (i.e. increment +the reference count) for every local variable that contains a pointer +to an object. In theory, the object's reference count goes up by one when the variable is made to point to it and it goes down by one when the variable goes out of scope. However, these two cancel each other out, so at the end the reference count hasn't changed. The only real reason to use the reference count is to prevent the object from being deallocated as long as our variable is pointing to it. If we know that there is at least one other reference to the object that lives at least as long as -our variable, there is no need to increment the reference count temporarily. +our variable, there is no need to take a new :term:`strong reference` +(i.e. increment the reference count) temporarily. An important situation where this arises is in objects that are passed as arguments to C functions in an extension module that are called from Python; the call mechanism guarantees to hold a reference to every argument for the duration of the call. However, a common pitfall is to extract an object from a list and hold on to it -for a while without incrementing its reference count. Some other operation might -conceivably remove the object from the list, decrementing its reference count +for a while without taking a new reference. Some other operation might +conceivably remove the object from the list, releasing that reference, and possibly deallocating it. The real danger is that innocent-looking operations may invoke arbitrary Python code which could do this; there is a code path which allows control to flow back to the user from a :c:func:`Py_DECREF`, so @@ -305,7 +311,8 @@ almost any operation is potentially dangerous. A safe approach is to always use the generic operations (functions whose name begins with ``PyObject_``, ``PyNumber_``, ``PySequence_`` or ``PyMapping_``). -These operations always increment the reference count of the object they return. +These operations always create a new :term:`strong reference` +(i.e. increment the reference count) of the object they return. This leaves the caller with the responsibility to call :c:func:`Py_DECREF` when they are done with the result; this soon becomes second nature. @@ -321,7 +328,7 @@ to objects (objects are not owned: they are always shared). "Owning a reference" means being responsible for calling Py_DECREF on it when the reference is no longer needed. Ownership can also be transferred, meaning that the code that receives ownership of the reference then becomes responsible for -eventually decref'ing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF` +eventually releasing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF` when it's no longer needed---or passing on this responsibility (usually to its caller). When a function passes ownership of a reference on to its caller, the caller is said to receive a *new* reference. When no ownership is transferred, @@ -379,9 +386,9 @@ For example, the above two blocks of code could be replaced by the following It is much more common to use :c:func:`PyObject_SetItem` and friends with items whose references you are only borrowing, like arguments that were passed in to -the function you are writing. In that case, their behaviour regarding reference -counts is much saner, since you don't have to increment a reference count so you -can give a reference away ("have it be stolen"). For example, this function +the function you are writing. In that case, their behaviour regarding references +is much saner, since you don't have to take a new reference just so you +can give that reference away ("have it be stolen"). For example, this function sets all items of a list (actually, any mutable sequence) to a given item:: int diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index c0351c8a6c72a..ae1e912e8e53e 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -498,7 +498,7 @@ state: .. note:: Unlike other functions that steal references, ``PyModule_AddObject()`` - only decrements the reference count of *value* **on success**. + only releases the reference to *value* **on success**. This means that its return value must be checked, and calling code must :c:func:`Py_DECREF` *value* manually on error. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 29920d24742ff..b8d775f6afc4d 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -15,8 +15,8 @@ Object Protocol .. c:macro:: Py_RETURN_NOTIMPLEMENTED Properly handle returning :c:data:`Py_NotImplemented` from within a C - function (that is, increment the reference count of NotImplemented and - return it). + function (that is, create a new :term:`strong reference` + to NotImplemented and return it). .. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags) @@ -298,11 +298,12 @@ Object Protocol When *o* is non-``NULL``, returns a type object corresponding to the object type of object *o*. On failure, raises :exc:`SystemError` and returns ``NULL``. This - is equivalent to the Python expression ``type(o)``. This function increments the - reference count of the return value. There's really no reason to use this + is equivalent to the Python expression ``type(o)``. + This function creates a new :term:`strong reference` to the return value. + There's really no reason to use this function instead of the :c:func:`Py_TYPE()` function, which returns a - pointer of type :c:expr:`PyTypeObject*`, except when the incremented reference - count is needed. + pointer of type :c:expr:`PyTypeObject*`, except when a new + :term:`strong reference` is needed. .. c:function:: int PyObject_TypeCheck(PyObject *o, PyTypeObject *type) diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 738bd77e9ce42..8e698d8d5b937 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -13,31 +13,36 @@ objects. .. c:function:: void Py_INCREF(PyObject *o) - Increment the reference count for object *o*. + Indicate taking a new :term:`strong reference` to object *o*, + indicating it is in use and should not be destroyed. This function is usually used to convert a :term:`borrowed reference` to a :term:`strong reference` in-place. The :c:func:`Py_NewRef` function can be used to create a new :term:`strong reference`. + When done using the object, release it by calling :c:func:`Py_DECREF`. + The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, use :c:func:`Py_XINCREF`. + Do not expect this function to actually modify *o* in any way. + .. c:function:: void Py_XINCREF(PyObject *o) - Increment the reference count for object *o*. The object may be ``NULL``, in - which case the macro has no effect. + Similar to :c:func:`Py_INCREF`, but the object *o* can be ``NULL``, + in which case this has no effect. See also :c:func:`Py_XNewRef`. .. c:function:: PyObject* Py_NewRef(PyObject *o) - Create a new :term:`strong reference` to an object: increment the reference - count of the object *o* and return the object *o*. + Create a new :term:`strong reference` to an object: + call :c:func:`Py_INCREF` on *o* and return the object *o*. When the :term:`strong reference` is no longer needed, :c:func:`Py_DECREF` - should be called on it to decrement the object reference count. + should be called on it to release the reference. The object *o* must not be ``NULL``; use :c:func:`Py_XNewRef` if *o* can be ``NULL``. @@ -67,9 +72,12 @@ objects. .. c:function:: void Py_DECREF(PyObject *o) - Decrement the reference count for object *o*. + Release a :term:`strong reference` to object *o*, indicating the + reference is no longer used. - If the reference count reaches zero, the object's type's deallocation + Once the last :term:`strong reference` is released + (i.e. the object's reference count reaches 0), + the object's type's deallocation function (which must not be ``NULL``) is invoked. This function is usually used to delete a :term:`strong reference` before @@ -78,6 +86,8 @@ objects. The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, use :c:func:`Py_XDECREF`. + Do not expect this function to actually modify *o* in any way. + .. warning:: The deallocation function can cause arbitrary Python code to be invoked (e.g. @@ -92,32 +102,35 @@ objects. .. c:function:: void Py_XDECREF(PyObject *o) - Decrement the reference count for object *o*. The object may be ``NULL``, in - which case the macro has no effect; otherwise the effect is the same as for - :c:func:`Py_DECREF`, and the same warning applies. + Similar to :c:func:`Py_DECREF`, but the object *o* can be ``NULL``, + in which case this has no effect. + The same warning from :c:func:`Py_DECREF` applies here as well. .. c:function:: void Py_CLEAR(PyObject *o) - Decrement the reference count for object *o*. The object may be ``NULL``, in + Release a :term:`strong reference` for object *o*. + The object may be ``NULL``, in which case the macro has no effect; otherwise the effect is the same as for :c:func:`Py_DECREF`, except that the argument is also set to ``NULL``. The warning for :c:func:`Py_DECREF` does not apply with respect to the object passed because the macro carefully uses a temporary variable and sets the argument to ``NULL`` - before decrementing its reference count. + before releasing the reference. - It is a good idea to use this macro whenever decrementing the reference - count of an object that might be traversed during garbage collection. + It is a good idea to use this macro whenever releasing a reference + to an object that might be traversed during garbage collection. .. c:function:: void Py_IncRef(PyObject *o) - Increment the reference count for object *o*. A function version of :c:func:`Py_XINCREF`. + Indicate taking a new :term:`strong reference` to object *o*. + A function version of :c:func:`Py_XINCREF`. It can be used for runtime dynamic embedding of Python. .. c:function:: void Py_DecRef(PyObject *o) - Decrement the reference count for object *o*. A function version of :c:func:`Py_XDECREF`. + Release a :term:`strong reference` to object *o*. + A function version of :c:func:`Py_XDECREF`. It can be used for runtime dynamic embedding of Python. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 8ea90dae0eef6..3aeb4dec0e65c 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -8,8 +8,9 @@ Operating System Utilities .. c:function:: PyObject* PyOS_FSPath(PyObject *path) Return the file system representation for *path*. If the object is a - :class:`str` or :class:`bytes` object, then its reference count is - incremented. If the object implements the :class:`os.PathLike` interface, + :class:`str` or :class:`bytes` object, then a new + :term:`strong reference` is returned. + If the object implements the :class:`os.PathLike` interface, then :meth:`~os.PathLike.__fspath__` is returned as long as it is a :class:`str` or :class:`bytes` object. Otherwise :exc:`TypeError` is raised and ``NULL`` is returned. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 937f52b073290..332c8f36d9e15 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -688,7 +688,8 @@ and :c:type:`PyType_Type` effectively act as defaults.) } Finally, if the type is heap allocated (:const:`Py_TPFLAGS_HEAPTYPE`), the - deallocator should decrement the reference count for its type object after + deallocator should release the owned reference to its type object + (via :c:func:`Py_DECREF`) after calling the type deallocator. In order to avoid dangling pointers, the recommended way to achieve this is: @@ -1394,9 +1395,10 @@ and :c:type:`PyType_Type` effectively act as defaults.) } The :c:func:`Py_CLEAR` macro should be used, because clearing references is - delicate: the reference to the contained object must not be decremented until + delicate: the reference to the contained object must not be released + (via :c:func:`Py_DECREF`) until after the pointer to the contained object is set to ``NULL``. This is because - decrementing the reference count may cause the contained object to become trash, + releasing the reference may cause the contained object to become trash, triggering a chain of reclamation activity that may include invoking arbitrary Python code (due to finalizers, or weakref callbacks, associated with the contained object). If it's possible for such code to reference *self* again, @@ -1472,7 +1474,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) they may be C ints or floats). The third argument specifies the requested operation, as for :c:func:`PyObject_RichCompare`. - The return value's reference count is properly incremented. + The returned value is a new :term:`strong reference`. On error, sets an exception and returns ``NULL`` from the function. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 0ef4ff9f288c4..c34e65ab75c06 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1715,11 +1715,11 @@ They all return ``NULL`` or ``-1`` if an exception occurs. Intern the argument *\*string* in place. The argument must be the address of a pointer variable pointing to a Python Unicode string object. If there is an existing interned string that is the same as *\*string*, it sets *\*string* to - it (decrementing the reference count of the old string object and incrementing - the reference count of the interned string object), otherwise it leaves - *\*string* alone and interns it (incrementing its reference count). - (Clarification: even though there is a lot of talk about reference counts, think - of this function as reference-count-neutral; you own the object after the call + it (releasing the reference to the old string object and creating a new + :term:`strong reference` to the interned string object), otherwise it leaves + *\*string* alone and interns it (creating a new :term:`strong reference`). + (Clarification: even though there is a lot of talk about references, think + of this function as reference-neutral; you own the object after the call if and only if you owned it before the call.) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 6e519df6a1e03..5dde8f8b16e2d 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -168,8 +168,9 @@ Glossary :class:`str` objects. borrowed reference - In Python's C API, a borrowed reference is a reference to an object. - It does not modify the object reference count. It becomes a dangling + In Python's C API, a borrowed reference is a reference to an object, + where the code using the object does not own the reference. + It becomes a dangling pointer if the object is destroyed. For example, a garbage collection can remove the last :term:`strong reference` to the object and so destroy it. @@ -1142,8 +1143,10 @@ Glossary strong reference In Python's C API, a strong reference is a reference to an object - which increments the object's reference count when it is created and - decrements the object's reference count when it is deleted. + which is owned by the code holding the reference. The strong + reference is taken by calling :c:func:`Py_INCREF` when the + reference is created and released with :c:func:`Py_DECREF` + when the reference is deleted. The :c:func:`Py_NewRef` function can be used to create a strong reference to an object. Usually, the :c:func:`Py_DECREF` function must be called on diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 5a859b68ef805..d16f380b60184 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -709,6 +709,9 @@ always available. higher than you might expect, because it includes the (temporary) reference as an argument to :func:`getrefcount`. + Note that the returned value may not actually reflect how many + references to the object are actually held. Consequently, do not rely + on the returned value to be accurate, other than a value of 0 or 1. .. function:: getrecursionlimit() From webhook-mailer at python.org Fri Aug 11 10:12:54 2023 From: webhook-mailer at python.org (Yhg1s) Date: Fri, 11 Aug 2023 14:12:54 -0000 Subject: [Python-checkins] [3.12] gh-107810: Improve DeprecationWarning for metaclasses with custom tp_new (GH-107834) (#107864) Message-ID: https://github.com/python/cpython/commit/431ce239d2a906ec4d9a27a83549258d7cef3701 commit: 431ce239d2a906ec4d9a27a83549258d7cef3701 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-11T16:12:50+02:00 summary: [3.12] gh-107810: Improve DeprecationWarning for metaclasses with custom tp_new (GH-107834) (#107864) gh-107810: Improve DeprecationWarning for metaclasses with custom tp_new (GH-107834) (cherry picked from commit 16dcce21768ba381996a88ac8c255bf1490b3680) Co-authored-by: Marc Mueller <30130371+cdce8p at users.noreply.github.com> Co-authored-by: Kirill Podoprigora files: A Misc/NEWS.d/next/C API/2023-08-10-11-12-25.gh-issue-107810.oJ40Qx.rst M Lib/test/test_capi/test_misc.py M Objects/typeobject.c diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index cd37fc71aa966..3f2736da7d93a 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -740,7 +740,7 @@ class Base(metaclass=metaclass): # Class creation from C with warnings_helper.check_warnings( - ('.*custom tp_new.*in Python 3.14.*', DeprecationWarning), + ('.* _testcapi.Subclass .* custom tp_new.*in Python 3.14.*', DeprecationWarning), ): sub = _testcapi.make_type_with_base(Base) self.assertTrue(issubclass(sub, Base)) diff --git a/Misc/NEWS.d/next/C API/2023-08-10-11-12-25.gh-issue-107810.oJ40Qx.rst b/Misc/NEWS.d/next/C API/2023-08-10-11-12-25.gh-issue-107810.oJ40Qx.rst new file mode 100644 index 0000000000000..c8a1f6d122b61 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-10-11-12-25.gh-issue-107810.oJ40Qx.rst @@ -0,0 +1 @@ +Improve :exc:`DeprecationWarning` for uses of :c:type:`PyType_Spec` with metaclasses that have custom ``tp_new``. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 40e187d4d8d22..5c71c28f75150 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4242,9 +4242,9 @@ _PyType_FromMetaclass_impl( if (_allow_tp_new) { if (PyErr_WarnFormat( PyExc_DeprecationWarning, 1, - "Using PyType_Spec with metaclasses that have custom " - "tp_new is deprecated and will no longer be allowed in " - "Python 3.14.") < 0) { + "Type %s uses PyType_Spec with a metaclass that has custom " + "tp_new. This is deprecated and will no longer be allowed in " + "Python 3.14.", spec->name) < 0) { goto finally; } } From webhook-mailer at python.org Fri Aug 11 10:13:17 2023 From: webhook-mailer at python.org (Yhg1s) Date: Fri, 11 Aug 2023 14:13:17 -0000 Subject: [Python-checkins] [3.12] gh-91054: make code watcher tests resilient to other watchers (GH-107821) (#107835) Message-ID: https://github.com/python/cpython/commit/98dd9d9725e6fa137e14f00b2907fdec19aee4ac commit: 98dd9d9725e6fa137e14f00b2907fdec19aee4ac branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-11T16:13:14+02:00 summary: [3.12] gh-91054: make code watcher tests resilient to other watchers (GH-107821) (#107835) gh-91054: make code watcher tests resilient to other watchers (GH-107821) (cherry picked from commit 2ec16fed14aae896e38dd5bd9e73e2eddc974439) Co-authored-by: Carl Meyer files: M Modules/_testcapi/watchers.c M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 7167943fffab3..fd695df749070 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -295,6 +295,7 @@ _testcapi_unwatch_type_impl(PyObject *module, int watcher_id, PyObject *type) // Test code object watching #define NUM_CODE_WATCHERS 2 +static int code_watcher_ids[NUM_CODE_WATCHERS] = {-1, -1}; static int num_code_object_created_events[NUM_CODE_WATCHERS] = {0, 0}; static int num_code_object_destroyed_events[NUM_CODE_WATCHERS] = {0, 0}; @@ -345,11 +346,13 @@ add_code_watcher(PyObject *self, PyObject *which_watcher) long which_l = PyLong_AsLong(which_watcher); if (which_l == 0) { watcher_id = PyCode_AddWatcher(first_code_object_callback); + code_watcher_ids[0] = watcher_id; num_code_object_created_events[0] = 0; num_code_object_destroyed_events[0] = 0; } else if (which_l == 1) { watcher_id = PyCode_AddWatcher(second_code_object_callback); + code_watcher_ids[1] = watcher_id; num_code_object_created_events[1] = 0; num_code_object_destroyed_events[1] = 0; } @@ -375,9 +378,14 @@ clear_code_watcher(PyObject *self, PyObject *watcher_id) return NULL; } // reset static events counters - if (watcher_id_l >= 0 && watcher_id_l < NUM_CODE_WATCHERS) { - num_code_object_created_events[watcher_id_l] = 0; - num_code_object_destroyed_events[watcher_id_l] = 0; + if (watcher_id_l >= 0) { + for (int i = 0; i < NUM_CODE_WATCHERS; i++) { + if (watcher_id_l == code_watcher_ids[i]) { + code_watcher_ids[i] = -1; + num_code_object_created_events[i] = 0; + num_code_object_destroyed_events[i] = 0; + } + } } Py_RETURN_NONE; } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 78be97da41b50..6a7c14ebb220a 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -426,6 +426,7 @@ Modules/_testcapi/watchers.c - g_dict_watch_events - Modules/_testcapi/watchers.c - g_dict_watchers_installed - Modules/_testcapi/watchers.c - g_type_modified_events - Modules/_testcapi/watchers.c - g_type_watchers_installed - +Modules/_testcapi/watchers.c - code_watcher_ids - Modules/_testcapi/watchers.c - num_code_object_created_events - Modules/_testcapi/watchers.c - num_code_object_destroyed_events - Modules/_testcapi/watchers.c - pyfunc_watchers - From webhook-mailer at python.org Fri Aug 11 11:08:42 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Fri, 11 Aug 2023 15:08:42 -0000 Subject: [Python-checkins] gh-84805: Autogenerate signature for METH_NOARGS and METH_O extension functions (GH-107794) Message-ID: https://github.com/python/cpython/commit/3901c991e169da6fba8c0033a86a6f2e6146bb7f commit: 3901c991e169da6fba8c0033a86a6f2e6146bb7f branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-11T18:08:38+03:00 summary: gh-84805: Autogenerate signature for METH_NOARGS and METH_O extension functions (GH-107794) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-09-08-31-20.gh-issue-84805.7JRWua.rst M Include/internal/pycore_object.h M Lib/test/test_inspect.py M Lib/test/test_pydoc.py M Lib/test/test_rlcompleter.py M Modules/_testcapi/docstring.c M Objects/descrobject.c M Objects/methodobject.c M Objects/typeobject.c M Tools/c-analyzer/cpython/ignored.tsv diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 7cdf64bcdacb9..857d6efec3b3b 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -381,7 +381,7 @@ extern PyObject *_PyType_NewManagedObject(PyTypeObject *type); extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); -extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *); +extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); extern int _PyObject_InitializeDict(PyObject *obj); int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 5c31748ce2caa..07c48eac5b48b 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -13,7 +13,9 @@ import _pickle import pickle import shutil +import stat import sys +import time import types import tempfile import textwrap @@ -22,6 +24,7 @@ import unittest.mock import warnings + try: from concurrent.futures import ThreadPoolExecutor except ImportError: @@ -136,6 +139,14 @@ def gen_coroutine_function_example(self): yield return 'spam' +def meth_noargs(): pass +def meth_o(object, /): pass +def meth_self_noargs(self, /): pass +def meth_self_o(self, object, /): pass +def meth_type_noargs(type, /): pass +def meth_type_o(type, object, /): pass + + class TestPredicates(IsTestBase): def test_excluding_predicates(self): @@ -1173,6 +1184,39 @@ def test_getfullargspec_builtin_func_no_signature(self): with self.assertRaises(TypeError): inspect.getfullargspec(builtin) + cls = _testcapi.DocStringNoSignatureTest + obj = _testcapi.DocStringNoSignatureTest() + for builtin, template in [ + (_testcapi.docstring_no_signature_noargs, meth_noargs), + (_testcapi.docstring_no_signature_o, meth_o), + (cls.meth_noargs, meth_self_noargs), + (cls.meth_o, meth_self_o), + (obj.meth_noargs, meth_self_noargs), + (obj.meth_o, meth_self_o), + (cls.meth_noargs_class, meth_type_noargs), + (cls.meth_o_class, meth_type_o), + (cls.meth_noargs_static, meth_noargs), + (cls.meth_o_static, meth_o), + (cls.meth_noargs_coexist, meth_self_noargs), + (cls.meth_o_coexist, meth_self_o), + + (time.time, meth_noargs), + (stat.S_IMODE, meth_o), + (str.lower, meth_self_noargs), + (''.lower, meth_self_noargs), + (set.add, meth_self_o), + (set().add, meth_self_o), + (set.__contains__, meth_self_o), + (set().__contains__, meth_self_o), + (datetime.datetime.__dict__['utcnow'], meth_type_noargs), + (datetime.datetime.utcnow, meth_type_noargs), + (dict.__dict__['__class_getitem__'], meth_type_o), + (dict.__class_getitem__, meth_type_o), + ]: + with self.subTest(builtin): + self.assertEqual(inspect.getfullargspec(builtin), + inspect.getfullargspec(template)) + def test_getfullargspec_definition_order_preserved_on_kwonly(self): for fn in signatures_with_lexicographic_keyword_only_parameters(): signature = inspect.getfullargspec(fn) @@ -2888,6 +2932,39 @@ def test_signature_on_builtins_no_signature(self): 'no signature found for builtin'): inspect.signature(str) + cls = _testcapi.DocStringNoSignatureTest + obj = _testcapi.DocStringNoSignatureTest() + for builtin, template in [ + (_testcapi.docstring_no_signature_noargs, meth_noargs), + (_testcapi.docstring_no_signature_o, meth_o), + (cls.meth_noargs, meth_self_noargs), + (cls.meth_o, meth_self_o), + (obj.meth_noargs, meth_noargs), + (obj.meth_o, meth_o), + (cls.meth_noargs_class, meth_noargs), + (cls.meth_o_class, meth_o), + (cls.meth_noargs_static, meth_noargs), + (cls.meth_o_static, meth_o), + (cls.meth_noargs_coexist, meth_self_noargs), + (cls.meth_o_coexist, meth_self_o), + + (time.time, meth_noargs), + (stat.S_IMODE, meth_o), + (str.lower, meth_self_noargs), + (''.lower, meth_noargs), + (set.add, meth_self_o), + (set().add, meth_o), + (set.__contains__, meth_self_o), + (set().__contains__, meth_o), + (datetime.datetime.__dict__['utcnow'], meth_type_noargs), + (datetime.datetime.utcnow, meth_noargs), + (dict.__dict__['__class_getitem__'], meth_type_o), + (dict.__class_getitem__, meth_o), + ]: + with self.subTest(builtin): + self.assertEqual(inspect.signature(builtin), + inspect.signature(template)) + def test_signature_on_non_function(self): with self.assertRaisesRegex(TypeError, 'is not a callable object'): inspect.signature(42) diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index ddb5187f90da9..8df8b608cf959 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -1,3 +1,4 @@ +import datetime import os import sys import contextlib @@ -12,6 +13,7 @@ import stat import tempfile import test.support +import time import types import typing import unittest @@ -1180,6 +1182,54 @@ def test_module_level_callable(self): self.assertEqual(self._get_summary_line(os.stat), "stat(path, *, dir_fd=None, follow_symlinks=True)") + def test_module_level_callable_noargs(self): + self.assertEqual(self._get_summary_line(time.time), + "time()") + + def test_module_level_callable_o(self): + self.assertEqual(self._get_summary_line(stat.S_IMODE), + "S_IMODE(object, /)") + + def test_unbound_builtin_method_noargs(self): + self.assertEqual(self._get_summary_line(str.lower), + "lower(self, /)") + + def test_bound_builtin_method_noargs(self): + self.assertEqual(self._get_summary_line(''.lower), + "lower() method of builtins.str instance") + + def test_unbound_builtin_method_o(self): + self.assertEqual(self._get_summary_line(set.add), + "add(self, object, /)") + + def test_bound_builtin_method_o(self): + self.assertEqual(self._get_summary_line(set().add), + "add(object, /) method of builtins.set instance") + + def test_unbound_builtin_method_coexist_o(self): + self.assertEqual(self._get_summary_line(set.__contains__), + "__contains__(self, object, /)") + + def test_bound_builtin_method_coexist_o(self): + self.assertEqual(self._get_summary_line(set().__contains__), + "__contains__(object, /) method of builtins.set instance") + + def test_unbound_builtin_classmethod_noargs(self): + self.assertEqual(self._get_summary_line(datetime.datetime.__dict__['utcnow']), + "utcnow(type, /)") + + def test_bound_builtin_classmethod_noargs(self): + self.assertEqual(self._get_summary_line(datetime.datetime.utcnow), + "utcnow() method of builtins.type instance") + + def test_unbound_builtin_classmethod_o(self): + self.assertEqual(self._get_summary_line(dict.__dict__['__class_getitem__']), + "__class_getitem__(type, object, /)") + + def test_bound_builtin_classmethod_o(self): + self.assertEqual(self._get_summary_line(dict.__class_getitem__), + "__class_getitem__(object, /) method of builtins.type instance") + @requires_docstrings def test_staticmethod(self): class X: diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 6b5fc9a0247f4..7347fca71be2f 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -53,7 +53,10 @@ def test_attr_matches(self): ['str.{}('.format(x) for x in dir(str) if x.startswith('s')]) self.assertEqual(self.stdcompleter.attr_matches('tuple.foospamegg'), []) - expected = sorted({'None.%s%s' % (x, '(' if x != '__doc__' else '') + expected = sorted({'None.%s%s' % (x, + '()' if x == '__init_subclass__' + else '' if x == '__doc__' + else '(') for x in dir(None)}) self.assertEqual(self.stdcompleter.attr_matches('None.'), expected) self.assertEqual(self.stdcompleter.attr_matches('None._'), expected) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-09-08-31-20.gh-issue-84805.7JRWua.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-09-08-31-20.gh-issue-84805.7JRWua.rst new file mode 100644 index 0000000000000..23dfba989fa55 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-09-08-31-20.gh-issue-84805.7JRWua.rst @@ -0,0 +1,2 @@ +Autogenerate signature for :c:macro:`METH_NOARGS` and :c:macro:`METH_O` +extension functions. diff --git a/Modules/_testcapi/docstring.c b/Modules/_testcapi/docstring.c index a997c54a8a69c..b680171cc1437 100644 --- a/Modules/_testcapi/docstring.c +++ b/Modules/_testcapi/docstring.c @@ -66,42 +66,88 @@ test_with_docstring(PyObject *self, PyObject *Py_UNUSED(ignored)) static PyMethodDef test_methods[] = { {"docstring_empty", - (PyCFunction)test_with_docstring, METH_NOARGS, + (PyCFunction)test_with_docstring, METH_VARARGS, docstring_empty}, {"docstring_no_signature", + (PyCFunction)test_with_docstring, METH_VARARGS, + docstring_no_signature}, + {"docstring_no_signature_noargs", (PyCFunction)test_with_docstring, METH_NOARGS, docstring_no_signature}, + {"docstring_no_signature_o", + (PyCFunction)test_with_docstring, METH_O, + docstring_no_signature}, {"docstring_with_invalid_signature", - (PyCFunction)test_with_docstring, METH_NOARGS, + (PyCFunction)test_with_docstring, METH_VARARGS, docstring_with_invalid_signature}, {"docstring_with_invalid_signature2", - (PyCFunction)test_with_docstring, METH_NOARGS, + (PyCFunction)test_with_docstring, METH_VARARGS, docstring_with_invalid_signature2}, {"docstring_with_signature", - (PyCFunction)test_with_docstring, METH_NOARGS, + (PyCFunction)test_with_docstring, METH_VARARGS, docstring_with_signature}, {"docstring_with_signature_and_extra_newlines", - (PyCFunction)test_with_docstring, METH_NOARGS, + (PyCFunction)test_with_docstring, METH_VARARGS, docstring_with_signature_and_extra_newlines}, {"docstring_with_signature_but_no_doc", - (PyCFunction)test_with_docstring, METH_NOARGS, + (PyCFunction)test_with_docstring, METH_VARARGS, docstring_with_signature_but_no_doc}, {"docstring_with_signature_with_defaults", - (PyCFunction)test_with_docstring, METH_NOARGS, + (PyCFunction)test_with_docstring, METH_VARARGS, docstring_with_signature_with_defaults}, {"no_docstring", - (PyCFunction)test_with_docstring, METH_NOARGS}, + (PyCFunction)test_with_docstring, METH_VARARGS}, {"test_with_docstring", - test_with_docstring, METH_NOARGS, + test_with_docstring, METH_VARARGS, PyDoc_STR("This is a pretty normal docstring.")}, {NULL}, }; +static PyMethodDef DocStringNoSignatureTest_methods[] = { + {"meth_noargs", + (PyCFunction)test_with_docstring, METH_NOARGS, + docstring_no_signature}, + {"meth_o", + (PyCFunction)test_with_docstring, METH_O, + docstring_no_signature}, + {"meth_noargs_class", + (PyCFunction)test_with_docstring, METH_NOARGS|METH_CLASS, + docstring_no_signature}, + {"meth_o_class", + (PyCFunction)test_with_docstring, METH_O|METH_CLASS, + docstring_no_signature}, + {"meth_noargs_static", + (PyCFunction)test_with_docstring, METH_NOARGS|METH_STATIC, + docstring_no_signature}, + {"meth_o_static", + (PyCFunction)test_with_docstring, METH_O|METH_STATIC, + docstring_no_signature}, + {"meth_noargs_coexist", + (PyCFunction)test_with_docstring, METH_NOARGS|METH_COEXIST, + docstring_no_signature}, + {"meth_o_coexist", + (PyCFunction)test_with_docstring, METH_O|METH_COEXIST, + docstring_no_signature}, + {NULL}, +}; + +static PyTypeObject DocStringNoSignatureTest = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testcapi.DocStringNoSignatureTest", + .tp_basicsize = sizeof(PyObject), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_methods = DocStringNoSignatureTest_methods, + .tp_new = PyType_GenericNew, +}; + int _PyTestCapi_Init_Docstring(PyObject *mod) { if (PyModule_AddFunctions(mod, test_methods) < 0) { return -1; } + if (PyModule_AddType(mod, &DocStringNoSignatureTest) < 0) { + return -1; + } return 0; } diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 60383dd06d1ad..a744c3d1e5865 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -588,7 +588,9 @@ method_get_doc(PyMethodDescrObject *descr, void *closure) static PyObject * method_get_text_signature(PyMethodDescrObject *descr, void *closure) { - return _PyType_GetTextSignatureFromInternalDoc(descr->d_method->ml_name, descr->d_method->ml_doc); + return _PyType_GetTextSignatureFromInternalDoc(descr->d_method->ml_name, + descr->d_method->ml_doc, + descr->d_method->ml_flags); } static PyObject * @@ -691,7 +693,8 @@ wrapperdescr_get_doc(PyWrapperDescrObject *descr, void *closure) static PyObject * wrapperdescr_get_text_signature(PyWrapperDescrObject *descr, void *closure) { - return _PyType_GetTextSignatureFromInternalDoc(descr->d_base->name, descr->d_base->doc); + return _PyType_GetTextSignatureFromInternalDoc(descr->d_base->name, + descr->d_base->doc, 0); } static PyGetSetDef wrapperdescr_getset[] = { @@ -1384,7 +1387,8 @@ wrapper_doc(wrapperobject *wp, void *Py_UNUSED(ignored)) static PyObject * wrapper_text_signature(wrapperobject *wp, void *Py_UNUSED(ignored)) { - return _PyType_GetTextSignatureFromInternalDoc(wp->descr->d_base->name, wp->descr->d_base->doc); + return _PyType_GetTextSignatureFromInternalDoc(wp->descr->d_base->name, + wp->descr->d_base->doc, 0); } static PyObject * diff --git a/Objects/methodobject.c b/Objects/methodobject.c index 628d227ef33c3..521c9059770ac 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -192,7 +192,9 @@ static PyMethodDef meth_methods[] = { static PyObject * meth_get__text_signature__(PyCFunctionObject *m, void *closure) { - return _PyType_GetTextSignatureFromInternalDoc(m->m_ml->ml_name, m->m_ml->ml_doc); + return _PyType_GetTextSignatureFromInternalDoc(m->m_ml->ml_name, + m->m_ml->ml_doc, + m->m_ml->ml_flags); } static PyObject * diff --git a/Objects/typeobject.c b/Objects/typeobject.c index aca14e79a1997..030e8bfc99b6d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -586,8 +586,29 @@ _PyType_GetDocFromInternalDoc(const char *name, const char *internal_doc) return PyUnicode_FromString(doc); } +static const char * +signature_from_flags(int flags) +{ + switch (flags & ~METH_COEXIST) { + case METH_NOARGS: + return "($self, /)"; + case METH_NOARGS|METH_CLASS: + return "($type, /)"; + case METH_NOARGS|METH_STATIC: + return "()"; + case METH_O: + return "($self, object, /)"; + case METH_O|METH_CLASS: + return "($type, object, /)"; + case METH_O|METH_STATIC: + return "(object, /)"; + default: + return NULL; + } +} + PyObject * -_PyType_GetTextSignatureFromInternalDoc(const char *name, const char *internal_doc) +_PyType_GetTextSignatureFromInternalDoc(const char *name, const char *internal_doc, int flags) { const char *start = find_signature(name, internal_doc); const char *end; @@ -597,6 +618,10 @@ _PyType_GetTextSignatureFromInternalDoc(const char *name, const char *internal_d else end = NULL; if (!end) { + start = signature_from_flags(flags); + if (start) { + return PyUnicode_FromString(start); + } Py_RETURN_NONE; } @@ -1429,7 +1454,7 @@ type_get_doc(PyTypeObject *type, void *context) static PyObject * type_get_text_signature(PyTypeObject *type, void *context) { - return _PyType_GetTextSignatureFromInternalDoc(type->tp_name, type->tp_doc); + return _PyType_GetTextSignatureFromInternalDoc(type->tp_name, type->tp_doc, 0); } static int diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 66815c72ffbc6..c64d391bae13b 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -419,6 +419,7 @@ Modules/_testbuffer.c staticarray_init kwlist - Modules/_testcapi/buffer.c - testBufType - Modules/_testcapi/code.c get_code_extra_index key - Modules/_testcapi/datetime.c - test_run_counter - +Modules/_testcapi/docstring.c - DocStringNoSignatureTest - Modules/_testcapi/exceptions.c - PyRecursingInfinitelyError_Type - Modules/_testcapi/heaptype.c - _testcapimodule - Modules/_testcapi/mem.c - FmData - From webhook-mailer at python.org Fri Aug 11 12:03:56 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Fri, 11 Aug 2023 16:03:56 -0000 Subject: [Python-checkins] Docs: Document PyBUF_MAX_NDIM (#107865) Message-ID: https://github.com/python/cpython/commit/637f7ff2c60f262659da0334f1cb672bd361f398 commit: 637f7ff2c60f262659da0334f1cb672bd361f398 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-11T18:03:53+02:00 summary: Docs: Document PyBUF_MAX_NDIM (#107865) files: M Doc/c-api/buffer.rst diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 8ca1c190dab9a..ba391a5279f20 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -161,10 +161,14 @@ a buffer, see :c:func:`PyObject_GetBuffer`. If it is ``0``, :c:member:`~Py_buffer.buf` points to a single item representing a scalar. In this case, :c:member:`~Py_buffer.shape`, :c:member:`~Py_buffer.strides` and :c:member:`~Py_buffer.suboffsets` MUST be ``NULL``. + The maximum number of dimensions is given by :c:macro:`PyBUF_MAX_NDIM`. - The macro :c:macro:`PyBUF_MAX_NDIM` limits the maximum number of dimensions - to 64. Exporters MUST respect this limit, consumers of multi-dimensional - buffers SHOULD be able to handle up to :c:macro:`PyBUF_MAX_NDIM` dimensions. + .. :c:macro:: PyBUF_MAX_NDIM + + The maximum number of dimensions the memory represents. + Exporters MUST respect this limit, consumers of multi-dimensional + buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions. + Currently set to 64. .. c:member:: Py_ssize_t *shape From webhook-mailer at python.org Fri Aug 11 12:13:56 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Fri, 11 Aug 2023 16:13:56 -0000 Subject: [Python-checkins] [3.11] Docs: Document PyBUF_MAX_NDIM (GH-107865) (#107872) Message-ID: https://github.com/python/cpython/commit/4ddfb042609bebe7fd1d9af5f5d7bbc781ea5b4f commit: 4ddfb042609bebe7fd1d9af5f5d7bbc781ea5b4f branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: erlend-aasland date: 2023-08-11T16:13:53Z summary: [3.11] Docs: Document PyBUF_MAX_NDIM (GH-107865) (#107872) (cherry picked from commit 637f7ff2c60f262659da0334f1cb672bd361f398) Co-authored-by: Erlend E. Aasland files: M Doc/c-api/buffer.rst diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 8ca1c190dab9a..ba391a5279f20 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -161,10 +161,14 @@ a buffer, see :c:func:`PyObject_GetBuffer`. If it is ``0``, :c:member:`~Py_buffer.buf` points to a single item representing a scalar. In this case, :c:member:`~Py_buffer.shape`, :c:member:`~Py_buffer.strides` and :c:member:`~Py_buffer.suboffsets` MUST be ``NULL``. + The maximum number of dimensions is given by :c:macro:`PyBUF_MAX_NDIM`. - The macro :c:macro:`PyBUF_MAX_NDIM` limits the maximum number of dimensions - to 64. Exporters MUST respect this limit, consumers of multi-dimensional - buffers SHOULD be able to handle up to :c:macro:`PyBUF_MAX_NDIM` dimensions. + .. :c:macro:: PyBUF_MAX_NDIM + + The maximum number of dimensions the memory represents. + Exporters MUST respect this limit, consumers of multi-dimensional + buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions. + Currently set to 64. .. c:member:: Py_ssize_t *shape From webhook-mailer at python.org Fri Aug 11 12:19:23 2023 From: webhook-mailer at python.org (rhettinger) Date: Fri, 11 Aug 2023 16:19:23 -0000 Subject: [Python-checkins] Extend _sqrtprod() to cover the full range of inputs. Add tests. (GH-107855) Message-ID: https://github.com/python/cpython/commit/52e0797f8e1c631eecf24cb3f997ace336f52271 commit: 52e0797f8e1c631eecf24cb3f997ace336f52271 branch: main author: Raymond Hettinger committer: rhettinger date: 2023-08-11T11:19:19-05:00 summary: Extend _sqrtprod() to cover the full range of inputs. Add tests. (GH-107855) files: M Lib/statistics.py M Lib/test/test_statistics.py diff --git a/Lib/statistics.py b/Lib/statistics.py index 93c44f1f13fab..a8036e9928c46 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -137,6 +137,7 @@ from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod +from math import isfinite, isinf from functools import reduce from operator import itemgetter from collections import Counter, namedtuple, defaultdict @@ -1005,14 +1006,25 @@ def _mean_stdev(data): return float(xbar), float(xbar) / float(ss) def _sqrtprod(x: float, y: float) -> float: - "Return sqrt(x * y) computed with high accuracy." - # Square root differential correction: - # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 + "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." h = sqrt(x * y) + if not isfinite(h): + if isinf(h) and not isinf(x) and not isinf(y): + # Finite inputs overflowed, so scale down, and recompute. + scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) + return _sqrtprod(scale * x, scale * y) / scale + return h if not h: - return 0.0 - x = sumprod((x, h), (y, -h)) - return h + x / (2.0 * h) + if x and y: + # Non-zero inputs underflowed, so scale up, and recompute. + # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) + scale = 2.0 ** 537 + return _sqrtprod(scale * x, scale * y) / scale + return h + # Improve accuracy with a differential correction. + # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 + d = sumprod((x, h), (y, -h)) + return h + d / (2.0 * h) # === Statistics for relations between two inputs === diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index f0fa6454b1f91..aa2cf2b1edc58 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -28,6 +28,12 @@ # === Helper functions and class === +# Test copied from Lib/test/test_math.py +# detect evidence of double-rounding: fsum is not always correctly +# rounded on machines that suffer from double rounding. +x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer +HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) + def sign(x): """Return -1.0 for negatives, including -0.0, otherwise +1.0.""" return math.copysign(1, x) @@ -2564,6 +2570,79 @@ def test_different_scales(self): self.assertAlmostEqual(statistics.correlation(x, y), 1) self.assertAlmostEqual(statistics.covariance(x, y), 0.1) + def test_sqrtprod_helper_function_fundamentals(self): + # Verify that results are close to sqrt(x * y) + for i in range(100): + x = random.expovariate() + y = random.expovariate() + expected = math.sqrt(x * y) + actual = statistics._sqrtprod(x, y) + with self.subTest(x=x, y=y, expected=expected, actual=actual): + self.assertAlmostEqual(expected, actual) + + x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 + self.assertEqual(statistics._sqrtprod(x, y), target) + self.assertNotEqual(math.sqrt(x * y), target) + + # Test that range extremes avoid underflow and overflow + smallest = sys.float_info.min * sys.float_info.epsilon + self.assertEqual(statistics._sqrtprod(smallest, smallest), smallest) + biggest = sys.float_info.max + self.assertEqual(statistics._sqrtprod(biggest, biggest), biggest) + + # Check special values and the sign of the result + special_values = [0.0, -0.0, 1.0, -1.0, 4.0, -4.0, + math.nan, -math.nan, math.inf, -math.inf] + for x, y in itertools.product(special_values, repeat=2): + try: + expected = math.sqrt(x * y) + except ValueError: + expected = 'ValueError' + try: + actual = statistics._sqrtprod(x, y) + except ValueError: + actual = 'ValueError' + with self.subTest(x=x, y=y, expected=expected, actual=actual): + if isinstance(expected, str) and expected == 'ValueError': + self.assertEqual(actual, 'ValueError') + continue + self.assertIsInstance(actual, float) + if math.isnan(expected): + self.assertTrue(math.isnan(actual)) + continue + self.assertEqual(actual, expected) + self.assertEqual(sign(actual), sign(expected)) + + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Allow for a weaker sumprod() implmentation + def test_sqrtprod_helper_function_improved_accuracy(self): + # Test a known example where accuracy is improved + x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 + self.assertEqual(statistics._sqrtprod(x, y), target) + self.assertNotEqual(math.sqrt(x * y), target) + + def reference_value(x: float, y: float) -> float: + x = decimal.Decimal(x) + y = decimal.Decimal(y) + with decimal.localcontext() as ctx: + ctx.prec = 200 + return float((x * y).sqrt()) + + # Verify that the new function with improved accuracy + # agrees with a reference value more often than old version. + new_agreements = 0 + old_agreements = 0 + for i in range(10_000): + x = random.expovariate() + y = random.expovariate() + new = statistics._sqrtprod(x, y) + old = math.sqrt(x * y) + ref = reference_value(x, y) + new_agreements += (new == ref) + old_agreements += (old == ref) + self.assertGreater(new_agreements, old_agreements) def test_correlation_spearman(self): # https://statistics.laerd.com/statistical-guides/spearmans-rank-order-correlation-statistical-guide-2.php From webhook-mailer at python.org Fri Aug 11 12:42:05 2023 From: webhook-mailer at python.org (iritkatriel) Date: Fri, 11 Aug 2023 16:42:05 -0000 Subject: [Python-checkins] gh-105481: split opcode_ids.h out of opcode.h so that it can be generated separately (#107866) Message-ID: https://github.com/python/cpython/commit/caa41a4f1db0112690cf610bab7d9c6dce9ff1ce commit: caa41a4f1db0112690cf610bab7d9c6dce9ff1ce branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-11T17:42:01+01:00 summary: gh-105481: split opcode_ids.h out of opcode.h so that it can be generated separately (#107866) files: A Include/opcode_ids.h M .gitattributes M Include/opcode.h M Makefile.pre.in M PCbuild/regen.targets M Tools/build/generate_opcode_h.py diff --git a/.gitattributes b/.gitattributes index 1f2f8a4cd8735..4a072bc990f0f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -75,6 +75,7 @@ Include/internal/pycore_opcode.h generated Include/internal/pycore_opcode_metadata.h generated Include/internal/pycore_*_generated.h generated Include/opcode.h generated +Include/opcode_ids.h generated Include/token.h generated Lib/_opcode_metadata.py generated Lib/keyword.py generated diff --git a/Include/opcode.h b/Include/opcode.h index b3d6cba63096c..e5c42d5a71828 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -6,227 +6,8 @@ extern "C" { #endif +#include "opcode_ids.h" -/* Instruction opcodes for compiled code */ -#define CACHE 0 -#define POP_TOP 1 -#define PUSH_NULL 2 -#define INTERPRETER_EXIT 3 -#define END_FOR 4 -#define END_SEND 5 -#define TO_BOOL 6 -#define NOP 9 -#define UNARY_NEGATIVE 11 -#define UNARY_NOT 12 -#define UNARY_INVERT 15 -#define EXIT_INIT_CHECK 16 -#define RESERVED 17 -#define MAKE_FUNCTION 24 -#define BINARY_SUBSCR 25 -#define BINARY_SLICE 26 -#define STORE_SLICE 27 -#define GET_LEN 30 -#define MATCH_MAPPING 31 -#define MATCH_SEQUENCE 32 -#define MATCH_KEYS 33 -#define PUSH_EXC_INFO 35 -#define CHECK_EXC_MATCH 36 -#define CHECK_EG_MATCH 37 -#define FORMAT_SIMPLE 40 -#define FORMAT_WITH_SPEC 41 -#define WITH_EXCEPT_START 49 -#define GET_AITER 50 -#define GET_ANEXT 51 -#define BEFORE_ASYNC_WITH 52 -#define BEFORE_WITH 53 -#define END_ASYNC_FOR 54 -#define CLEANUP_THROW 55 -#define STORE_SUBSCR 60 -#define DELETE_SUBSCR 61 -#define GET_ITER 68 -#define GET_YIELD_FROM_ITER 69 -#define LOAD_BUILD_CLASS 71 -#define LOAD_ASSERTION_ERROR 74 -#define RETURN_GENERATOR 75 -#define RETURN_VALUE 83 -#define SETUP_ANNOTATIONS 85 -#define LOAD_LOCALS 87 -#define POP_EXCEPT 89 -#define STORE_NAME 90 -#define DELETE_NAME 91 -#define UNPACK_SEQUENCE 92 -#define FOR_ITER 93 -#define UNPACK_EX 94 -#define STORE_ATTR 95 -#define DELETE_ATTR 96 -#define STORE_GLOBAL 97 -#define DELETE_GLOBAL 98 -#define SWAP 99 -#define LOAD_CONST 100 -#define LOAD_NAME 101 -#define BUILD_TUPLE 102 -#define BUILD_LIST 103 -#define BUILD_SET 104 -#define BUILD_MAP 105 -#define LOAD_ATTR 106 -#define COMPARE_OP 107 -#define IMPORT_NAME 108 -#define IMPORT_FROM 109 -#define JUMP_FORWARD 110 -#define POP_JUMP_IF_FALSE 114 -#define POP_JUMP_IF_TRUE 115 -#define LOAD_GLOBAL 116 -#define IS_OP 117 -#define CONTAINS_OP 118 -#define RERAISE 119 -#define COPY 120 -#define RETURN_CONST 121 -#define BINARY_OP 122 -#define SEND 123 -#define LOAD_FAST 124 -#define STORE_FAST 125 -#define DELETE_FAST 126 -#define LOAD_FAST_CHECK 127 -#define POP_JUMP_IF_NOT_NONE 128 -#define POP_JUMP_IF_NONE 129 -#define RAISE_VARARGS 130 -#define GET_AWAITABLE 131 -#define BUILD_SLICE 133 -#define JUMP_BACKWARD_NO_INTERRUPT 134 -#define MAKE_CELL 135 -#define LOAD_DEREF 137 -#define STORE_DEREF 138 -#define DELETE_DEREF 139 -#define JUMP_BACKWARD 140 -#define LOAD_SUPER_ATTR 141 -#define CALL_FUNCTION_EX 142 -#define LOAD_FAST_AND_CLEAR 143 -#define EXTENDED_ARG 144 -#define LIST_APPEND 145 -#define SET_ADD 146 -#define MAP_ADD 147 -#define COPY_FREE_VARS 149 -#define YIELD_VALUE 150 -#define RESUME 151 -#define MATCH_CLASS 152 -#define BUILD_CONST_KEY_MAP 156 -#define BUILD_STRING 157 -#define CONVERT_VALUE 158 -#define LIST_EXTEND 162 -#define SET_UPDATE 163 -#define DICT_MERGE 164 -#define DICT_UPDATE 165 -#define LOAD_FAST_LOAD_FAST 168 -#define STORE_FAST_LOAD_FAST 169 -#define STORE_FAST_STORE_FAST 170 -#define CALL 171 -#define KW_NAMES 172 -#define CALL_INTRINSIC_1 173 -#define CALL_INTRINSIC_2 174 -#define LOAD_FROM_DICT_OR_GLOBALS 175 -#define LOAD_FROM_DICT_OR_DEREF 176 -#define SET_FUNCTION_ATTRIBUTE 177 -#define ENTER_EXECUTOR 230 -#define MIN_INSTRUMENTED_OPCODE 237 -#define INSTRUMENTED_LOAD_SUPER_ATTR 237 -#define INSTRUMENTED_POP_JUMP_IF_NONE 238 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 239 -#define INSTRUMENTED_RESUME 240 -#define INSTRUMENTED_CALL 241 -#define INSTRUMENTED_RETURN_VALUE 242 -#define INSTRUMENTED_YIELD_VALUE 243 -#define INSTRUMENTED_CALL_FUNCTION_EX 244 -#define INSTRUMENTED_JUMP_FORWARD 245 -#define INSTRUMENTED_JUMP_BACKWARD 246 -#define INSTRUMENTED_RETURN_CONST 247 -#define INSTRUMENTED_FOR_ITER 248 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 249 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 250 -#define INSTRUMENTED_END_FOR 251 -#define INSTRUMENTED_END_SEND 252 -#define INSTRUMENTED_INSTRUCTION 253 -#define INSTRUMENTED_LINE 254 -#define SETUP_FINALLY 256 -#define SETUP_CLEANUP 257 -#define SETUP_WITH 258 -#define POP_BLOCK 259 -#define JUMP 260 -#define JUMP_NO_INTERRUPT 261 -#define LOAD_METHOD 262 -#define LOAD_SUPER_METHOD 263 -#define LOAD_ZERO_SUPER_METHOD 264 -#define LOAD_ZERO_SUPER_ATTR 265 -#define STORE_FAST_MAYBE_NULL 266 -#define LOAD_CLOSURE 267 -#define TO_BOOL_ALWAYS_TRUE 7 -#define TO_BOOL_BOOL 8 -#define TO_BOOL_INT 10 -#define TO_BOOL_LIST 13 -#define TO_BOOL_NONE 14 -#define TO_BOOL_STR 18 -#define BINARY_OP_MULTIPLY_INT 19 -#define BINARY_OP_ADD_INT 20 -#define BINARY_OP_SUBTRACT_INT 21 -#define BINARY_OP_MULTIPLY_FLOAT 22 -#define BINARY_OP_ADD_FLOAT 23 -#define BINARY_OP_SUBTRACT_FLOAT 28 -#define BINARY_OP_ADD_UNICODE 29 -#define BINARY_OP_INPLACE_ADD_UNICODE 34 -#define BINARY_SUBSCR_DICT 38 -#define BINARY_SUBSCR_GETITEM 39 -#define BINARY_SUBSCR_LIST_INT 42 -#define BINARY_SUBSCR_STR_INT 43 -#define BINARY_SUBSCR_TUPLE_INT 44 -#define STORE_SUBSCR_DICT 45 -#define STORE_SUBSCR_LIST_INT 46 -#define SEND_GEN 47 -#define UNPACK_SEQUENCE_TWO_TUPLE 48 -#define UNPACK_SEQUENCE_TUPLE 56 -#define UNPACK_SEQUENCE_LIST 57 -#define STORE_ATTR_INSTANCE_VALUE 58 -#define STORE_ATTR_SLOT 59 -#define STORE_ATTR_WITH_HINT 62 -#define LOAD_GLOBAL_MODULE 63 -#define LOAD_GLOBAL_BUILTIN 64 -#define LOAD_SUPER_ATTR_ATTR 65 -#define LOAD_SUPER_ATTR_METHOD 66 -#define LOAD_ATTR_INSTANCE_VALUE 67 -#define LOAD_ATTR_MODULE 70 -#define LOAD_ATTR_WITH_HINT 72 -#define LOAD_ATTR_SLOT 73 -#define LOAD_ATTR_CLASS 76 -#define LOAD_ATTR_PROPERTY 77 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 78 -#define LOAD_ATTR_METHOD_WITH_VALUES 79 -#define LOAD_ATTR_METHOD_NO_DICT 80 -#define LOAD_ATTR_METHOD_LAZY_DICT 81 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 82 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 84 -#define COMPARE_OP_FLOAT 86 -#define COMPARE_OP_INT 88 -#define COMPARE_OP_STR 111 -#define FOR_ITER_LIST 112 -#define FOR_ITER_TUPLE 113 -#define FOR_ITER_RANGE 132 -#define FOR_ITER_GEN 136 -#define CALL_BOUND_METHOD_EXACT_ARGS 148 -#define CALL_PY_EXACT_ARGS 153 -#define CALL_PY_WITH_DEFAULTS 154 -#define CALL_NO_KW_TYPE_1 155 -#define CALL_NO_KW_STR_1 159 -#define CALL_NO_KW_TUPLE_1 160 -#define CALL_BUILTIN_CLASS 161 -#define CALL_NO_KW_BUILTIN_O 166 -#define CALL_NO_KW_BUILTIN_FAST 167 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 178 -#define CALL_NO_KW_LEN 179 -#define CALL_NO_KW_ISINSTANCE 180 -#define CALL_NO_KW_LIST_APPEND 181 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 182 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 183 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 184 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 185 -#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 186 #define NB_ADD 0 #define NB_AND 1 diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h new file mode 100644 index 0000000000000..2d9d24cca4542 --- /dev/null +++ b/Include/opcode_ids.h @@ -0,0 +1,235 @@ +// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py + +#ifndef Py_OPCODE_IDS_H +#define Py_OPCODE_IDS_H +#ifdef __cplusplus +extern "C" { +#endif + + +/* Instruction opcodes for compiled code */ +#define CACHE 0 +#define POP_TOP 1 +#define PUSH_NULL 2 +#define INTERPRETER_EXIT 3 +#define END_FOR 4 +#define END_SEND 5 +#define TO_BOOL 6 +#define NOP 9 +#define UNARY_NEGATIVE 11 +#define UNARY_NOT 12 +#define UNARY_INVERT 15 +#define EXIT_INIT_CHECK 16 +#define RESERVED 17 +#define MAKE_FUNCTION 24 +#define BINARY_SUBSCR 25 +#define BINARY_SLICE 26 +#define STORE_SLICE 27 +#define GET_LEN 30 +#define MATCH_MAPPING 31 +#define MATCH_SEQUENCE 32 +#define MATCH_KEYS 33 +#define PUSH_EXC_INFO 35 +#define CHECK_EXC_MATCH 36 +#define CHECK_EG_MATCH 37 +#define FORMAT_SIMPLE 40 +#define FORMAT_WITH_SPEC 41 +#define WITH_EXCEPT_START 49 +#define GET_AITER 50 +#define GET_ANEXT 51 +#define BEFORE_ASYNC_WITH 52 +#define BEFORE_WITH 53 +#define END_ASYNC_FOR 54 +#define CLEANUP_THROW 55 +#define STORE_SUBSCR 60 +#define DELETE_SUBSCR 61 +#define GET_ITER 68 +#define GET_YIELD_FROM_ITER 69 +#define LOAD_BUILD_CLASS 71 +#define LOAD_ASSERTION_ERROR 74 +#define RETURN_GENERATOR 75 +#define RETURN_VALUE 83 +#define SETUP_ANNOTATIONS 85 +#define LOAD_LOCALS 87 +#define POP_EXCEPT 89 +#define STORE_NAME 90 +#define DELETE_NAME 91 +#define UNPACK_SEQUENCE 92 +#define FOR_ITER 93 +#define UNPACK_EX 94 +#define STORE_ATTR 95 +#define DELETE_ATTR 96 +#define STORE_GLOBAL 97 +#define DELETE_GLOBAL 98 +#define SWAP 99 +#define LOAD_CONST 100 +#define LOAD_NAME 101 +#define BUILD_TUPLE 102 +#define BUILD_LIST 103 +#define BUILD_SET 104 +#define BUILD_MAP 105 +#define LOAD_ATTR 106 +#define COMPARE_OP 107 +#define IMPORT_NAME 108 +#define IMPORT_FROM 109 +#define JUMP_FORWARD 110 +#define POP_JUMP_IF_FALSE 114 +#define POP_JUMP_IF_TRUE 115 +#define LOAD_GLOBAL 116 +#define IS_OP 117 +#define CONTAINS_OP 118 +#define RERAISE 119 +#define COPY 120 +#define RETURN_CONST 121 +#define BINARY_OP 122 +#define SEND 123 +#define LOAD_FAST 124 +#define STORE_FAST 125 +#define DELETE_FAST 126 +#define LOAD_FAST_CHECK 127 +#define POP_JUMP_IF_NOT_NONE 128 +#define POP_JUMP_IF_NONE 129 +#define RAISE_VARARGS 130 +#define GET_AWAITABLE 131 +#define BUILD_SLICE 133 +#define JUMP_BACKWARD_NO_INTERRUPT 134 +#define MAKE_CELL 135 +#define LOAD_DEREF 137 +#define STORE_DEREF 138 +#define DELETE_DEREF 139 +#define JUMP_BACKWARD 140 +#define LOAD_SUPER_ATTR 141 +#define CALL_FUNCTION_EX 142 +#define LOAD_FAST_AND_CLEAR 143 +#define EXTENDED_ARG 144 +#define LIST_APPEND 145 +#define SET_ADD 146 +#define MAP_ADD 147 +#define COPY_FREE_VARS 149 +#define YIELD_VALUE 150 +#define RESUME 151 +#define MATCH_CLASS 152 +#define BUILD_CONST_KEY_MAP 156 +#define BUILD_STRING 157 +#define CONVERT_VALUE 158 +#define LIST_EXTEND 162 +#define SET_UPDATE 163 +#define DICT_MERGE 164 +#define DICT_UPDATE 165 +#define LOAD_FAST_LOAD_FAST 168 +#define STORE_FAST_LOAD_FAST 169 +#define STORE_FAST_STORE_FAST 170 +#define CALL 171 +#define KW_NAMES 172 +#define CALL_INTRINSIC_1 173 +#define CALL_INTRINSIC_2 174 +#define LOAD_FROM_DICT_OR_GLOBALS 175 +#define LOAD_FROM_DICT_OR_DEREF 176 +#define SET_FUNCTION_ATTRIBUTE 177 +#define ENTER_EXECUTOR 230 +#define MIN_INSTRUMENTED_OPCODE 237 +#define INSTRUMENTED_LOAD_SUPER_ATTR 237 +#define INSTRUMENTED_POP_JUMP_IF_NONE 238 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 239 +#define INSTRUMENTED_RESUME 240 +#define INSTRUMENTED_CALL 241 +#define INSTRUMENTED_RETURN_VALUE 242 +#define INSTRUMENTED_YIELD_VALUE 243 +#define INSTRUMENTED_CALL_FUNCTION_EX 244 +#define INSTRUMENTED_JUMP_FORWARD 245 +#define INSTRUMENTED_JUMP_BACKWARD 246 +#define INSTRUMENTED_RETURN_CONST 247 +#define INSTRUMENTED_FOR_ITER 248 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 249 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 250 +#define INSTRUMENTED_END_FOR 251 +#define INSTRUMENTED_END_SEND 252 +#define INSTRUMENTED_INSTRUCTION 253 +#define INSTRUMENTED_LINE 254 +#define SETUP_FINALLY 256 +#define SETUP_CLEANUP 257 +#define SETUP_WITH 258 +#define POP_BLOCK 259 +#define JUMP 260 +#define JUMP_NO_INTERRUPT 261 +#define LOAD_METHOD 262 +#define LOAD_SUPER_METHOD 263 +#define LOAD_ZERO_SUPER_METHOD 264 +#define LOAD_ZERO_SUPER_ATTR 265 +#define STORE_FAST_MAYBE_NULL 266 +#define LOAD_CLOSURE 267 +#define TO_BOOL_ALWAYS_TRUE 7 +#define TO_BOOL_BOOL 8 +#define TO_BOOL_INT 10 +#define TO_BOOL_LIST 13 +#define TO_BOOL_NONE 14 +#define TO_BOOL_STR 18 +#define BINARY_OP_MULTIPLY_INT 19 +#define BINARY_OP_ADD_INT 20 +#define BINARY_OP_SUBTRACT_INT 21 +#define BINARY_OP_MULTIPLY_FLOAT 22 +#define BINARY_OP_ADD_FLOAT 23 +#define BINARY_OP_SUBTRACT_FLOAT 28 +#define BINARY_OP_ADD_UNICODE 29 +#define BINARY_OP_INPLACE_ADD_UNICODE 34 +#define BINARY_SUBSCR_DICT 38 +#define BINARY_SUBSCR_GETITEM 39 +#define BINARY_SUBSCR_LIST_INT 42 +#define BINARY_SUBSCR_STR_INT 43 +#define BINARY_SUBSCR_TUPLE_INT 44 +#define STORE_SUBSCR_DICT 45 +#define STORE_SUBSCR_LIST_INT 46 +#define SEND_GEN 47 +#define UNPACK_SEQUENCE_TWO_TUPLE 48 +#define UNPACK_SEQUENCE_TUPLE 56 +#define UNPACK_SEQUENCE_LIST 57 +#define STORE_ATTR_INSTANCE_VALUE 58 +#define STORE_ATTR_SLOT 59 +#define STORE_ATTR_WITH_HINT 62 +#define LOAD_GLOBAL_MODULE 63 +#define LOAD_GLOBAL_BUILTIN 64 +#define LOAD_SUPER_ATTR_ATTR 65 +#define LOAD_SUPER_ATTR_METHOD 66 +#define LOAD_ATTR_INSTANCE_VALUE 67 +#define LOAD_ATTR_MODULE 70 +#define LOAD_ATTR_WITH_HINT 72 +#define LOAD_ATTR_SLOT 73 +#define LOAD_ATTR_CLASS 76 +#define LOAD_ATTR_PROPERTY 77 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 78 +#define LOAD_ATTR_METHOD_WITH_VALUES 79 +#define LOAD_ATTR_METHOD_NO_DICT 80 +#define LOAD_ATTR_METHOD_LAZY_DICT 81 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 82 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 84 +#define COMPARE_OP_FLOAT 86 +#define COMPARE_OP_INT 88 +#define COMPARE_OP_STR 111 +#define FOR_ITER_LIST 112 +#define FOR_ITER_TUPLE 113 +#define FOR_ITER_RANGE 132 +#define FOR_ITER_GEN 136 +#define CALL_BOUND_METHOD_EXACT_ARGS 148 +#define CALL_PY_EXACT_ARGS 153 +#define CALL_PY_WITH_DEFAULTS 154 +#define CALL_NO_KW_TYPE_1 155 +#define CALL_NO_KW_STR_1 159 +#define CALL_NO_KW_TUPLE_1 160 +#define CALL_BUILTIN_CLASS 161 +#define CALL_NO_KW_BUILTIN_O 166 +#define CALL_NO_KW_BUILTIN_FAST 167 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 178 +#define CALL_NO_KW_LEN 179 +#define CALL_NO_KW_ISINSTANCE 180 +#define CALL_NO_KW_LIST_APPEND 181 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 182 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 183 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 184 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 185 +#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 186 + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_OPCODE_IDS_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index d8fdb34747011..52236f7924503 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1431,9 +1431,11 @@ regen-opcode: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \ $(srcdir)/Lib/opcode.py \ $(srcdir)/Lib/_opcode_metadata.py \ + $(srcdir)/Include/opcode_ids.h.new \ $(srcdir)/Include/opcode.h.new \ $(srcdir)/Python/opcode_targets.h.new \ $(srcdir)/Include/internal/pycore_opcode.h.new + $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 99cfff5acc0ba..c1189c883b667 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -59,7 +59,7 @@ Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)" DependsOnTargets="FindPythonForBuild"> - diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 3a817326c94cb..67f4a2c2d5d76 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -6,7 +6,7 @@ SCRIPT_NAME = "Tools/build/generate_opcode_h.py" PYTHON_OPCODE = "Lib/opcode.py" -header = f""" +opcode_h_header = f""" // Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} #ifndef Py_OPCODE_H @@ -15,11 +15,11 @@ extern "C" {{ #endif +#include "opcode_ids.h" -/* Instruction opcodes for compiled code */ """.lstrip() -footer = """ +opcode_h_footer = """ #ifdef __cplusplus } @@ -27,6 +27,27 @@ #endif /* !Py_OPCODE_H */ """ +opcode_ids_h_header = f""" +// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} + +#ifndef Py_OPCODE_IDS_H +#define Py_OPCODE_IDS_H +#ifdef __cplusplus +extern "C" {{ +#endif + + +/* Instruction opcodes for compiled code */ +""".lstrip() + +opcode_ids_h_footer = """ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_OPCODE_IDS_H */ +""" + internal_header = f""" // Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} @@ -63,9 +84,10 @@ def get_python_module_dict(filename): def main(opcode_py, _opcode_metadata_py='Lib/_opcode_metadata.py', - outfile='Include/opcode.h', + opcode_ids_h='Include/opcode_ids.h', + opcode_h='Include/opcode.h', opcode_targets_h='Python/opcode_targets.h', - internaloutfile='Include/internal/pycore_opcode.h'): + internal_opcode_h='Include/internal/pycore_opcode.h'): _opcode_metadata = get_python_module_dict(_opcode_metadata_py) @@ -91,9 +113,8 @@ def main(opcode_py, opname_including_specialized[next_op] = name used[next_op] = True - with open(outfile, 'w') as fobj, open(internaloutfile, 'w') as iobj: - fobj.write(header) - iobj.write(internal_header) + with open(opcode_ids_h, 'w') as fobj: + fobj.write(opcode_ids_h_header) for name in opname: if name in opmap: @@ -107,6 +128,20 @@ def main(opcode_py, for name, op in specialized_opmap.items(): fobj.write(DEFINE.format(name, op)) + fobj.write(opcode_ids_h_footer) + + with open(opcode_h, 'w') as fobj: + fobj.write(opcode_h_header) + + fobj.write("\n") + for i, (op, _) in enumerate(opcode["_nb_ops"]): + fobj.write(DEFINE.format(op, i)) + + fobj.write(opcode_h_footer) + + with open(internal_opcode_h, 'w') as iobj: + iobj.write(internal_header) + iobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n") iobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n") iobj.write("\n#ifdef NEED_OPCODE_TABLES\n") @@ -129,10 +164,6 @@ def main(opcode_py, iobj.write("};\n") iobj.write("#endif // NEED_OPCODE_TABLES\n") - fobj.write("\n") - for i, (op, _) in enumerate(opcode["_nb_ops"]): - fobj.write(DEFINE.format(op, i)) - iobj.write("\n") iobj.write(f"\nextern const char *const _PyOpcode_OpName[{NUM_OPCODES}];\n") iobj.write("\n#ifdef NEED_OPCODE_TABLES\n") @@ -151,7 +182,6 @@ def main(opcode_py, iobj.write(f" case {i}: \\\n") iobj.write(" ;\n") - fobj.write(footer) iobj.write(internal_footer) with open(opcode_targets_h, "w") as f: @@ -164,8 +194,9 @@ def main(opcode_py, f.write(",\n".join([f" &&{s}" for s in targets])) f.write("\n};\n") - print(f"{outfile} regenerated from {opcode_py}") + print(f"{opcode_h} regenerated from {opcode_py}") if __name__ == '__main__': - main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5]) + main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], + sys.argv[5], sys.argv[6]) From webhook-mailer at python.org Fri Aug 11 13:44:22 2023 From: webhook-mailer at python.org (iritkatriel) Date: Fri, 11 Aug 2023 17:44:22 -0000 Subject: [Python-checkins] gh-106558: break ref cycles through exceptions in multiprocessing manager (#106559) Message-ID: https://github.com/python/cpython/commit/5f7d4ecf301ef12eb1d1d347add054f4fcd8fc5c commit: 5f7d4ecf301ef12eb1d1d347add054f4fcd8fc5c branch: main author: Andrew Geng committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-11T17:44:18Z summary: gh-106558: break ref cycles through exceptions in multiprocessing manager (#106559) files: A Misc/NEWS.d/next/Library/2023-07-09-00-36-33.gh-issue-106558.Zqsj6F.rst M Lib/multiprocessing/managers.py M Lib/test/_test_multiprocessing.py diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index b6534939b4d98..273c22a7654f0 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -90,7 +90,10 @@ def dispatch(c, id, methodname, args=(), kwds={}): kind, result = c.recv() if kind == '#RETURN': return result - raise convert_to_error(kind, result) + try: + raise convert_to_error(kind, result) + finally: + del result # break reference cycle def convert_to_error(kind, result): if kind == '#ERROR': @@ -833,7 +836,10 @@ def _callmethod(self, methodname, args=(), kwds={}): conn = self._Client(token.address, authkey=self._authkey) dispatch(conn, None, 'decref', (token.id,)) return proxy - raise convert_to_error(kind, result) + try: + raise convert_to_error(kind, result) + finally: + del result # break reference cycle def _getvalue(self): ''' diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index c1f9487ae8051..f881a5d467469 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3149,6 +3149,44 @@ def test_rapid_restart(self): if hasattr(manager, "shutdown"): self.addCleanup(manager.shutdown) + +class FakeConnection: + def send(self, payload): + pass + + def recv(self): + return '#ERROR', pyqueue.Empty() + +class TestManagerExceptions(unittest.TestCase): + # Issue 106558: Manager exceptions avoids creating cyclic references. + def setUp(self): + self.mgr = multiprocessing.Manager() + + def tearDown(self): + self.mgr.shutdown() + self.mgr.join() + + def test_queue_get(self): + queue = self.mgr.Queue() + if gc.isenabled(): + gc.disable() + self.addCleanup(gc.enable) + try: + queue.get_nowait() + except pyqueue.Empty as e: + wr = weakref.ref(e) + self.assertEqual(wr(), None) + + def test_dispatch(self): + if gc.isenabled(): + gc.disable() + self.addCleanup(gc.enable) + try: + multiprocessing.managers.dispatch(FakeConnection(), None, None) + except pyqueue.Empty as e: + wr = weakref.ref(e) + self.assertEqual(wr(), None) + # # # diff --git a/Misc/NEWS.d/next/Library/2023-07-09-00-36-33.gh-issue-106558.Zqsj6F.rst b/Misc/NEWS.d/next/Library/2023-07-09-00-36-33.gh-issue-106558.Zqsj6F.rst new file mode 100644 index 0000000000000..8fe677f5d84b5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-09-00-36-33.gh-issue-106558.Zqsj6F.rst @@ -0,0 +1,3 @@ +Remove ref cycle in callers of +:func:`~multiprocessing.managers.convert_to_error` by deleting ``result`` +from scope in a ``finally`` block. From webhook-mailer at python.org Fri Aug 11 13:51:40 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Fri, 11 Aug 2023 17:51:40 -0000 Subject: [Python-checkins] gh-107782: Pydoc: fall back to __text_signature__ if inspect.signature() fails (GH-107786) Message-ID: https://github.com/python/cpython/commit/a39f0a350662f1978104ee1136472d784aa6f29c commit: a39f0a350662f1978104ee1136472d784aa6f29c branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-11T20:51:36+03:00 summary: gh-107782: Pydoc: fall back to __text_signature__ if inspect.signature() fails (GH-107786) It allows to show signatures which are not representable in Python, e.g. for getattr and dict.pop. files: A Misc/NEWS.d/next/Library/2023-08-08-19-57-45.gh-issue-107782.mInjFE.rst M Lib/pydoc.py M Lib/test/test_pydoc.py diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 185f09e603df2..c9a55799b39f0 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -197,6 +197,24 @@ def splitdoc(doc): return lines[0], '\n'.join(lines[2:]) return '', '\n'.join(lines) +def _getargspec(object): + try: + signature = inspect.signature(object) + if signature: + return str(signature) + except (ValueError, TypeError): + argspec = getattr(object, '__text_signature__', None) + if argspec: + if argspec[:2] == '($': + argspec = '(' + argspec[2:] + if getattr(object, '__self__', None) is not None: + # Strip the bound argument. + m = re.match(r'\(\w+(?:(?=\))|,\s*(?:/(?:(?=\))|,\s*))?)', argspec) + if m: + argspec = '(' + argspec[m.end():] + return argspec + return None + def classname(object, modname): """Get a class name and qualify it with a module name if necessary.""" name = object.__name__ @@ -1003,14 +1021,9 @@ def spilldata(msg, attrs, predicate): title = title + '(%s)' % ', '.join(parents) decl = '' - try: - signature = inspect.signature(object) - except (ValueError, TypeError): - signature = None - if signature: - argspec = str(signature) - if argspec and argspec != '()': - decl = name + self.escape(argspec) + '\n\n' + argspec = _getargspec(object) + if argspec and argspec != '()': + decl = name + self.escape(argspec) + '\n\n' doc = getdoc(object) if decl: @@ -1063,18 +1076,13 @@ def docroutine(self, object, name=None, mod=None, anchor, name, reallink) argspec = None if inspect.isroutine(object): - try: - signature = inspect.signature(object) - except (ValueError, TypeError): - signature = None - if signature: - argspec = str(signature) - if realname == '': - title = '%s lambda ' % name - # XXX lambda's won't usually have func_annotations['return'] - # since the syntax doesn't support but it is possible. - # So removing parentheses isn't truly safe. - argspec = argspec[1:-1] # remove parentheses + argspec = _getargspec(object) + if argspec and realname == '': + title = '%s lambda ' % name + # XXX lambda's won't usually have func_annotations['return'] + # since the syntax doesn't support but it is possible. + # So removing parentheses isn't truly safe. + argspec = argspec[1:-1] # remove parentheses if not argspec: argspec = '(...)' @@ -1321,14 +1329,9 @@ def makename(c, m=object.__module__): contents = [] push = contents.append - try: - signature = inspect.signature(object) - except (ValueError, TypeError): - signature = None - if signature: - argspec = str(signature) - if argspec and argspec != '()': - push(name + argspec + '\n') + argspec = _getargspec(object) + if argspec and argspec != '()': + push(name + argspec + '\n') doc = getdoc(object) if doc: @@ -1492,18 +1495,13 @@ def docroutine(self, object, name=None, mod=None, cl=None): argspec = None if inspect.isroutine(object): - try: - signature = inspect.signature(object) - except (ValueError, TypeError): - signature = None - if signature: - argspec = str(signature) - if realname == '': - title = self.bold(name) + ' lambda ' - # XXX lambda's won't usually have func_annotations['return'] - # since the syntax doesn't support but it is possible. - # So removing parentheses isn't truly safe. - argspec = argspec[1:-1] # remove parentheses + argspec = _getargspec(object) + if argspec and realname == '': + title = self.bold(name) + ' lambda ' + # XXX lambda's won't usually have func_annotations['return'] + # since the syntax doesn't support but it is possible. + # So removing parentheses isn't truly safe. + argspec = argspec[1:-1] # remove parentheses if not argspec: argspec = '(...)' decl = asyncqualifier + title + argspec + note diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 8df8b608cf959..fe4e37d4858c8 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -1230,6 +1230,60 @@ def test_bound_builtin_classmethod_o(self): self.assertEqual(self._get_summary_line(dict.__class_getitem__), "__class_getitem__(object, /) method of builtins.type instance") + def test_module_level_callable_unrepresentable_default(self): + self.assertEqual(self._get_summary_line(getattr), + "getattr(object, name, default=, /)") + + def test_builtin_staticmethod_unrepresentable_default(self): + self.assertEqual(self._get_summary_line(str.maketrans), + "maketrans(x, y=, z=, /)") + + def test_unbound_builtin_method_unrepresentable_default(self): + self.assertEqual(self._get_summary_line(dict.pop), + "pop(self, key, default=, /)") + + def test_bound_builtin_method_unrepresentable_default(self): + self.assertEqual(self._get_summary_line({}.pop), + "pop(key, default=, /) " + "method of builtins.dict instance") + + def test_overridden_text_signature(self): + class C: + def meth(*args, **kwargs): + pass + @classmethod + def cmeth(*args, **kwargs): + pass + @staticmethod + def smeth(*args, **kwargs): + pass + for text_signature, unbound, bound in [ + ("($slf)", "(slf, /)", "()"), + ("($slf, /)", "(slf, /)", "()"), + ("($slf, /, arg)", "(slf, /, arg)", "(arg)"), + ("($slf, /, arg=)", "(slf, /, arg=)", "(arg=)"), + ("($slf, arg, /)", "(slf, arg, /)", "(arg, /)"), + ("($slf, arg=, /)", "(slf, arg=, /)", "(arg=, /)"), + ("(/, slf, arg)", "(/, slf, arg)", "(/, slf, arg)"), + ("(/, slf, arg=)", "(/, slf, arg=)", "(/, slf, arg=)"), + ("(slf, /, arg)", "(slf, /, arg)", "(arg)"), + ("(slf, /, arg=)", "(slf, /, arg=)", "(arg=)"), + ("(slf, arg, /)", "(slf, arg, /)", "(arg, /)"), + ("(slf, arg=, /)", "(slf, arg=, /)", "(arg=, /)"), + ]: + with self.subTest(text_signature): + C.meth.__text_signature__ = text_signature + self.assertEqual(self._get_summary_line(C.meth), + "meth" + unbound) + self.assertEqual(self._get_summary_line(C().meth), + "meth" + bound + " method of test.test_pydoc.C instance") + C.cmeth.__func__.__text_signature__ = text_signature + self.assertEqual(self._get_summary_line(C.cmeth), + "cmeth" + bound + " method of builtins.type instance") + C.smeth.__text_signature__ = text_signature + self.assertEqual(self._get_summary_line(C.smeth), + "smeth" + unbound) + @requires_docstrings def test_staticmethod(self): class X: diff --git a/Misc/NEWS.d/next/Library/2023-08-08-19-57-45.gh-issue-107782.mInjFE.rst b/Misc/NEWS.d/next/Library/2023-08-08-19-57-45.gh-issue-107782.mInjFE.rst new file mode 100644 index 0000000000000..fb8a50de3a9ee --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-08-19-57-45.gh-issue-107782.mInjFE.rst @@ -0,0 +1,2 @@ +:mod:`pydoc` is now able to show signatures which are not representable in +Python, e.g. for ``getattr`` and ``dict.pop``. From webhook-mailer at python.org Fri Aug 11 14:13:50 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Fri, 11 Aug 2023 18:13:50 -0000 Subject: [Python-checkins] gh-106844: Fix issues in _winapi.LCMapStringEx (GH-107832) Message-ID: https://github.com/python/cpython/commit/04cc01453db2f0af72a06440831637f8bf512daf commit: 04cc01453db2f0af72a06440831637f8bf512daf branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-11T21:13:46+03:00 summary: gh-106844: Fix issues in _winapi.LCMapStringEx (GH-107832) * Strings with length from 2**31-1 to 2**32-2 always caused MemoryError, it doesn't matter how much memory is available. * Strings with length exactly 2**32-1 caused OSError. * Strings longer than 2**32-1 characters were truncated due to integer overflow bug. * Strings containing the null character were truncated at the first null character. Now strings longer than 2**31-1 characters caused OverflowError and the null character is allowed. files: A Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst M Lib/test/test_ntpath.py M Modules/_winapi.c M Modules/clinic/_winapi.c.h diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 538d758624c9d..78e1cb582512b 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1036,6 +1036,7 @@ def test_path_normcase(self): self._check_function(self.path.normcase) if sys.platform == 'win32': self.assertEqual(ntpath.normcase('\u03a9\u2126'), '??') + self.assertEqual(ntpath.normcase('abc\x00def'), 'abc\x00def') def test_path_isabs(self): self._check_function(self.path.isabs) diff --git a/Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst b/Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst new file mode 100644 index 0000000000000..1fdf162ef4ecd --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst @@ -0,0 +1 @@ +Fix integer overflow and truncating by the null character in :func:`!_winapi.LCMapStringEx` which affects :func:`ntpath.normcase`. diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 0edb36d18dd3f..eec33499b983f 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1539,40 +1539,56 @@ _winapi.LCMapStringEx locale: LPCWSTR flags: DWORD - src: LPCWSTR + src: unicode [clinic start generated code]*/ static PyObject * _winapi_LCMapStringEx_impl(PyObject *module, LPCWSTR locale, DWORD flags, - LPCWSTR src) -/*[clinic end generated code: output=cf4713d80e2b47c9 input=9fe26f95d5ab0001]*/ + PyObject *src) +/*[clinic end generated code: output=b90e6b26e028ff0a input=3e3dcd9b8164012f]*/ { if (flags & (LCMAP_SORTHANDLE | LCMAP_HASH | LCMAP_BYTEREV | LCMAP_SORTKEY)) { return PyErr_Format(PyExc_ValueError, "unsupported flags"); } - int dest_size = LCMapStringEx(locale, flags, src, -1, NULL, 0, + Py_ssize_t src_size; + wchar_t *src_ = PyUnicode_AsWideCharString(src, &src_size); + if (!src_) { + return NULL; + } + if (src_size > INT_MAX) { + PyMem_Free(src_); + PyErr_SetString(PyExc_OverflowError, "input string is too long"); + return NULL; + } + + int dest_size = LCMapStringEx(locale, flags, src_, (int)src_size, NULL, 0, NULL, NULL, 0); - if (dest_size == 0) { - return PyErr_SetFromWindowsErr(0); + if (dest_size <= 0) { + DWORD error = GetLastError(); + PyMem_Free(src_); + return PyErr_SetFromWindowsErr(error); } wchar_t* dest = PyMem_NEW(wchar_t, dest_size); if (dest == NULL) { + PyMem_Free(src_); return PyErr_NoMemory(); } - int nmapped = LCMapStringEx(locale, flags, src, -1, dest, dest_size, + int nmapped = LCMapStringEx(locale, flags, src_, (int)src_size, dest, dest_size, NULL, NULL, 0); - if (nmapped == 0) { + if (nmapped <= 0) { DWORD error = GetLastError(); + PyMem_Free(src_); PyMem_DEL(dest); return PyErr_SetFromWindowsErr(error); } - PyObject *ret = PyUnicode_FromWideChar(dest, dest_size - 1); + PyMem_Free(src_); + PyObject *ret = PyUnicode_FromWideChar(dest, nmapped); PyMem_DEL(dest); return ret; diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 8f46b8f1095e9..35ac053547121 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -884,7 +884,7 @@ PyDoc_STRVAR(_winapi_LCMapStringEx__doc__, static PyObject * _winapi_LCMapStringEx_impl(PyObject *module, LPCWSTR locale, DWORD flags, - LPCWSTR src); + PyObject *src); static PyObject * _winapi_LCMapStringEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -911,16 +911,16 @@ _winapi_LCMapStringEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static const char * const _keywords[] = {"locale", "flags", "src", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .format = "O&kO&:LCMapStringEx", + .format = "O&kU:LCMapStringEx", .kwtuple = KWTUPLE, }; #undef KWTUPLE LPCWSTR locale = NULL; DWORD flags; - LPCWSTR src = NULL; + PyObject *src; if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - _PyUnicode_WideCharString_Converter, &locale, &flags, _PyUnicode_WideCharString_Converter, &src)) { + _PyUnicode_WideCharString_Converter, &locale, &flags, &src)) { goto exit; } return_value = _winapi_LCMapStringEx_impl(module, locale, flags, src); @@ -928,8 +928,6 @@ _winapi_LCMapStringEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: /* Cleanup for locale */ PyMem_Free((void *)locale); - /* Cleanup for src */ - PyMem_Free((void *)src); return return_value; } @@ -1480,4 +1478,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=f32fe6ecdbffd74d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ff91ab5cae8961dd input=a9049054013a1b77]*/ From webhook-mailer at python.org Fri Aug 11 14:42:29 2023 From: webhook-mailer at python.org (carljm) Date: Fri, 11 Aug 2023 18:42:29 -0000 Subject: [Python-checkins] gh-91051: fix segfault when using all 8 type watchers (#107853) Message-ID: https://github.com/python/cpython/commit/66e4edd7346b1cd65ddff6da890a0d725e325116 commit: 66e4edd7346b1cd65ddff6da890a0d725e325116 branch: main author: Carl Meyer committer: carljm date: 2023-08-11T12:42:26-06:00 summary: gh-91051: fix segfault when using all 8 type watchers (#107853) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-10-17-36-27.gh-issue-91051.LfaeNW.rst M Doc/c-api/typeobj.rst M Doc/includes/typestruct.h M Include/cpython/object.h M Lib/test/test_capi/test_watchers.py diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 221a05b192240..faa183e27fcfa 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -147,7 +147,7 @@ Quick Reference +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_vectorcall` | :c:type:`vectorcallfunc` | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_watched`] | char | | | | | | + | [:c:member:`~PyTypeObject.tp_watched`] | unsigned char | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ .. [#slots] @@ -2141,7 +2141,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. versionadded:: 3.9 (the field exists since 3.8 but it's only used since 3.9) -.. c:member:: char PyTypeObject.tp_watched +.. c:member:: unsigned char PyTypeObject.tp_watched Internal. Do not use. diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h index f0ad1e47cb0d8..ec939c28831c3 100644 --- a/Doc/includes/typestruct.h +++ b/Doc/includes/typestruct.h @@ -82,5 +82,5 @@ typedef struct _typeobject { vectorcallfunc tp_vectorcall; /* bitset of which type-watchers care about this type */ - char tp_watched; + unsigned char tp_watched; } PyTypeObject; diff --git a/Include/cpython/object.h b/Include/cpython/object.h index fd45fa57e6282..5f8b1f7c19550 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -227,7 +227,7 @@ struct _typeobject { vectorcallfunc tp_vectorcall; /* bitset of which type-watchers care about this type */ - char tp_watched; + unsigned char tp_watched; }; /* This struct is used by the specializer diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 93f6ef752d066..10b76e163bfb2 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -294,6 +294,18 @@ class C2: pass C2.hmm = "baz" self.assert_events([C1, [C2]]) + def test_all_watchers(self): + class C: pass + with ExitStack() as stack: + last_wid = -1 + # don't make assumptions about how many watchers are already + # registered, just go until we reach the max ID + while last_wid < self.TYPE_MAX_WATCHERS - 1: + last_wid = stack.enter_context(self.watcher()) + self.watch(last_wid, C) + C.foo = "bar" + self.assert_events([C]) + def test_watch_non_type(self): with self.watcher() as wid: with self.assertRaisesRegex(ValueError, r"Cannot watch non-type"): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-10-17-36-27.gh-issue-91051.LfaeNW.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-10-17-36-27.gh-issue-91051.LfaeNW.rst new file mode 100644 index 0000000000000..b4b90ad4ea0ec --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-10-17-36-27.gh-issue-91051.LfaeNW.rst @@ -0,0 +1,2 @@ +Fix abort / segfault when using all eight type watcher slots, on platforms +where ``char`` is signed by default. From webhook-mailer at python.org Fri Aug 11 15:06:00 2023 From: webhook-mailer at python.org (markshannon) Date: Fri, 11 Aug 2023 19:06:00 -0000 Subject: [Python-checkins] GH-106485: Create object's dict-values instead of creating __dict__, when we can. (GH-107843) Message-ID: https://github.com/python/cpython/commit/666b68e8f252e3c6238d6eed1fc82937a774316f commit: 666b68e8f252e3c6238d6eed1fc82937a774316f branch: main author: Mark Shannon committer: markshannon date: 2023-08-11T20:05:56+01:00 summary: GH-106485: Create object's dict-values instead of creating __dict__, when we can. (GH-107843) files: M Objects/dictobject.c M Objects/object.c diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 36375f50646fd..f9701f6b4b09a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5762,10 +5762,8 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, assert(dictptr != NULL); dict = *dictptr; if (dict == NULL) { + assert(!_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)); dictkeys_incref(cached); - if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { - OBJECT_STAT_INC(dict_materialized_on_request); - } dict = new_dict_with_shared_keys(interp, cached); if (dict == NULL) return -1; diff --git a/Objects/object.c b/Objects/object.c index d1154eb344fab..868623a9f7bff 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1577,6 +1577,14 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, goto error_check; } dictptr = &dorv_ptr->dict; + if (*dictptr == NULL) { + if (_PyObject_InitInlineValues(obj, tp) < 0) { + goto done; + } + res = _PyObject_StoreInstanceAttribute( + obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value); + goto error_check; + } } else { dictptr = _PyObject_ComputedDictPointer(obj); From webhook-mailer at python.org Fri Aug 11 15:12:15 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Fri, 11 Aug 2023 19:12:15 -0000 Subject: [Python-checkins] gh-101162: Forbid using issubclass() with GenericAlias as the 1st arg (GH-103369) Message-ID: https://github.com/python/cpython/commit/d93b4ac2ff7bce07fb1c8805f43838818598191c commit: d93b4ac2ff7bce07fb1c8805f43838818598191c branch: main author: Nikita Sobolev committer: serhiy-storchaka date: 2023-08-11T22:12:11+03:00 summary: gh-101162: Forbid using issubclass() with GenericAlias as the 1st arg (GH-103369) files: A Misc/NEWS.d/next/Library/2023-04-08-12-43-52.gh-issue-101162.yOCd_J.rst M Lib/test/test_typing.py M Objects/abstract.c M Objects/genericaliasobject.c diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 0450a87577ece..fa39c79619795 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -4091,6 +4091,22 @@ class C(Generic[T]): pass with self.assertRaises(TypeError): C[()] + def test_generic_subclass_checks(self): + for typ in [list[int], List[int], + tuple[int, str], Tuple[int, str], + typing.Callable[..., None], + collections.abc.Callable[..., None]]: + with self.subTest(typ=typ): + self.assertRaises(TypeError, issubclass, typ, object) + self.assertRaises(TypeError, issubclass, typ, type) + self.assertRaises(TypeError, issubclass, typ, typ) + self.assertRaises(TypeError, issubclass, object, typ) + + # isinstance is fine: + self.assertTrue(isinstance(typ, object)) + # but, not when the right arg is also a generic: + self.assertRaises(TypeError, isinstance, typ, typ) + def test_init(self): T = TypeVar('T') S = TypeVar('S') diff --git a/Misc/NEWS.d/next/Library/2023-04-08-12-43-52.gh-issue-101162.yOCd_J.rst b/Misc/NEWS.d/next/Library/2023-04-08-12-43-52.gh-issue-101162.yOCd_J.rst new file mode 100644 index 0000000000000..e9fadc8f436d9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-08-12-43-52.gh-issue-101162.yOCd_J.rst @@ -0,0 +1,2 @@ +Forbid using :func:`builtins.issubclass` with :class:`types.GenericAlias` as +the first argument. diff --git a/Objects/abstract.c b/Objects/abstract.c index b4edcec600771..c113364a88a26 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2812,7 +2812,7 @@ object_issubclass(PyThreadState *tstate, PyObject *derived, PyObject *cls) return -1; } - /* Probably never reached anymore. */ + /* Can be reached when infinite recursion happens. */ return recursive_issubclass(derived, cls); } diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index df8873454aeb3..faf517b66b935 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -626,6 +626,7 @@ ga_vectorcall(PyObject *self, PyObject *const *args, static const char* const attr_exceptions[] = { "__class__", + "__bases__", "__origin__", "__args__", "__unpacked__", From webhook-mailer at python.org Fri Aug 11 16:30:00 2023 From: webhook-mailer at python.org (Yhg1s) Date: Fri, 11 Aug 2023 20:30:00 -0000 Subject: [Python-checkins] [3.12] Docs: Document PyBUF_MAX_NDIM (GH-107865) (#107871) Message-ID: https://github.com/python/cpython/commit/d20d52bc4d9910666706ea1fb4ca790f87acd06e commit: d20d52bc4d9910666706ea1fb4ca790f87acd06e branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-11T22:29:57+02:00 summary: [3.12] Docs: Document PyBUF_MAX_NDIM (GH-107865) (#107871) Docs: Document PyBUF_MAX_NDIM (GH-107865) (cherry picked from commit 637f7ff2c60f262659da0334f1cb672bd361f398) Co-authored-by: Erlend E. Aasland files: M Doc/c-api/buffer.rst diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 8ca1c190dab9a..ba391a5279f20 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -161,10 +161,14 @@ a buffer, see :c:func:`PyObject_GetBuffer`. If it is ``0``, :c:member:`~Py_buffer.buf` points to a single item representing a scalar. In this case, :c:member:`~Py_buffer.shape`, :c:member:`~Py_buffer.strides` and :c:member:`~Py_buffer.suboffsets` MUST be ``NULL``. + The maximum number of dimensions is given by :c:macro:`PyBUF_MAX_NDIM`. - The macro :c:macro:`PyBUF_MAX_NDIM` limits the maximum number of dimensions - to 64. Exporters MUST respect this limit, consumers of multi-dimensional - buffers SHOULD be able to handle up to :c:macro:`PyBUF_MAX_NDIM` dimensions. + .. :c:macro:: PyBUF_MAX_NDIM + + The maximum number of dimensions the memory represents. + Exporters MUST respect this limit, consumers of multi-dimensional + buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions. + Currently set to 64. .. c:member:: Py_ssize_t *shape From webhook-mailer at python.org Fri Aug 11 20:57:38 2023 From: webhook-mailer at python.org (corona10) Date: Sat, 12 Aug 2023 00:57:38 -0000 Subject: [Python-checkins] gh-104469 : Convert _testcapi/vectorcall_limited.c to use AC (gh-107857) Message-ID: https://github.com/python/cpython/commit/2e27da18952c6561f48dab706b5911135cedd7cf commit: 2e27da18952c6561f48dab706b5911135cedd7cf branch: main author: nahyeon <55136494+nahyeon-an at users.noreply.github.com> committer: corona10 date: 2023-08-12T09:57:35+09:00 summary: gh-104469 : Convert _testcapi/vectorcall_limited.c to use AC (gh-107857) files: A Modules/_testcapi/clinic/vectorcall_limited.c.h M Modules/_testcapi/vectorcall_limited.c diff --git a/Modules/_testcapi/clinic/vectorcall_limited.c.h b/Modules/_testcapi/clinic/vectorcall_limited.c.h new file mode 100644 index 0000000000000..bc89f58b65739 --- /dev/null +++ b/Modules/_testcapi/clinic/vectorcall_limited.c.h @@ -0,0 +1,42 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +#if defined(LIMITED_API_AVAILABLE) + +PyDoc_STRVAR(_testcapi_call_vectorcall__doc__, +"call_vectorcall($module, callable, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_CALL_VECTORCALL_METHODDEF \ + {"call_vectorcall", (PyCFunction)_testcapi_call_vectorcall, METH_O, _testcapi_call_vectorcall__doc__}, + +#endif /* defined(LIMITED_API_AVAILABLE) */ + +#if defined(LIMITED_API_AVAILABLE) + +PyDoc_STRVAR(_testcapi_call_vectorcall_method__doc__, +"call_vectorcall_method($module, callable, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF \ + {"call_vectorcall_method", (PyCFunction)_testcapi_call_vectorcall_method, METH_O, _testcapi_call_vectorcall_method__doc__}, + +#endif /* defined(LIMITED_API_AVAILABLE) */ + +#ifndef _TESTCAPI_CALL_VECTORCALL_METHODDEF + #define _TESTCAPI_CALL_VECTORCALL_METHODDEF +#endif /* !defined(_TESTCAPI_CALL_VECTORCALL_METHODDEF) */ + +#ifndef _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF + #define _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF +#endif /* !defined(_TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF) */ +/*[clinic end generated code: output=409028b637aba77b input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/vectorcall_limited.c b/Modules/_testcapi/vectorcall_limited.c index a96925e840121..9f02c7afa7c97 100644 --- a/Modules/_testcapi/vectorcall_limited.c +++ b/Modules/_testcapi/vectorcall_limited.c @@ -1,5 +1,6 @@ #define Py_LIMITED_API 0x030c0000 // 3.12 #include "parts.h" +#include "clinic/vectorcall_limited.c.h" #ifdef LIMITED_API_AVAILABLE @@ -7,6 +8,11 @@ /* Test Vectorcall in the limited API */ +/*[clinic input] +module _testcapi +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ + static PyObject * LimitedVectorCallClass_tpcall(PyObject *self, PyObject *args, PyObject *kwargs) { return PyUnicode_FromString("tp_call called"); @@ -32,8 +38,16 @@ LimitedVectorCallClass_new(PyTypeObject *tp, PyTypeObject *a, PyTypeObject *kw) return self; } +/*[clinic input] +_testcapi.call_vectorcall + + callable: object + / +[clinic start generated code]*/ + static PyObject * -call_vectorcall(PyObject* self, PyObject *callable) +_testcapi_call_vectorcall(PyObject *module, PyObject *callable) +/*[clinic end generated code: output=bae81eec97fcaad7 input=55d88f92240957ee]*/ { PyObject *args[3] = { NULL, NULL, NULL }; PyObject *kwname = NULL, *kwnames = NULL, *result = NULL; @@ -77,8 +91,16 @@ call_vectorcall(PyObject* self, PyObject *callable) return result; } +/*[clinic input] +_testcapi.call_vectorcall_method + + callable: object + / +[clinic start generated code]*/ + static PyObject * -call_vectorcall_method(PyObject* self, PyObject *callable) +_testcapi_call_vectorcall_method(PyObject *module, PyObject *callable) +/*[clinic end generated code: output=e661f48dda08b6fb input=5ba81c27511395b6]*/ { PyObject *args[3] = { NULL, NULL, NULL }; PyObject *name = NULL, *kwname = NULL, @@ -153,8 +175,8 @@ static PyType_Spec LimitedVectorCallClass_spec = { }; static PyMethodDef TestMethods[] = { - {"call_vectorcall", call_vectorcall, METH_O}, - {"call_vectorcall_method", call_vectorcall_method, METH_O}, + _TESTCAPI_CALL_VECTORCALL_METHODDEF + _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF {NULL}, }; From webhook-mailer at python.org Sat Aug 12 07:07:00 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Sat, 12 Aug 2023 11:07:00 -0000 Subject: [Python-checkins] [3.11] gh-106844: Fix issues in _winapi.LCMapStringEx (GH-107832) (GH-107875) Message-ID: https://github.com/python/cpython/commit/ec254c5dfa8c99f1ec061b252d155386e93f19ef commit: ec254c5dfa8c99f1ec061b252d155386e93f19ef branch: 3.11 author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-12T14:06:56+03:00 summary: [3.11] gh-106844: Fix issues in _winapi.LCMapStringEx (GH-107832) (GH-107875) * Strings with length from 2**31-1 to 2**32-2 always caused MemoryError, it doesn't matter how much memory is available. * Strings with length exactly 2**32-1 caused OSError. * Strings longer than 2**32-1 characters were truncated due to integer overflow bug. Now strings longer than 2**31-1 characters caused OverflowError. (cherry picked from commit 04cc01453db2f0af72a06440831637f8bf512daf) files: A Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst M Lib/test/test_ntpath.py M Modules/_winapi.c diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 646e81d1e2fa9..75e50d92ed1eb 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -908,6 +908,7 @@ def test_path_normcase(self): self._check_function(self.path.normcase) if sys.platform == 'win32': self.assertEqual(ntpath.normcase('\u03a9\u2126'), '??') + self.assertEqual(ntpath.normcase('abc\x00def'), 'abc\x00def') def test_path_isabs(self): self._check_function(self.path.isabs) diff --git a/Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst b/Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst new file mode 100644 index 0000000000000..11fca7e0452f4 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst @@ -0,0 +1 @@ +Fix integer overflow in :func:`!_winapi.LCMapStringEx` which affects :func:`ntpath.normcase`. diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 5c61d99a83751..7fb1f2f561b0f 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1571,24 +1571,26 @@ _winapi_LCMapStringEx_impl(PyObject *module, PyObject *locale, DWORD flags, if (!locale_) { return NULL; } - Py_ssize_t srcLenAsSsize; - int srcLen; - wchar_t *src_ = PyUnicode_AsWideCharString(src, &srcLenAsSsize); + Py_ssize_t src_size; + wchar_t *src_ = PyUnicode_AsWideCharString(src, &src_size); if (!src_) { PyMem_Free(locale_); return NULL; } - srcLen = (int)srcLenAsSsize; - if (srcLen != srcLenAsSsize) { - srcLen = -1; + if (src_size > INT_MAX) { + PyMem_Free(locale_); + PyMem_Free(src_); + PyErr_SetString(PyExc_OverflowError, "input string is too long"); + return NULL; } - int dest_size = LCMapStringEx(locale_, flags, src_, srcLen, NULL, 0, + int dest_size = LCMapStringEx(locale_, flags, src_, (int)src_size, NULL, 0, NULL, NULL, 0); - if (dest_size == 0) { + if (dest_size <= 0) { + DWORD error = GetLastError(); PyMem_Free(locale_); PyMem_Free(src_); - return PyErr_SetFromWindowsErr(0); + return PyErr_SetFromWindowsErr(error); } wchar_t* dest = PyMem_NEW(wchar_t, dest_size); @@ -1598,9 +1600,9 @@ _winapi_LCMapStringEx_impl(PyObject *module, PyObject *locale, DWORD flags, return PyErr_NoMemory(); } - int nmapped = LCMapStringEx(locale_, flags, src_, srcLen, dest, dest_size, + int nmapped = LCMapStringEx(locale_, flags, src_, (int)src_size, dest, dest_size, NULL, NULL, 0); - if (nmapped == 0) { + if (nmapped <= 0) { DWORD error = GetLastError(); PyMem_Free(locale_); PyMem_Free(src_); @@ -1608,9 +1610,9 @@ _winapi_LCMapStringEx_impl(PyObject *module, PyObject *locale, DWORD flags, return PyErr_SetFromWindowsErr(error); } - PyObject *ret = PyUnicode_FromWideChar(dest, dest_size); PyMem_Free(locale_); PyMem_Free(src_); + PyObject *ret = PyUnicode_FromWideChar(dest, nmapped); PyMem_DEL(dest); return ret; From webhook-mailer at python.org Sat Aug 12 07:57:53 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Sat, 12 Aug 2023 11:57:53 -0000 Subject: [Python-checkins] [3.11] gh-107715: Escape class name in regular expression (GH-107716) (GH-107727) Message-ID: https://github.com/python/cpython/commit/5f36e5ca5fe6768ae639aaf7c04ed41b665880e1 commit: 5f36e5ca5fe6768ae639aaf7c04ed41b665880e1 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: serhiy-storchaka date: 2023-08-12T11:57:49Z summary: [3.11] gh-107715: Escape class name in regular expression (GH-107716) (GH-107727) This patch escapes the class name before embedding it in the regular expression for `pat` in `doctest.DocTestFinder._find_lineno`. While class names do not ordinarily contain special characters, it is possible to encounter these when a class is created dynamically. Escaping the name will correctly return `None` in this scenario, rather than potentially matching a different class or raising `re.error` depending on the symbols used. (cherry picked from commit 85793278793708ad6b7132a54ac9fb4b2c5bcac1) Co-authored-by: Gertjan van Zwieten files: A Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst M Lib/doctest.py diff --git a/Lib/doctest.py b/Lib/doctest.py index dafad505ef0e8..8fcb4ee0a02ee 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1104,7 +1104,7 @@ def _find_lineno(self, obj, source_lines): if source_lines is None: return None pat = re.compile(r'^\s*class\s*%s\b' % - getattr(obj, '__name__', '-')) + re.escape(getattr(obj, '__name__', '-'))) for i, line in enumerate(source_lines): if pat.match(line): lineno = i diff --git a/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst b/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst new file mode 100644 index 0000000000000..4bf08c071df2f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst @@ -0,0 +1 @@ +Fix :meth:`doctest.DocTestFinder.find` in presence of class names with special characters. Patch by Gertjan van Zwieten. From webhook-mailer at python.org Sat Aug 12 08:05:25 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Sat, 12 Aug 2023 12:05:25 -0000 Subject: [Python-checkins] gh-107891: Fix typo in 3.12 whatsnew (#107892) Message-ID: https://github.com/python/cpython/commit/2e1f688fe0f0a612e54c09f5a7027a834dd8b8d5 commit: 2e1f688fe0f0a612e54c09f5a7027a834dd8b8d5 branch: main author: wookie184 committer: AlexWaygood date: 2023-08-12T12:05:22Z summary: gh-107891: Fix typo in 3.12 whatsnew (#107892) files: M Doc/whatsnew/3.12.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 8ed435476c9ce..65ca4c332ce35 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1576,7 +1576,7 @@ Changes in the Python API 1,13-1,17: FSTRING_MIDDLE ' end' 1,17-1,18: FSTRING_END '"' - Additionally, there may be some minor behavioral changes as a consecuence of the + Additionally, there may be some minor behavioral changes as a consequence of the changes required to support :pep:`701`. Some of these changes include: * The ``type`` attribute of the tokens emitted when tokenizing some invalid Python From webhook-mailer at python.org Sat Aug 12 15:36:50 2023 From: webhook-mailer at python.org (corona10) Date: Sat, 12 Aug 2023 19:36:50 -0000 Subject: [Python-checkins] gh-106797: Remove warning logs from Python/generated_cases.c.h and executor_cases.c.h (gh-107889) Message-ID: https://github.com/python/cpython/commit/bf707749e87f47b2fee2a208a654511ac318d4b9 commit: bf707749e87f47b2fee2a208a654511ac318d4b9 branch: main author: Dong-hee Na committer: corona10 date: 2023-08-13T04:36:46+09:00 summary: gh-106797: Remove warning logs from Python/generated_cases.c.h and executor_cases.c.h (gh-107889) gh-106797: Remove warning logs from Python/generated_cases.c.h files: M Python/executor_cases.c.h M Python/generated_cases.c.h M Tools/cases_generator/stacking.py diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a7b5054417ed8..5e3c84b2b4b61 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1513,7 +1513,7 @@ Py_DECREF(self); if (attr == NULL) goto pop_3_error; STACK_SHRINK(2); - stack_pointer[-1 - (0 ? 1 : 0)] = attr; + stack_pointer[-1] = attr; break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ccf43c727b9e0..2661a39e047c4 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2251,7 +2251,7 @@ Py_DECREF(self); if (attr == NULL) goto pop_3_error; STACK_SHRINK(2); - stack_pointer[-1 - (0 ? 1 : 0)] = attr; + stack_pointer[-1] = attr; next_instr += 1; DISPATCH(); } @@ -3574,7 +3574,7 @@ assert(descr != NULL); Py_DECREF(owner); attr = Py_NewRef(descr); - stack_pointer[-1 - (0 ? 1 : 0)] = attr; + stack_pointer[-1] = attr; next_instr += 9; DISPATCH(); } @@ -3594,7 +3594,7 @@ assert(descr != NULL); Py_DECREF(owner); attr = Py_NewRef(descr); - stack_pointer[-1 - (0 ? 1 : 0)] = attr; + stack_pointer[-1] = attr; next_instr += 9; DISPATCH(); } diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index d457ce01a8f43..9bb7f46844224 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -61,14 +61,14 @@ def as_terms(self) -> list[tuple[str, str]]: for eff in self.deep: if eff.size: terms.append(("-", maybe_parenthesize(eff.size))) - elif eff.cond and eff.cond != "1": + elif eff.cond and eff.cond not in ("0", "1"): terms.append(("-", f"({parenthesize_cond(eff.cond)} ? 1 : 0)")) elif eff.cond != "0": num -= 1 for eff in self.high: if eff.size: terms.append(("+", maybe_parenthesize(eff.size))) - elif eff.cond and eff.cond != "1": + elif eff.cond and eff.cond not in ("0", "1"): terms.append(("+", f"({parenthesize_cond(eff.cond)} ? 1 : 0)")) elif eff.cond != "0": num += 1 From webhook-mailer at python.org Sat Aug 12 19:46:03 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Sat, 12 Aug 2023 23:46:03 -0000 Subject: [Python-checkins] gh-107883: Argument Clinic: Handle full module/class path in Function.fulldisplayname (#107884) Message-ID: https://github.com/python/cpython/commit/ee40b3e20d9b8d62a9b36b777dff42db1e9049d5 commit: ee40b3e20d9b8d62a9b36b777dff42db1e9049d5 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-12T23:46:00Z summary: gh-107883: Argument Clinic: Handle full module/class path in Function.fulldisplayname (#107884) Co-authored-by: Alex Waygood files: M Lib/test/test_clinic.py M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index a649b5fe2201c..55251b516d2b2 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1513,6 +1513,60 @@ def test_parameters_required_after_star(self): with self.subTest(block=block): self.expect_failure(block, err) + def test_fulldisplayname_class(self): + dataset = ( + ("T", """ + class T "void *" "" + T.__init__ + """), + ("m.T", """ + module m + class m.T "void *" "" + @classmethod + m.T.__new__ + """), + ("m.T.C", """ + module m + class m.T "void *" "" + class m.T.C "void *" "" + m.T.C.__init__ + """), + ) + for name, code in dataset: + with self.subTest(name=name, code=code): + block = self.parse(code) + func = block.signatures[-1] + self.assertEqual(func.fulldisplayname, name) + + def test_fulldisplayname_meth(self): + dataset = ( + ("func", "func"), + ("m.func", """ + module m + m.func + """), + ("T.meth", """ + class T "void *" "" + T.meth + """), + ("m.T.meth", """ + module m + class m.T "void *" "" + m.T.meth + """), + ("m.T.C.meth", """ + module m + class m.T "void *" "" + class m.T.C "void *" "" + m.T.C.meth + """), + ) + for name, code in dataset: + with self.subTest(name=name, code=code): + block = self.parse(code) + func = block.signatures[-1] + self.assertEqual(func.fulldisplayname, name) + def test_depr_star_invalid_format_1(self): block = """ module foo diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 70b066cce82fa..2d23f9d12875f 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2682,9 +2682,16 @@ def displayname(self) -> str: @functools.cached_property def fulldisplayname(self) -> str: - if isinstance(self.module, Module): - return f"{self.module.name}.{self.displayname}" - return self.displayname + parent: Class | Module | Clinic | None + if self.kind.new_or_init: + parent = getattr(self.cls, "parent", None) + else: + parent = self.parent + name = self.displayname + while isinstance(parent, (Module, Class)): + name = f"{parent.name}.{name}" + parent = parent.parent + return name @property def render_parameters(self) -> list[Parameter]: From webhook-mailer at python.org Sun Aug 13 05:25:02 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Sun, 13 Aug 2023 09:25:02 -0000 Subject: [Python-checkins] Improve `_typing.__doc__` (#107908) Message-ID: https://github.com/python/cpython/commit/7ddc1eaff16fa946516283be82d8b2e2ef315b03 commit: 7ddc1eaff16fa946516283be82d8b2e2ef315b03 branch: main author: Nikita Sobolev committer: AlexWaygood date: 2023-08-13T10:24:59+01:00 summary: Improve `_typing.__doc__` (#107908) files: M Modules/_typingmodule.c diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 59d3a80a9305d..9ea72bf89ce0b 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -40,7 +40,7 @@ static PyMethodDef typing_methods[] = { }; PyDoc_STRVAR(typing_doc, -"Accelerators for the typing module.\n"); +"Primitives and accelerators for the typing module.\n"); static int _typing_exec(PyObject *m) From webhook-mailer at python.org Sun Aug 13 06:13:15 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Sun, 13 Aug 2023 10:13:15 -0000 Subject: [Python-checkins] gh-107880: Teach Argument Clinic to clone __init__ and __new__ methods (#107885) Message-ID: https://github.com/python/cpython/commit/9b75ada6e4232ff03b716b1c5930d1a3ba3b16b8 commit: 9b75ada6e4232ff03b716b1c5930d1a3ba3b16b8 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-13T12:13:11+02:00 summary: gh-107880: Teach Argument Clinic to clone __init__ and __new__ methods (#107885) files: A Misc/NEWS.d/next/Tools-Demos/2023-08-13-11-18-06.gh-issue-107880.gBVVQ7.rst M Lib/test/test_clinic.py M Modules/_testclinic.c M Modules/clinic/_testclinic_depr_star.c.h M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 55251b516d2b2..f067a26d1fb3a 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2973,6 +2973,17 @@ def test_depr_star_new(self): ac_tester.DeprStarNew(None) self.assertEqual(cm.filename, __file__) + def test_depr_star_new_cloned(self): + regex = re.escape( + "Passing positional arguments to _testclinic.DeprStarNew.cloned() " + "is deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14." + ) + obj = ac_tester.DeprStarNew(a=None) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + obj.cloned(None) + self.assertEqual(cm.filename, __file__) + def test_depr_star_init(self): regex = re.escape( "Passing positional arguments to _testclinic.DeprStarInit() is " @@ -2983,6 +2994,17 @@ def test_depr_star_init(self): ac_tester.DeprStarInit(None) self.assertEqual(cm.filename, __file__) + def test_depr_star_init_cloned(self): + regex = re.escape( + "Passing positional arguments to _testclinic.DeprStarInit.cloned() " + "is deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14." + ) + obj = ac_tester.DeprStarInit(a=None) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + obj.cloned(None) + self.assertEqual(cm.filename, __file__) + def test_depr_star_pos0_len1(self): fn = ac_tester.depr_star_pos0_len1 fn(a=None) diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-13-11-18-06.gh-issue-107880.gBVVQ7.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-13-11-18-06.gh-issue-107880.gBVVQ7.rst new file mode 100644 index 0000000000000..fd9d6717f3a33 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-13-11-18-06.gh-issue-107880.gBVVQ7.rst @@ -0,0 +1,2 @@ +Argument Clinic can now clone :meth:`!__init__` and :meth:`!__new__` +methods. diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 8fa3cc83d871b..c33536234af0b 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1230,12 +1230,29 @@ depr_star_new_impl(PyTypeObject *type, PyObject *a) return type->tp_alloc(type, 0); } +/*[clinic input] +_testclinic.DeprStarNew.cloned as depr_star_new_clone = _testclinic.DeprStarNew.__new__ +[clinic start generated code]*/ + +static PyObject * +depr_star_new_clone_impl(PyObject *type, PyObject *a) +/*[clinic end generated code: output=3b17bf885fa736bc input=ea659285d5dbec6c]*/ +{ + Py_RETURN_NONE; +} + +static struct PyMethodDef depr_star_new_methods[] = { + DEPR_STAR_NEW_CLONE_METHODDEF + {NULL, NULL} +}; + static PyTypeObject DeprStarNew = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "_testclinic.DeprStarNew", .tp_basicsize = sizeof(PyObject), .tp_new = depr_star_new, .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_methods = depr_star_new_methods, }; @@ -1254,6 +1271,22 @@ depr_star_init_impl(PyObject *self, PyObject *a) return 0; } +/*[clinic input] +_testclinic.DeprStarInit.cloned as depr_star_init_clone = _testclinic.DeprStarInit.__init__ +[clinic start generated code]*/ + +static PyObject * +depr_star_init_clone_impl(PyObject *self, PyObject *a) +/*[clinic end generated code: output=ddfe8a1b5531e7cc input=561e103fe7f8e94f]*/ +{ + Py_RETURN_NONE; +} + +static struct PyMethodDef depr_star_init_methods[] = { + DEPR_STAR_INIT_CLONE_METHODDEF + {NULL, NULL} +}; + static PyTypeObject DeprStarInit = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "_testclinic.DeprStarInit", @@ -1261,6 +1294,7 @@ static PyTypeObject DeprStarInit = { .tp_new = PyType_GenericNew, .tp_init = depr_star_init, .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_methods = depr_star_init_methods, }; diff --git a/Modules/clinic/_testclinic_depr_star.c.h b/Modules/clinic/_testclinic_depr_star.c.h index 0c2fa088268c6..1aa42dd405977 100644 --- a/Modules/clinic/_testclinic_depr_star.c.h +++ b/Modules/clinic/_testclinic_depr_star.c.h @@ -92,6 +92,89 @@ depr_star_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(depr_star_new_clone__doc__, +"cloned($self, /, a)\n" +"--\n" +"\n" +"Note: Passing positional arguments to _testclinic.DeprStarNew.cloned()\n" +"is deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_NEW_CLONE_METHODDEF \ + {"cloned", _PyCFunction_CAST(depr_star_new_clone), METH_FASTCALL|METH_KEYWORDS, depr_star_new_clone__doc__}, + +static PyObject * +depr_star_new_clone_impl(PyObject *type, PyObject *a); + +static PyObject * +depr_star_new_clone(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "cloned", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *a; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarNew.cloned' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarNew.cloned' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarNew.cloned' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to _testclinic.DeprStarNew.cloned()" + " is deprecated. Parameter 'a' will become a keyword-only " + "parameter in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + return_value = depr_star_new_clone_impl(type, a); + +exit: + return return_value; +} + PyDoc_STRVAR(depr_star_init__doc__, "DeprStarInit(a)\n" "--\n" @@ -176,6 +259,89 @@ depr_star_init(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(depr_star_init_clone__doc__, +"cloned($self, /, a)\n" +"--\n" +"\n" +"Note: Passing positional arguments to\n" +"_testclinic.DeprStarInit.cloned() is deprecated. Parameter \'a\' will\n" +"become a keyword-only parameter in Python 3.14.\n" +""); + +#define DEPR_STAR_INIT_CLONE_METHODDEF \ + {"cloned", _PyCFunction_CAST(depr_star_init_clone), METH_FASTCALL|METH_KEYWORDS, depr_star_init_clone__doc__}, + +static PyObject * +depr_star_init_clone_impl(PyObject *self, PyObject *a); + +static PyObject * +depr_star_init_clone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "cloned", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *a; + + // Emit compiler warnings when we get to Python 3.14. + #if PY_VERSION_HEX >= 0x030e00C0 + # error \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarInit.cloned' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030e00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarInit.cloned' to be keyword-only.") + # else + # warning \ + "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ + " '_testclinic.DeprStarInit.cloned' to be keyword-only." + # endif + #endif + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to " + "_testclinic.DeprStarInit.cloned() is deprecated. Parameter 'a' " + "will become a keyword-only parameter in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + return_value = depr_star_init_clone_impl(self, a); + +exit: + return return_value; +} + PyDoc_STRVAR(depr_star_pos0_len1__doc__, "depr_star_pos0_len1($module, /, a)\n" "--\n" @@ -971,4 +1137,4 @@ depr_star_pos2_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t exit: return return_value; } -/*[clinic end generated code: output=18ab056f6cc06d7e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7a16fee4d6742d54 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 2d23f9d12875f..1e0303c77087e 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4888,13 +4888,25 @@ def state_modulename_name(self, line: str) -> None: function_name = fields.pop() module, cls = self.clinic._module_and_class(fields) - if not (existing_function.kind is self.kind and existing_function.coexist == self.coexist): - fail("'kind' of function and cloned function don't match! " - "(@classmethod/@staticmethod/@coexist)") - function = existing_function.copy( - name=function_name, full_name=full_name, module=module, - cls=cls, c_basename=c_basename, docstring='' - ) + overrides: dict[str, Any] = { + "name": function_name, + "full_name": full_name, + "module": module, + "cls": cls, + "c_basename": c_basename, + "docstring": "", + } + if not (existing_function.kind is self.kind and + existing_function.coexist == self.coexist): + # Allow __new__ or __init__ methods. + if existing_function.kind.new_or_init: + overrides["kind"] = self.kind + # Future enhancement: allow custom return converters + overrides["return_converter"] = CReturnConverter() + else: + fail("'kind' of function and cloned function don't match! " + "(@classmethod/@staticmethod/@coexist)") + function = existing_function.copy(**overrides) self.function = function self.block.signatures.append(function) (cls or module).functions.append(function) From webhook-mailer at python.org Sun Aug 13 09:01:26 2023 From: webhook-mailer at python.org (rhettinger) Date: Sun, 13 Aug 2023 13:01:26 -0000 Subject: [Python-checkins] Add another example to the statistics docs (GH-107904) Message-ID: https://github.com/python/cpython/commit/2b6dc2accc315ce279d259ed39e058a225068531 commit: 2b6dc2accc315ce279d259ed39e058a225068531 branch: main author: Raymond Hettinger committer: rhettinger date: 2023-08-13T08:01:23-05:00 summary: Add another example to the statistics docs (GH-107904) files: A Doc/library/kde_example.png M Doc/library/statistics.rst diff --git a/Doc/library/kde_example.png b/Doc/library/kde_example.png new file mode 100644 index 0000000000000..f450489569997 Binary files /dev/null and b/Doc/library/kde_example.png differ diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 395b324c86038..483ebea67f0c6 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -922,6 +922,10 @@ of applications in statistics. :class:`NormalDist` Examples and Recipes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Classic probability problems +**************************** + :class:`NormalDist` readily solves classic probability problems. For example, given `historical data for SAT exams @@ -947,6 +951,10 @@ Find the `quartiles `_ and `deciles >>> list(map(round, sat.quantiles(n=10))) [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310] + +Monte Carlo inputs for simulations +********************************** + To estimate the distribution for a model than isn't easy to solve analytically, :class:`NormalDist` can generate input samples for a `Monte Carlo simulation `_: @@ -963,6 +971,9 @@ Carlo simulation `_: >>> quantiles(map(model, X, Y, Z)) # doctest: +SKIP [1.4591308524824727, 1.8035946855390597, 2.175091447274739] +Approximating binomial distributions +************************************ + Normal distributions can be used to approximate `Binomial distributions `_ when the sample size is large and when the probability of a successful @@ -1000,6 +1011,10 @@ probability that the Python room will stay within its capacity limits? >>> mean(trial() <= k for i in range(10_000)) 0.8398 + +Naive bayesian classifier +************************* + Normal distributions commonly arise in machine learning problems. Wikipedia has a `nice example of a Naive Bayesian Classifier @@ -1054,6 +1069,48 @@ The final prediction goes to the largest posterior. This is known as the 'female' +Kernel density estimation +************************* + +It is possible to estimate a continuous probability density function +from a fixed number of discrete samples. + +The basic idea is to smooth the data using `a kernel function such as a +normal distribution, triangular distribution, or uniform distribution +`_. +The degree of smoothing is controlled by a single +parameter, ``h``, representing the variance of the kernel function. + +.. testcode:: + + import math + + def kde_normal(sample, h): + "Create a continous probability density function from a sample." + # Smooth the sample with a normal distribution of variance h. + kernel_h = NormalDist(0.0, math.sqrt(h)).pdf + n = len(sample) + def pdf(x): + return sum(kernel_h(x - x_i) for x_i in sample) / n + return pdf + +`Wikipedia has an example +`_ +where we can use the ``kde_normal()`` recipe to generate and plot +a probability density function estimated from a small sample: + +.. doctest:: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde_normal(sample, h=2.25) + >>> xarr = [i/100 for i in range(-750, 1100)] + >>> yarr = [f_hat(x) for x in xarr] + +The points in ``xarr`` and ``yarr`` can be used to make a PDF plot: + +.. image:: kde_example.png + :alt: Scatter plot of the estimated probability density function. + .. # This modelines must appear within the last ten lines of the file. kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; From webhook-mailer at python.org Sun Aug 13 09:19:44 2023 From: webhook-mailer at python.org (vsajip) Date: Sun, 13 Aug 2023 13:19:44 -0000 Subject: [Python-checkins] gh-107877: Update logging levels reference table with usage criteria. (#107894) Message-ID: https://github.com/python/cpython/commit/cc2cf85d03cf29994a707aae5cc9a349a4165b84 commit: cc2cf85d03cf29994a707aae5cc9a349a4165b84 branch: main author: Vinay Sajip committer: vsajip date: 2023-08-13T14:19:41+01:00 summary: gh-107877: Update logging levels reference table with usage criteria. (#107894) Co-authored-by: Alex Waygood files: M Doc/library/logging.rst diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 4c6e74ff66a11..b582c918df023 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -397,21 +397,39 @@ have specific values relative to the predefined levels. If you define a level with the same numeric value, it overwrites the predefined value; the predefined name is lost. -+-----------------------+---------------+ -| Level | Numeric value | -+=======================+===============+ -| .. py:data:: CRITICAL | 50 | -+-----------------------+---------------+ -| .. py:data:: ERROR | 40 | -+-----------------------+---------------+ -| .. py:data:: WARNING | 30 | -+-----------------------+---------------+ -| .. py:data:: INFO | 20 | -+-----------------------+---------------+ -| .. py:data:: DEBUG | 10 | -+-----------------------+---------------+ -| .. py:data:: NOTSET | 0 | -+-----------------------+---------------+ ++-----------------------+---------------+-------------------------------------+ +| Level | Numeric value | What it means / When to use it | ++=======================+===============+=====================================+ +| .. py:data:: NOTSET | 0 | When set on a logger, indicates that| +| | | ancestor loggers are to be consulted| +| | | to determine the effective level. | +| | | If that still resolves to | +| | | :const:`!NOTSET`, then all events | +| | | are logged. When set on a handler, | +| | | all events are handled. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: DEBUG | 10 | Detailed information, typically only| +| | | of interest to a developer trying to| +| | | diagnose a problem. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: INFO | 20 | Confirmation that things are working| +| | | as expected. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: WARNING | 30 | An indication that something | +| | | unexpected happened, or that a | +| | | problem might occur in the near | +| | | future (e.g. 'disk space low'). The | +| | | software is still working as | +| | | expected. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: ERROR | 40 | Due to a more serious problem, the | +| | | software has not been able to | +| | | perform some function. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: CRITICAL | 50 | A serious error, indicating that the| +| | | program itself may be unable to | +| | | continue running. | ++-----------------------+---------------+-------------------------------------+ .. _handler: From webhook-mailer at python.org Sun Aug 13 09:43:18 2023 From: webhook-mailer at python.org (vsajip) Date: Sun, 13 Aug 2023 13:43:18 -0000 Subject: [Python-checkins] [3.11] gh-107877: Update logging levels reference table with usage criteria. (GH-107894) (GH-107921) Message-ID: https://github.com/python/cpython/commit/e2420c5cae7a173c5242b3507979010a933e53a7 commit: e2420c5cae7a173c5242b3507979010a933e53a7 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: vsajip date: 2023-08-13T14:43:14+01:00 summary: [3.11] gh-107877: Update logging levels reference table with usage criteria. (GH-107894) (GH-107921) gh-107877: Update logging levels reference table with usage criteria. (GH-107894) (cherry picked from commit cc2cf85d03cf29994a707aae5cc9a349a4165b84) files: M Doc/library/logging.rst diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 954681efb4436..09dc887ee2415 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -385,21 +385,39 @@ have specific values relative to the predefined levels. If you define a level with the same numeric value, it overwrites the predefined value; the predefined name is lost. -+-----------------------+---------------+ -| Level | Numeric value | -+=======================+===============+ -| .. py:data:: CRITICAL | 50 | -+-----------------------+---------------+ -| .. py:data:: ERROR | 40 | -+-----------------------+---------------+ -| .. py:data:: WARNING | 30 | -+-----------------------+---------------+ -| .. py:data:: INFO | 20 | -+-----------------------+---------------+ -| .. py:data:: DEBUG | 10 | -+-----------------------+---------------+ -| .. py:data:: NOTSET | 0 | -+-----------------------+---------------+ ++-----------------------+---------------+-------------------------------------+ +| Level | Numeric value | What it means / When to use it | ++=======================+===============+=====================================+ +| .. py:data:: NOTSET | 0 | When set on a logger, indicates that| +| | | ancestor loggers are to be consulted| +| | | to determine the effective level. | +| | | If that still resolves to | +| | | :const:`!NOTSET`, then all events | +| | | are logged. When set on a handler, | +| | | all events are handled. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: DEBUG | 10 | Detailed information, typically only| +| | | of interest to a developer trying to| +| | | diagnose a problem. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: INFO | 20 | Confirmation that things are working| +| | | as expected. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: WARNING | 30 | An indication that something | +| | | unexpected happened, or that a | +| | | problem might occur in the near | +| | | future (e.g. 'disk space low'). The | +| | | software is still working as | +| | | expected. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: ERROR | 40 | Due to a more serious problem, the | +| | | software has not been able to | +| | | perform some function. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: CRITICAL | 50 | A serious error, indicating that the| +| | | program itself may be unable to | +| | | continue running. | ++-----------------------+---------------+-------------------------------------+ .. _handler: From webhook-mailer at python.org Mon Aug 14 05:26:22 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 14 Aug 2023 09:26:22 -0000 Subject: [Python-checkins] gh-107910: Remove not needing newline in error message (GH-107928) Message-ID: https://github.com/python/cpython/commit/c3887b57a75a105615dd555aaf74e6c9a243ebdd commit: c3887b57a75a105615dd555aaf74e6c9a243ebdd branch: main author: Joon Hwan ??? committer: serhiy-storchaka date: 2023-08-14T12:26:18+03:00 summary: gh-107910: Remove not needing newline in error message (GH-107928) files: M Lib/test/test_descr.py M Objects/typeobject.c diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index ad3eefba36585..fc8d0a305d8da 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -989,8 +989,8 @@ class EditableScrollablePane(ScrollablePane,EditablePane): pass def test_mro_disagreement(self): # Testing error messages for MRO disagreement... - mro_err_msg = """Cannot create a consistent method resolution -order (MRO) for bases """ + mro_err_msg = ("Cannot create a consistent method resolution " + "order (MRO) for bases ") def raises(exc, expected, callable, *args): try: diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 030e8bfc99b6d..7ce3de4d58d0d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2444,7 +2444,7 @@ set_mro_error(PyObject **to_merge, Py_ssize_t to_merge_size, int *remain) n = PyDict_GET_SIZE(set); off = PyOS_snprintf(buf, sizeof(buf), "Cannot create a \ -consistent method resolution\norder (MRO) for bases"); +consistent method resolution order (MRO) for bases"); i = 0; while (PyDict_Next(set, &i, &k, &v) && (size_t)off < sizeof(buf)) { PyObject *name = class_name(k); From webhook-mailer at python.org Mon Aug 14 05:51:54 2023 From: webhook-mailer at python.org (iritkatriel) Date: Mon, 14 Aug 2023 09:51:54 -0000 Subject: [Python-checkins] gh-103082: use IS_VALID_OPCODE instead of _PyOpcode_OpName to check if an opcode is defined (#107882) Message-ID: https://github.com/python/cpython/commit/608927b01447b110de5094271fbc4d49c60130b0 commit: 608927b01447b110de5094271fbc4d49c60130b0 branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-14T10:51:50+01:00 summary: gh-103082: use IS_VALID_OPCODE instead of _PyOpcode_OpName to check if an opcode is defined (#107882) files: M Python/instrumentation.c diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 6d11649c07fbe..b50e8e26476cd 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -7,6 +7,7 @@ #include "pycore_namespace.h" #include "pycore_object.h" #include "pycore_opcode.h" +#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -437,11 +438,10 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) static bool valid_opcode(int opcode) { - if (opcode > 0 && + if (IS_VALID_OPCODE(opcode) && + opcode != CACHE && opcode != RESERVED && - opcode < 255 && - _PyOpcode_OpName[opcode] && - _PyOpcode_OpName[opcode][0] != '<') + opcode < 255) { return true; } From webhook-mailer at python.org Mon Aug 14 08:39:34 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 14 Aug 2023 12:39:34 -0000 Subject: [Python-checkins] [3.12] gh-107891: Fix typo in 3.12 whatsnew (GH-107892) (#107893) Message-ID: https://github.com/python/cpython/commit/39ce30dd3ed13e5ce275d1aaa9ee1d8c5ba62f17 commit: 39ce30dd3ed13e5ce275d1aaa9ee1d8c5ba62f17 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-14T14:39:30+02:00 summary: [3.12] gh-107891: Fix typo in 3.12 whatsnew (GH-107892) (#107893) gh-107891: Fix typo in 3.12 whatsnew (GH-107892) (cherry picked from commit 2e1f688fe0f0a612e54c09f5a7027a834dd8b8d5) Co-authored-by: wookie184 files: M Doc/whatsnew/3.12.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 79e618588143a..bf9eadcaefc85 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1576,7 +1576,7 @@ Changes in the Python API 1,13-1,17: FSTRING_MIDDLE ' end' 1,17-1,18: FSTRING_END '"' - Additionally, there may be some minor behavioral changes as a consecuence of the + Additionally, there may be some minor behavioral changes as a consequence of the changes required to support :pep:`701`. Some of these changes include: * The ``type`` attribute of the tokens emitted when tokenizing some invalid Python From webhook-mailer at python.org Mon Aug 14 08:40:51 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 14 Aug 2023 12:40:51 -0000 Subject: [Python-checkins] [3.12] gh-107877: Update logging levels reference table with usage criteria. (GH-107894) (#107922) Message-ID: https://github.com/python/cpython/commit/ba8ab4e8aecfb5b34b591bf93bd1b31f62b6e0a3 commit: ba8ab4e8aecfb5b34b591bf93bd1b31f62b6e0a3 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-14T14:40:47+02:00 summary: [3.12] gh-107877: Update logging levels reference table with usage criteria. (GH-107894) (#107922) gh-107877: Update logging levels reference table with usage criteria. (GH-107894) (cherry picked from commit cc2cf85d03cf29994a707aae5cc9a349a4165b84) Co-authored-by: Vinay Sajip Co-authored-by: Alex Waygood files: M Doc/library/logging.rst diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 70a9c8b9aa86e..a92ef1787af74 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -397,21 +397,39 @@ have specific values relative to the predefined levels. If you define a level with the same numeric value, it overwrites the predefined value; the predefined name is lost. -+-----------------------+---------------+ -| Level | Numeric value | -+=======================+===============+ -| .. py:data:: CRITICAL | 50 | -+-----------------------+---------------+ -| .. py:data:: ERROR | 40 | -+-----------------------+---------------+ -| .. py:data:: WARNING | 30 | -+-----------------------+---------------+ -| .. py:data:: INFO | 20 | -+-----------------------+---------------+ -| .. py:data:: DEBUG | 10 | -+-----------------------+---------------+ -| .. py:data:: NOTSET | 0 | -+-----------------------+---------------+ ++-----------------------+---------------+-------------------------------------+ +| Level | Numeric value | What it means / When to use it | ++=======================+===============+=====================================+ +| .. py:data:: NOTSET | 0 | When set on a logger, indicates that| +| | | ancestor loggers are to be consulted| +| | | to determine the effective level. | +| | | If that still resolves to | +| | | :const:`!NOTSET`, then all events | +| | | are logged. When set on a handler, | +| | | all events are handled. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: DEBUG | 10 | Detailed information, typically only| +| | | of interest to a developer trying to| +| | | diagnose a problem. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: INFO | 20 | Confirmation that things are working| +| | | as expected. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: WARNING | 30 | An indication that something | +| | | unexpected happened, or that a | +| | | problem might occur in the near | +| | | future (e.g. 'disk space low'). The | +| | | software is still working as | +| | | expected. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: ERROR | 40 | Due to a more serious problem, the | +| | | software has not been able to | +| | | perform some function. | ++-----------------------+---------------+-------------------------------------+ +| .. py:data:: CRITICAL | 50 | A serious error, indicating that the| +| | | program itself may be unable to | +| | | continue running. | ++-----------------------+---------------+-------------------------------------+ .. _handler: From webhook-mailer at python.org Mon Aug 14 10:37:48 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Mon, 14 Aug 2023 14:37:48 -0000 Subject: [Python-checkins] gh-107938: Synchonise the signature of of sqlite3.connect and sqlite3.Connection.__init__ (#107939) Message-ID: https://github.com/python/cpython/commit/6fbaba552a52f93ecbe8be000888afa0b65b967e commit: 6fbaba552a52f93ecbe8be000888afa0b65b967e branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-14T16:37:44+02:00 summary: gh-107938: Synchonise the signature of of sqlite3.connect and sqlite3.Connection.__init__ (#107939) files: A Modules/_sqlite/clinic/_sqlite3.connect.c.h M Modules/_sqlite/connection.c M Modules/_sqlite/module.c diff --git a/Modules/_sqlite/clinic/_sqlite3.connect.c.h b/Modules/_sqlite/clinic/_sqlite3.connect.c.h new file mode 100644 index 0000000000000..0c4ebdf0590c9 --- /dev/null +++ b/Modules/_sqlite/clinic/_sqlite3.connect.c.h @@ -0,0 +1,25 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(pysqlite_connect__doc__, +"connect($module, /, database, timeout=5.0, detect_types=0,\n" +" isolation_level=\'\', check_same_thread=True,\n" +" factory=ConnectionType, cached_statements=128, uri=False, *,\n" +" autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL)\n" +"--\n" +"\n" +"Open a connection to the SQLite database file \'database\'.\n" +"\n" +"You can use \":memory:\" to open a database connection to a database that\n" +"resides in RAM instead of on disk."); + +#define PYSQLITE_CONNECT_METHODDEF \ + {"connect", _PyCFunction_CAST(pysqlite_connect), METH_FASTCALL|METH_KEYWORDS, pysqlite_connect__doc__}, +/*[clinic end generated code: output=6a8458c9edf8fb7f input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index ddd7ace81198b..3d2fcd3ae0788 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -212,7 +212,6 @@ class sqlite3_int64_converter(CConverter): [python start generated code]*/ /*[python end generated code: output=da39a3ee5e6b4b0d input=dff8760fb1eba6a1]*/ -// NB: This needs to be in sync with the sqlite3.connect docstring /*[clinic input] _sqlite3.Connection.__init__ as pysqlite_connection_init @@ -346,6 +345,34 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, return -1; } +/*[clinic input] +# Create a new destination 'connect' for the docstring and methoddef only. +# This makes it possible to keep the signatures for Connection.__init__ and +# sqlite3.connect() synchronised. +output push +destination connect new file '{dirname}/clinic/_sqlite3.connect.c.h' + +# Only output the docstring and the PyMethodDef entry. +output everything suppress +output docstring_definition connect +output methoddef_define connect + +# Define the sqlite3.connect function by cloning Connection.__init__. +_sqlite3.connect as pysqlite_connect = _sqlite3.Connection.__init__ + +Open a connection to the SQLite database file 'database'. + +You can use ":memory:" to open a database connection to a database that +resides in RAM instead of on disk. +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=92260edff95d1720]*/ + +/*[clinic input] +# Restore normal Argument Clinic operation for the rest of this file. +output pop +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b899ba9273edcce7]*/ + #define VISIT_CALLBACK_CONTEXT(ctx) \ do { \ if (ctx) { \ diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 368e581b4f335..0c503dfbebbd6 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -48,25 +48,16 @@ module _sqlite3 [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=81e330492d57488e]*/ -// NB: This needs to be in sync with the Connection.__init__ docstring. -PyDoc_STRVAR(module_connect_doc, -"connect($module, /, database, timeout=5.0, detect_types=0,\n" -" isolation_level='', check_same_thread=True,\n" -" factory=ConnectionType, cached_statements=128, uri=False, *,\n" -" autocommit=sqlite3.LEGACY_TRANSACTION_CONTROL)\n" -"--\n" -"\n" -"Opens a connection to the SQLite database file database.\n" -"\n" -"You can use \":memory:\" to open a database connection to a database that resides\n" -"in RAM instead of on disk."); - -#define PYSQLITE_CONNECT_METHODDEF \ - {"connect", _PyCFunction_CAST(module_connect), METH_FASTCALL|METH_KEYWORDS, module_connect_doc}, +/* + * We create 'clinic/_sqlite3.connect.c.h' in connection.c, in order to + * keep the signatures of sqlite3.Connection.__init__ and + * sqlite3.connect() synchronised. + */ +#include "clinic/_sqlite3.connect.c.h" static PyObject * -module_connect(PyObject *module, PyObject *const *args, Py_ssize_t nargsf, - PyObject *kwnames) +pysqlite_connect(PyObject *module, PyObject *const *args, Py_ssize_t nargsf, + PyObject *kwnames) { pysqlite_state *state = pysqlite_get_state(module); PyObject *factory = (PyObject *)state->ConnectionType; From webhook-mailer at python.org Mon Aug 14 10:58:59 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 14 Aug 2023 14:58:59 -0000 Subject: [Python-checkins] [3.12] Add another example to the statistics docs (GH-107904) (#107941) Message-ID: https://github.com/python/cpython/commit/e8963a86ead0f5516948b3ad55b410b7fc5fb194 commit: e8963a86ead0f5516948b3ad55b410b7fc5fb194 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-14T16:58:54+02:00 summary: [3.12] Add another example to the statistics docs (GH-107904) (#107941) Add another example to the statistics docs (GH-107904) (cherry picked from commit 2b6dc2accc315ce279d259ed39e058a225068531) Co-authored-by: Raymond Hettinger files: A Doc/library/kde_example.png M Doc/library/statistics.rst diff --git a/Doc/library/kde_example.png b/Doc/library/kde_example.png new file mode 100644 index 0000000000000..f450489569997 Binary files /dev/null and b/Doc/library/kde_example.png differ diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 395b324c86038..483ebea67f0c6 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -922,6 +922,10 @@ of applications in statistics. :class:`NormalDist` Examples and Recipes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Classic probability problems +**************************** + :class:`NormalDist` readily solves classic probability problems. For example, given `historical data for SAT exams @@ -947,6 +951,10 @@ Find the `quartiles `_ and `deciles >>> list(map(round, sat.quantiles(n=10))) [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310] + +Monte Carlo inputs for simulations +********************************** + To estimate the distribution for a model than isn't easy to solve analytically, :class:`NormalDist` can generate input samples for a `Monte Carlo simulation `_: @@ -963,6 +971,9 @@ Carlo simulation `_: >>> quantiles(map(model, X, Y, Z)) # doctest: +SKIP [1.4591308524824727, 1.8035946855390597, 2.175091447274739] +Approximating binomial distributions +************************************ + Normal distributions can be used to approximate `Binomial distributions `_ when the sample size is large and when the probability of a successful @@ -1000,6 +1011,10 @@ probability that the Python room will stay within its capacity limits? >>> mean(trial() <= k for i in range(10_000)) 0.8398 + +Naive bayesian classifier +************************* + Normal distributions commonly arise in machine learning problems. Wikipedia has a `nice example of a Naive Bayesian Classifier @@ -1054,6 +1069,48 @@ The final prediction goes to the largest posterior. This is known as the 'female' +Kernel density estimation +************************* + +It is possible to estimate a continuous probability density function +from a fixed number of discrete samples. + +The basic idea is to smooth the data using `a kernel function such as a +normal distribution, triangular distribution, or uniform distribution +`_. +The degree of smoothing is controlled by a single +parameter, ``h``, representing the variance of the kernel function. + +.. testcode:: + + import math + + def kde_normal(sample, h): + "Create a continous probability density function from a sample." + # Smooth the sample with a normal distribution of variance h. + kernel_h = NormalDist(0.0, math.sqrt(h)).pdf + n = len(sample) + def pdf(x): + return sum(kernel_h(x - x_i) for x_i in sample) / n + return pdf + +`Wikipedia has an example +`_ +where we can use the ``kde_normal()`` recipe to generate and plot +a probability density function estimated from a small sample: + +.. doctest:: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde_normal(sample, h=2.25) + >>> xarr = [i/100 for i in range(-750, 1100)] + >>> yarr = [f_hat(x) for x in xarr] + +The points in ``xarr`` and ``yarr`` can be used to make a PDF plot: + +.. image:: kde_example.png + :alt: Scatter plot of the estimated probability density function. + .. # This modelines must appear within the last ten lines of the file. kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; From webhook-mailer at python.org Mon Aug 14 14:36:33 2023 From: webhook-mailer at python.org (iritkatriel) Date: Mon, 14 Aug 2023 18:36:33 -0000 Subject: [Python-checkins] gh-105481: reduce repetition in opcode metadata generation code (#107942) Message-ID: https://github.com/python/cpython/commit/39745347f645ac99021f4ab981ff02ab5647b19c commit: 39745347f645ac99021f4ab981ff02ab5647b19c branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-14T18:36:29Z summary: gh-105481: reduce repetition in opcode metadata generation code (#107942) files: M Include/internal/pycore_opcode_metadata.h M Tools/cases_generator/generate_cases.py diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 9f4437c09e92c..c0a2e9d91c4f4 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -56,11 +56,9 @@ #define _POP_JUMP_IF_TRUE 332 #define JUMP_TO_TOP 333 -#ifndef NEED_OPCODE_METADATA extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); -#else -int -_PyOpcode_num_popped(int opcode, int oparg, bool jump) { +#ifdef NEED_OPCODE_METADATA +int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { switch(opcode) { case NOP: return 0; @@ -500,13 +498,11 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return -1; } } -#endif +#endif // NEED_OPCODE_METADATA -#ifndef NEED_OPCODE_METADATA extern int _PyOpcode_num_pushed(int opcode, int oparg, bool jump); -#else -int -_PyOpcode_num_pushed(int opcode, int oparg, bool jump) { +#ifdef NEED_OPCODE_METADATA +int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { switch(opcode) { case NOP: return 0; @@ -946,7 +942,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return -1; } } -#endif +#endif // NEED_OPCODE_METADATA enum InstructionFormat { INSTR_FMT_IB, @@ -1004,11 +1000,8 @@ struct opcode_macro_expansion { #define OPCODE_UOP_NAME_SIZE 512 #define OPCODE_MACRO_EXPANSION_SIZE 256 -#ifndef NEED_OPCODE_METADATA extern const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE]; -extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]; -extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]; -#else // if NEED_OPCODE_METADATA +#ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [NOP] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1228,6 +1221,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [CACHE] = { true, INSTR_FMT_IX, 0 }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, }; +#endif // NEED_OPCODE_METADATA + +extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]; +#ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE] = { [NOP] = { .nuops = 1, .uops = { { NOP, 0, 0 } } }, [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { LOAD_FAST_CHECK, 0, 0 } } }, @@ -1357,6 +1354,10 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [BINARY_OP] = { .nuops = 1, .uops = { { BINARY_OP, 0, 0 } } }, [SWAP] = { .nuops = 1, .uops = { { SWAP, 0, 0 } } }, }; +#endif // NEED_OPCODE_METADATA + +extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]; +#ifdef NEED_OPCODE_METADATA const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [EXIT_TRACE] = "EXIT_TRACE", [SAVE_IP] = "SAVE_IP", diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index d35a16a80e8d0..e78b0edff9a0c 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -4,6 +4,7 @@ """ import argparse +import contextlib import os import posixpath import sys @@ -141,6 +142,15 @@ def effect_str(effects: list[StackEffect]) -> str: typing.assert_never(thing) return instr, popped, pushed + @contextlib.contextmanager + def metadata_item(self, signature, open, close): + self.out.emit("") + self.out.emit(f"extern {signature};") + self.out.emit("#ifdef NEED_OPCODE_METADATA") + with self.out.block(f"{signature} {open}", close): + yield + self.out.emit("#endif // NEED_OPCODE_METADATA") + def write_stack_effect_functions(self) -> None: popped_data: list[tuple[AnyInstruction, str]] = [] pushed_data: list[tuple[AnyInstruction, str]] = [] @@ -156,25 +166,16 @@ def write_stack_effect_functions(self) -> None: def write_function( direction: str, data: list[tuple[AnyInstruction, str]] ) -> None: - self.out.emit("") - self.out.emit("#ifndef NEED_OPCODE_METADATA") - self.out.emit( - f"extern int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump);" - ) - self.out.emit("#else") - self.out.emit("int") - self.out.emit( - f"_PyOpcode_num_{direction}(int opcode, int oparg, bool jump) {{" - ) - self.out.emit(" switch(opcode) {") - for instr, effect in data: - self.out.emit(f" case {instr.name}:") - self.out.emit(f" return {effect};") - self.out.emit(" default:") - self.out.emit(" return -1;") - self.out.emit(" }") - self.out.emit("}") - self.out.emit("#endif") + + with self.metadata_item( + f"int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump)", "", "" + ): + with self.out.block("switch(opcode)"): + for instr, effect in data: + self.out.emit(f"case {instr.name}:") + self.out.emit(f" return {effect};") + self.out.emit("default:") + self.out.emit(" return -1;") write_function("popped", popped_data) write_function("pushed", pushed_data) @@ -290,48 +291,33 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.out.emit("#define OPCODE_METADATA_SIZE 512") self.out.emit("#define OPCODE_UOP_NAME_SIZE 512") self.out.emit("#define OPCODE_MACRO_EXPANSION_SIZE 256") - self.out.emit("") - self.out.emit("#ifndef NEED_OPCODE_METADATA") - self.out.emit( - "extern const struct opcode_metadata " - "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE];" - ) - self.out.emit( - "extern const struct opcode_macro_expansion " - "_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE];" - ) - self.out.emit( - "extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE];" - ) - self.out.emit("#else // if NEED_OPCODE_METADATA") - self.out.emit( + with self.metadata_item( "const struct opcode_metadata " - "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {" - ) - - # Write metadata for each instruction - for thing in self.everything: - match thing: - case OverriddenInstructionPlaceHolder(): - continue - case parsing.InstDef(): - if thing.kind != "op": - self.write_metadata_for_inst(self.instrs[thing.name]) - case parsing.Macro(): - self.write_metadata_for_macro(self.macro_instrs[thing.name]) - case parsing.Pseudo(): - self.write_metadata_for_pseudo(self.pseudo_instrs[thing.name]) - case _: - typing.assert_never(thing) - - # Write end of array - self.out.emit("};") + "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE]", + "=", + ";" + ): + # Write metadata for each instruction + for thing in self.everything: + match thing: + case OverriddenInstructionPlaceHolder(): + continue + case parsing.InstDef(): + if thing.kind != "op": + self.write_metadata_for_inst(self.instrs[thing.name]) + case parsing.Macro(): + self.write_metadata_for_macro(self.macro_instrs[thing.name]) + case parsing.Pseudo(): + self.write_metadata_for_pseudo(self.pseudo_instrs[thing.name]) + case _: + typing.assert_never(thing) - with self.out.block( + with self.metadata_item( "const struct opcode_macro_expansion " - "_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE] =", - ";", + "_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]", + "=", + ";" ): # Write macro expansion for each non-pseudo instruction for thing in self.everything: @@ -360,13 +346,11 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No case _: typing.assert_never(thing) - with self.out.block( - "const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] =", ";" + with self.metadata_item( + "const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]", "=", ";" ): self.write_uop_items(lambda name, counter: f'[{name}] = "{name}",') - self.out.emit("#endif // NEED_OPCODE_METADATA") - with open(pymetadata_filename, "w") as f: # Create formatter self.out = Formatter(f, 0, comment="#") @@ -511,7 +495,7 @@ def emit_metadata_entry(self, name: str, fmt: str, flags: InstructionFlags) -> N if not flag_names: flag_names.append("0") self.out.emit( - f" [{name}] = {{ true, {INSTR_FMT_PREFIX}{fmt}," + f"[{name}] = {{ true, {INSTR_FMT_PREFIX}{fmt}," f" {' | '.join(flag_names)} }}," ) From webhook-mailer at python.org Mon Aug 14 17:41:31 2023 From: webhook-mailer at python.org (gvanrossum) Date: Mon, 14 Aug 2023 21:41:31 -0000 Subject: [Python-checkins] Attempt to speed up deepfreeze.py (#107887) Message-ID: https://github.com/python/cpython/commit/a2a4b9f1ec86b9762a5d35895ac5b528e03d5b98 commit: a2a4b9f1ec86b9762a5d35895ac5b528e03d5b98 branch: main author: Guido van Rossum committer: gvanrossum date: 2023-08-14T14:41:27-07:00 summary: Attempt to speed up deepfreeze.py (#107887) * Instead of calling get_identifiers_and_strings(), extract identifiers and strings from pycore_global_strings.h. * Avoid ast.literal_eval(), it's very slow. files: M Makefile.pre.in M Tools/build/deepfreeze.py diff --git a/Makefile.pre.in b/Makefile.pre.in index 52236f7924503..3a628bf49e97c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1253,7 +1253,7 @@ regen-frozen: Tools/build/freeze_modules.py $(FROZEN_FILES_IN) .PHONY: regen-deepfreeze regen-deepfreeze: $(DEEPFREEZE_OBJS) -DEEPFREEZE_DEPS=$(srcdir)/Tools/build/deepfreeze.py $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT) +DEEPFREEZE_DEPS=$(srcdir)/Tools/build/deepfreeze.py Include/internal/pycore_global_strings.h $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT) # BEGIN: deepfreeze modules Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS) diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index a11fe6a62811a..ce609bd089874 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -6,7 +6,6 @@ by Python 3.10, and 3.11 features are not available. """ import argparse -import ast import builtins import collections import contextlib @@ -17,10 +16,10 @@ from typing import Dict, FrozenSet, TextIO, Tuple import umarshal -from generate_global_objects import get_identifiers_and_strings + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) verbose = False -identifiers, strings = get_identifiers_and_strings() # This must be kept in sync with opcode.py RESUME = 151 @@ -114,6 +113,7 @@ def __init__(self, file: TextIO) -> None: self.hits, self.misses = 0, 0 self.finis: list[str] = [] self.inits: list[str] = [] + self.identifiers, self.strings = self.get_identifiers_and_strings() self.write('#include "Python.h"') self.write('#include "internal/pycore_gc.h"') self.write('#include "internal/pycore_code.h"') @@ -121,6 +121,19 @@ def __init__(self, file: TextIO) -> None: self.write('#include "internal/pycore_long.h"') self.write("") + def get_identifiers_and_strings(self) -> tuple[set[str], dict[str, str]]: + filename = os.path.join(ROOT, "Include", "internal", "pycore_global_strings.h") + with open(filename) as fp: + lines = fp.readlines() + identifiers: set[str] = set() + strings: dict[str, str] = {} + for line in lines: + if m := re.search(r"STRUCT_FOR_ID\((\w+)\)", line): + identifiers.add(m.group(1)) + if m := re.search(r'STRUCT_FOR_STR\((\w+), "(.*?)"\)', line): + strings[m.group(2)] = m.group(1) + return identifiers, strings + @contextlib.contextmanager def indent(self) -> None: save_level = self.level @@ -171,9 +184,9 @@ def generate_bytes(self, name: str, b: bytes) -> str: return f"& {name}.ob_base.ob_base" def generate_unicode(self, name: str, s: str) -> str: - if s in strings: - return f"&_Py_STR({strings[s]})" - if s in identifiers: + if s in self.strings: + return f"&_Py_STR({self.strings[s]})" + if s in self.identifiers: return f"&_Py_ID({s})" if len(s) == 1: c = ord(s) @@ -441,12 +454,10 @@ def is_frozen_header(source: str) -> bool: def decode_frozen_data(source: str) -> types.CodeType: - lines = source.splitlines() - while lines and re.match(FROZEN_DATA_LINE, lines[0]) is None: - del lines[0] - while lines and re.match(FROZEN_DATA_LINE, lines[-1]) is None: - del lines[-1] - values: Tuple[int, ...] = ast.literal_eval("".join(lines).strip()) + values: list[int] = [] + for line in source.splitlines(): + if re.match(FROZEN_DATA_LINE, line): + values.extend([int(x) for x in line.split(",") if x.strip()]) data = bytes(values) return umarshal.loads(data) From webhook-mailer at python.org Mon Aug 14 18:37:31 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Mon, 14 Aug 2023 22:37:31 -0000 Subject: [Python-checkins] Convert the GitHub issue templates into GitHub issue forms (#107920) Message-ID: https://github.com/python/cpython/commit/8d3cb1bc4b5de091d7b5fcc5ce7378151a8f4f45 commit: 8d3cb1bc4b5de091d7b5fcc5ce7378151a8f4f45 branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-14T22:37:27Z summary: Convert the GitHub issue templates into GitHub issue forms (#107920) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Hugo van Kemenade Co-authored-by: Ezio Melotti files: A .github/ISSUE_TEMPLATE/bug.yml A .github/ISSUE_TEMPLATE/crash.yml A .github/ISSUE_TEMPLATE/feature.yml D .github/ISSUE_TEMPLATE/bug.md D .github/ISSUE_TEMPLATE/crash.md D .github/ISSUE_TEMPLATE/feature.md diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md deleted file mode 100644 index 47037cd319e7e..0000000000000 --- a/.github/ISSUE_TEMPLATE/bug.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: Bug report -about: Submit a bug report -labels: "type-bug" ---- - - - -# Bug report - -## Checklist - - - -- [ ] I am confident this is a bug in CPython, not a bug in a third-party project -- [ ] I have searched the CPython issue tracker, and am confident this bug has not been reported before - -## A clear and concise description of the bug - - - - - -# Your environment - - - -- CPython versions tested on: -- Operating system and architecture: - - diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 0000000000000..05f4f317ccc97 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,69 @@ +name: Bug report +description: Submit a bug report +labels: ["type-bug"] +body: + - type: markdown + attributes: + value: | + **New to Python?** + + For help or advice on using Python, try one of the following options instead of opening a GitHub issue: + + - Posting on [Discourse](https://discuss.python.org/c/users/7) + - Reading the [Python tutorial](https://docs.python.org/3/tutorial/) + - Emailing [python-list](https://mail.python.org/mailman/listinfo/python-list) + - type: checkboxes + attributes: + label: Checklist + description: A bug in a third-party project (for example, `pip` or `requests`) should be reported to that project's issue tracker, not CPython + options: + - label: I am confident this is a bug in CPython, not a bug in a third-party project + required: false + - label: | + I have searched the [CPython issue tracker](https://github.com/python/cpython/issues?q=is%3Aissue+sort%3Acreated-desc), + and am confident this bug has not been reported before + required: false + - type: dropdown + attributes: + label: "CPython versions tested on:" + multiple: true + options: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "CPython main branch" + validations: + required: true + - type: dropdown + attributes: + label: "Operating systems tested on:" + multiple: true + options: + - Linux + - macOS + - Windows + - Other + validations: + required: false + - type: input + attributes: + label: "Output from running 'python -VV' on the command line:" + description: If you tested with multiple operating systems or architectures, feel free to provide details in the main bug description. + validations: + required: false + - type: textarea + attributes: + label: "A clear and concise description of the bug:" + description: > + Tell us what happened. + Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible. + Put any code blocks inside triple backticks. + + value: | + ```python + # Add a code block here, if required + ``` + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/crash.md b/.github/ISSUE_TEMPLATE/crash.md deleted file mode 100644 index a268249d1c1e6..0000000000000 --- a/.github/ISSUE_TEMPLATE/crash.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -name: Crash report -about: A hard crash of the interpreter, possibly with a core dump -labels: "type-crash" ---- - - - -# Crash report - - - - - -# Error messages - - - - - -# Your environment - - - -- CPython versions tested on: -- Operating system and architecture: - - diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml new file mode 100644 index 0000000000000..1ea84b893697a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/crash.yml @@ -0,0 +1,71 @@ +name: Crash report +description: A hard crash of the interpreter, possibly with a core dump +labels: ["type-crash"] +body: + - type: markdown + attributes: + value: | + This form is for hard crashes of the Python interpreter, segmentation faults, failed C-level assertions, and similar. Unexpected exceptions raised from Python functions in the standard library count as bugs rather than crashes. + + The CPython interpreter is written in a different programming language, C. A "CPython crash" is when Python itself fails, leading to a traceback in the C stack. + - type: dropdown + attributes: + label: "CPython versions tested on:" + multiple: true + options: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "CPython main branch" + validations: + required: true + - type: dropdown + attributes: + label: "Operating systems tested on:" + multiple: true + options: + - Linux + - macOS + - Windows + - Other + validations: + required: false + - type: input + attributes: + label: "Output from running 'python -VV' on the command line:" + description: If you tested with multiple operating systems or architectures, feel free to provide details in the main bug description. + validations: + required: false + - type: textarea + attributes: + label: What happened? + description: > + Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible. + Put any code blocks inside triple backticks. + + value: | + ```python + # Add a code block here, if required + ``` + validations: + required: true + - type: textarea + attributes: + label: Error messages + description: > + Enter any error messages caused by the crash, including a core dump if there is one. + Feel free to leave this bit blank if it isn't relevant. + placeholder: | + Error messages should be formatted like this: + +
+ Error messages/core dump + + ``` + # paste errors here, if you have any + ``` +
+ validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature.md b/.github/ISSUE_TEMPLATE/feature.md deleted file mode 100644 index 7e96bc9df665c..0000000000000 --- a/.github/ISSUE_TEMPLATE/feature.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: Feature or enhancement -about: Submit a proposal for a new CPython feature or enhancement -labels: "type-feature" ---- - - - -# Feature or enhancement - - - - - -# Pitch - - - - - -# Previous discussion - - - - - - diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 0000000000000..a1c48bbff829c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,42 @@ +name: Feature or enhancement +description: Submit a proposal for a new CPython feature or enhancement +labels: ["type-feature"] +body: + - type: markdown + attributes: + value: | + # Proposing a feature to CPython? + + You'll need to demonstrate widespread support for your idea among the community. + + Major feature proposals should generally be discussed on [Discourse](https://discuss.python.org/c/ideas/6) before opening a GitHub issue. Wait until it's clear that most people support your idea before filling in this form. + - type: checkboxes + attributes: + label: Has this already been discussed elsewhere? + options: + - label: I have already discussed this feature proposal on Discourse + - label: This is a minor feature, which does not need previous discussion elsewhere + - type: textarea + attributes: + label: "Links to previous discussion of this feature:" + validations: + required: false + - type: input + attributes: + label: "Summary of proposal:" + description: A one-line summary of your proposal. + validations: + required: true + - type: textarea + attributes: + label: "Pitch:" + description: > + Explain why this feature or enhancement should be implemented and how it would be used. + Add examples, if applicable. + Put any code blocks inside triple backticks. + value: | + ```python + # Add a code block here, if required + ``` + validations: + required: true From webhook-mailer at python.org Mon Aug 14 19:26:01 2023 From: webhook-mailer at python.org (corona10) Date: Mon, 14 Aug 2023 23:26:01 -0000 Subject: [Python-checkins] =?utf-8?q?Revert_=22gh-104469_=3A_Convert_=5Ft?= =?utf-8?q?estcapi/vectorcall=5Flimited=2Ec_to_use_AC_=E2=80=A6_=28gh-1079?= =?utf-8?b?NTEp?= Message-ID: https://github.com/python/cpython/commit/580f357c663bd704a0903e4174cdf458cac56416 commit: 580f357c663bd704a0903e4174cdf458cac56416 branch: main author: Dong-hee Na committer: corona10 date: 2023-08-14T23:25:57Z summary: Revert "gh-104469 : Convert _testcapi/vectorcall_limited.c to use AC ? (gh-107951) Revert "gh-104469 : Convert _testcapi/vectorcall_limited.c to use AC (gh-107857)" This reverts commit 2e27da18952c6561f48dab706b5911135cedd7cf. files: D Modules/_testcapi/clinic/vectorcall_limited.c.h M Modules/_testcapi/vectorcall_limited.c diff --git a/Modules/_testcapi/clinic/vectorcall_limited.c.h b/Modules/_testcapi/clinic/vectorcall_limited.c.h deleted file mode 100644 index bc89f58b65739..0000000000000 --- a/Modules/_testcapi/clinic/vectorcall_limited.c.h +++ /dev/null @@ -1,42 +0,0 @@ -/*[clinic input] -preserve -[clinic start generated code]*/ - -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - -#if defined(LIMITED_API_AVAILABLE) - -PyDoc_STRVAR(_testcapi_call_vectorcall__doc__, -"call_vectorcall($module, callable, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_CALL_VECTORCALL_METHODDEF \ - {"call_vectorcall", (PyCFunction)_testcapi_call_vectorcall, METH_O, _testcapi_call_vectorcall__doc__}, - -#endif /* defined(LIMITED_API_AVAILABLE) */ - -#if defined(LIMITED_API_AVAILABLE) - -PyDoc_STRVAR(_testcapi_call_vectorcall_method__doc__, -"call_vectorcall_method($module, callable, /)\n" -"--\n" -"\n"); - -#define _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF \ - {"call_vectorcall_method", (PyCFunction)_testcapi_call_vectorcall_method, METH_O, _testcapi_call_vectorcall_method__doc__}, - -#endif /* defined(LIMITED_API_AVAILABLE) */ - -#ifndef _TESTCAPI_CALL_VECTORCALL_METHODDEF - #define _TESTCAPI_CALL_VECTORCALL_METHODDEF -#endif /* !defined(_TESTCAPI_CALL_VECTORCALL_METHODDEF) */ - -#ifndef _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF - #define _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF -#endif /* !defined(_TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF) */ -/*[clinic end generated code: output=409028b637aba77b input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/vectorcall_limited.c b/Modules/_testcapi/vectorcall_limited.c index 9f02c7afa7c97..a96925e840121 100644 --- a/Modules/_testcapi/vectorcall_limited.c +++ b/Modules/_testcapi/vectorcall_limited.c @@ -1,6 +1,5 @@ #define Py_LIMITED_API 0x030c0000 // 3.12 #include "parts.h" -#include "clinic/vectorcall_limited.c.h" #ifdef LIMITED_API_AVAILABLE @@ -8,11 +7,6 @@ /* Test Vectorcall in the limited API */ -/*[clinic input] -module _testcapi -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/ - static PyObject * LimitedVectorCallClass_tpcall(PyObject *self, PyObject *args, PyObject *kwargs) { return PyUnicode_FromString("tp_call called"); @@ -38,16 +32,8 @@ LimitedVectorCallClass_new(PyTypeObject *tp, PyTypeObject *a, PyTypeObject *kw) return self; } -/*[clinic input] -_testcapi.call_vectorcall - - callable: object - / -[clinic start generated code]*/ - static PyObject * -_testcapi_call_vectorcall(PyObject *module, PyObject *callable) -/*[clinic end generated code: output=bae81eec97fcaad7 input=55d88f92240957ee]*/ +call_vectorcall(PyObject* self, PyObject *callable) { PyObject *args[3] = { NULL, NULL, NULL }; PyObject *kwname = NULL, *kwnames = NULL, *result = NULL; @@ -91,16 +77,8 @@ _testcapi_call_vectorcall(PyObject *module, PyObject *callable) return result; } -/*[clinic input] -_testcapi.call_vectorcall_method - - callable: object - / -[clinic start generated code]*/ - static PyObject * -_testcapi_call_vectorcall_method(PyObject *module, PyObject *callable) -/*[clinic end generated code: output=e661f48dda08b6fb input=5ba81c27511395b6]*/ +call_vectorcall_method(PyObject* self, PyObject *callable) { PyObject *args[3] = { NULL, NULL, NULL }; PyObject *name = NULL, *kwname = NULL, @@ -175,8 +153,8 @@ static PyType_Spec LimitedVectorCallClass_spec = { }; static PyMethodDef TestMethods[] = { - _TESTCAPI_CALL_VECTORCALL_METHODDEF - _TESTCAPI_CALL_VECTORCALL_METHOD_METHODDEF + {"call_vectorcall", call_vectorcall, METH_O}, + {"call_vectorcall_method", call_vectorcall_method, METH_O}, {NULL}, }; From webhook-mailer at python.org Tue Aug 15 03:23:58 2023 From: webhook-mailer at python.org (vsajip) Date: Tue, 15 Aug 2023 07:23:58 -0000 Subject: [Python-checkins] gh-76913: Add "merge extras" feature to LoggerAdapter (GH-107292) Message-ID: https://github.com/python/cpython/commit/a482e5bf0022f85424a6308529a9ad51f1bfbb71 commit: a482e5bf0022f85424a6308529a9ad51f1bfbb71 branch: main author: Romuald Brunet committer: vsajip date: 2023-08-15T08:23:54+01:00 summary: gh-76913: Add "merge extras" feature to LoggerAdapter (GH-107292) files: A Misc/NEWS.d/next/Library/2023-08-14-17-15-59.gh-issue-76913.LLD0rT.rst M Doc/library/logging.rst M Lib/logging/__init__.py M Lib/test/test_logging.py diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index b582c918df023..49e870e9e2479 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1002,10 +1002,14 @@ LoggerAdapter Objects information into logging calls. For a usage example, see the section on :ref:`adding contextual information to your logging output `. -.. class:: LoggerAdapter(logger, extra) +.. class:: LoggerAdapter(logger, extra, merge_extra=False) Returns an instance of :class:`LoggerAdapter` initialized with an - underlying :class:`Logger` instance and a dict-like object. + underlying :class:`Logger` instance, a dict-like object (*extra*), and a + boolean (*merge_extra*) indicating whether or not the *extra* argument of + individual log calls should be merged with the :class:`LoggerAdapter` extra. + The default behavior is to ignore the *extra* argument of individual log + calls and only use the one of the :class:`LoggerAdapter` instance .. method:: process(msg, kwargs) @@ -1037,6 +1041,9 @@ interchangeably. Remove the undocumented ``warn()`` method which was an alias to the ``warning()`` method. +.. versionchanged:: 3.13 + The *merge_extra* argument was added. + Thread Safety ------------- diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 527fc5c631730..2d228e563094c 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1879,7 +1879,7 @@ class LoggerAdapter(object): information in logging output. """ - def __init__(self, logger, extra=None): + def __init__(self, logger, extra=None, merge_extra=False): """ Initialize the adapter with a logger and a dict-like object which provides contextual information. This constructor signature allows @@ -1889,9 +1889,20 @@ def __init__(self, logger, extra=None): following example: adapter = LoggerAdapter(someLogger, dict(p1=v1, p2="v2")) + + By default, LoggerAdapter objects will drop the "extra" argument + passed on the individual log calls to use its own instead. + + Initializing it with merge_extra=True will instead merge both + maps when logging, the individual call extra taking precedence + over the LoggerAdapter instance extra + + .. versionchanged:: 3.13 + The *merge_extra* argument was added. """ self.logger = logger self.extra = extra + self.merge_extra = merge_extra def process(self, msg, kwargs): """ @@ -1903,7 +1914,10 @@ def process(self, msg, kwargs): Normally, you'll only need to override this one method in a LoggerAdapter subclass for your specific needs. """ - kwargs["extra"] = self.extra + if self.merge_extra and "extra" in kwargs: + kwargs["extra"] = {**self.extra, **kwargs["extra"]} + else: + kwargs["extra"] = self.extra return msg, kwargs # diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index def976fbe96ba..f26846f9663e5 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -5433,6 +5433,46 @@ def process(self, msg, kwargs): self.assertIs(adapter.manager, orig_manager) self.assertIs(self.logger.manager, orig_manager) + def test_extra_in_records(self): + self.adapter = logging.LoggerAdapter(logger=self.logger, + extra={'foo': '1'}) + + self.adapter.critical('foo should be here') + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertTrue(hasattr(record, 'foo')) + self.assertEqual(record.foo, '1') + + def test_extra_not_merged_by_default(self): + self.adapter.critical('foo should NOT be here', extra={'foo': 'nope'}) + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertFalse(hasattr(record, 'foo')) + + def test_extra_merged(self): + self.adapter = logging.LoggerAdapter(logger=self.logger, + extra={'foo': '1'}, + merge_extra=True) + + self.adapter.critical('foo and bar should be here', extra={'bar': '2'}) + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertTrue(hasattr(record, 'foo')) + self.assertTrue(hasattr(record, 'bar')) + self.assertEqual(record.foo, '1') + self.assertEqual(record.bar, '2') + + def test_extra_merged_log_call_has_precedence(self): + self.adapter = logging.LoggerAdapter(logger=self.logger, + extra={'foo': '1'}, + merge_extra=True) + + self.adapter.critical('foo shall be min', extra={'foo': '2'}) + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertTrue(hasattr(record, 'foo')) + self.assertEqual(record.foo, '2') + class LoggerTest(BaseTest, AssertErrorMessage): diff --git a/Misc/NEWS.d/next/Library/2023-08-14-17-15-59.gh-issue-76913.LLD0rT.rst b/Misc/NEWS.d/next/Library/2023-08-14-17-15-59.gh-issue-76913.LLD0rT.rst new file mode 100644 index 0000000000000..5f9a84e714ae2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-14-17-15-59.gh-issue-76913.LLD0rT.rst @@ -0,0 +1 @@ +Add *merge_extra* parameter/feature to :class:`logging.LoggerAdapter` From webhook-mailer at python.org Tue Aug 15 04:10:00 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 15 Aug 2023 08:10:00 -0000 Subject: [Python-checkins] gh-93057: Deprecate positional use of optional sqlite3.connect() params (#107948) Message-ID: https://github.com/python/cpython/commit/13c36dc9ae5240124932137de4a94d81292c6c5f commit: 13c36dc9ae5240124932137de4a94d81292c6c5f branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-15T08:09:56Z summary: gh-93057: Deprecate positional use of optional sqlite3.connect() params (#107948) files: A Misc/NEWS.d/next/Library/2023-08-14-19-49-02.gh-issue-93057.5nJwO5.rst M Doc/library/sqlite3.rst M Doc/whatsnew/3.13.rst M Lib/test/test_sqlite3/test_dbapi.py M Lib/test/test_sqlite3/test_factory.py M Modules/_sqlite/clinic/_sqlite3.connect.c.h M Modules/_sqlite/clinic/connection.c.h M Modules/_sqlite/connection.c M Modules/_sqlite/module.c diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 50344058c2604..5f1676e5ae2f5 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -355,6 +355,12 @@ Module functions .. versionadded:: 3.12 The *autocommit* parameter. + .. versionchanged:: 3.13 + Positional use of the parameters *timeout*, *detect_types*, + *isolation_level*, *check_same_thread*, *factory*, *cached_statements*, + and *uri* is deprecated. + They will become keyword-only parameters in Python 3.15. + .. function:: complete_statement(statement) Return ``True`` if the string *statement* appears to contain diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 63cdee6cf1a4f..a65a98bae1357 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -219,6 +219,11 @@ Deprecated They will be removed in Python 3.15. (Contributed by Victor Stinner in :gh:`105096`.) +* Passing more than one positional argument to :func:`sqlite3.connect` and the + :class:`sqlite3.Connection` constructor is deprecated. The remaining + parameters will become keyword-only in Python 3.15. + (Contributed by Erlend E. Aasland in :gh:`107948`.) + Pending Removal in Python 3.14 ------------------------------ diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 3f9bd0248a8b9..c9a9e1353938c 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -582,6 +582,19 @@ def test_connection_config(self): with self.assertRaisesRegex(sqlite.IntegrityError, "constraint"): cx.execute("insert into u values(0)") + def test_connect_positional_arguments(self): + regex = ( + r"Passing more than 1 positional argument to sqlite3.connect\(\)" + " is deprecated. Parameters 'timeout', 'detect_types', " + "'isolation_level', 'check_same_thread', 'factory', " + "'cached_statements' and 'uri' will become keyword-only " + "parameters in Python 3.15." + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + sqlite.connect(":memory:", 1.0) + self.assertEqual(cm.filename, __file__) + + class UninitialisedConnectionTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index 7c36135ecadcc..d63589483e104 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -66,7 +66,16 @@ class Factory(sqlite.Connection): def __init__(self, *args, **kwargs): super(Factory, self).__init__(*args, **kwargs) - con = sqlite.connect(":memory:", 5.0, 0, None, True, Factory) + regex = ( + r"Passing more than 1 positional argument to _sqlite3.Connection\(\) " + r"is deprecated. Parameters 'timeout', 'detect_types', " + r"'isolation_level', 'check_same_thread', 'factory', " + r"'cached_statements' and 'uri' will become keyword-only " + r"parameters in Python 3.15." + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + con = sqlite.connect(":memory:", 5.0, 0, None, True, Factory) + self.assertEqual(cm.filename, __file__) self.assertIsNone(con.isolation_level) self.assertIsInstance(con, Factory) diff --git a/Misc/NEWS.d/next/Library/2023-08-14-19-49-02.gh-issue-93057.5nJwO5.rst b/Misc/NEWS.d/next/Library/2023-08-14-19-49-02.gh-issue-93057.5nJwO5.rst new file mode 100644 index 0000000000000..6a4feaac25bc1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-14-19-49-02.gh-issue-93057.5nJwO5.rst @@ -0,0 +1,3 @@ +Passing more than one positional argument to :func:`sqlite3.connect` and the +:class:`sqlite3.Connection` constructor is deprecated. The remaining parameters +will become keyword-only in Python 3.15. Patch by Erlend E. Aasland. diff --git a/Modules/_sqlite/clinic/_sqlite3.connect.c.h b/Modules/_sqlite/clinic/_sqlite3.connect.c.h index 0c4ebdf0590c9..998c8de1e09f1 100644 --- a/Modules/_sqlite/clinic/_sqlite3.connect.c.h +++ b/Modules/_sqlite/clinic/_sqlite3.connect.c.h @@ -18,8 +18,14 @@ PyDoc_STRVAR(pysqlite_connect__doc__, "Open a connection to the SQLite database file \'database\'.\n" "\n" "You can use \":memory:\" to open a database connection to a database that\n" -"resides in RAM instead of on disk."); +"resides in RAM instead of on disk.\n" +"\n" +"Note: Passing more than 1 positional argument to _sqlite3.connect() is\n" +"deprecated. Parameters \'timeout\', \'detect_types\', \'isolation_level\',\n" +"\'check_same_thread\', \'factory\', \'cached_statements\' and \'uri\' will\n" +"become keyword-only parameters in Python 3.15.\n" +""); #define PYSQLITE_CONNECT_METHODDEF \ {"connect", _PyCFunction_CAST(pysqlite_connect), METH_FASTCALL|METH_KEYWORDS, pysqlite_connect__doc__}, -/*[clinic end generated code: output=6a8458c9edf8fb7f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8d49736db880f09a input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index e869d7d9e9384..af98d61ea7ccc 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -59,6 +59,39 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) int uri = 0; enum autocommit_mode autocommit = LEGACY_TRANSACTION_CONTROL; + // Emit compiler warnings when we get to Python 3.15. + #if PY_VERSION_HEX >= 0x030f00C0 + # error \ + "In connection.c, update parameter(s) 'timeout', 'detect_types', " \ + "'isolation_level', 'check_same_thread', 'factory', " \ + "'cached_statements' and 'uri' in the clinic input of " \ + "'_sqlite3.Connection.__init__' to be keyword-only." + #elif PY_VERSION_HEX >= 0x030f00A0 + # ifdef _MSC_VER + # pragma message ( \ + "In connection.c, update parameter(s) 'timeout', 'detect_types', " \ + "'isolation_level', 'check_same_thread', 'factory', " \ + "'cached_statements' and 'uri' in the clinic input of " \ + "'_sqlite3.Connection.__init__' to be keyword-only.") + # else + # warning \ + "In connection.c, update parameter(s) 'timeout', 'detect_types', " \ + "'isolation_level', 'check_same_thread', 'factory', " \ + "'cached_statements' and 'uri' in the clinic input of " \ + "'_sqlite3.Connection.__init__' to be keyword-only." + # endif + #endif + if (nargs > 1 && nargs <= 8) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to _sqlite3.Connection()" + " is deprecated. Parameters 'timeout', 'detect_types', " + "'isolation_level', 'check_same_thread', 'factory', " + "'cached_statements' and 'uri' will become keyword-only " + "parameters in Python 3.15.", 1)) + { + goto exit; + } + } fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 8, 0, argsbuf); if (!fastargs) { goto exit; @@ -1659,4 +1692,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=d3c6cb9326736ea5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5a05e5294ad9d2ce input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 3d2fcd3ae0788..0819acd094096 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -216,6 +216,7 @@ class sqlite3_int64_converter(CConverter): _sqlite3.Connection.__init__ as pysqlite_connection_init database: object + * [from 3.15] timeout: double = 5.0 detect_types: int = 0 isolation_level: IsolationLevel = "" @@ -234,7 +235,7 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, int check_same_thread, PyObject *factory, int cache_size, int uri, enum autocommit_mode autocommit) -/*[clinic end generated code: output=cba057313ea7712f input=9b0ab6c12f674fa3]*/ +/*[clinic end generated code: output=cba057313ea7712f input=219c3dbecbae7d99]*/ { if (PySys_Audit("sqlite3.connect", "O", database) < 0) { return -1; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 0c503dfbebbd6..dd45ffc1988f7 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -64,6 +64,17 @@ pysqlite_connect(PyObject *module, PyObject *const *args, Py_ssize_t nargsf, static const int FACTORY_POS = 5; Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + if (nargs > 1 && nargs <= 8) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to sqlite3.connect()" + " is deprecated. Parameters 'timeout', 'detect_types', " + "'isolation_level', 'check_same_thread', 'factory', " + "'cached_statements' and 'uri' will become keyword-only " + "parameters in Python 3.15.", 1)) + { + return NULL; + } + } if (nargs > FACTORY_POS) { factory = args[FACTORY_POS]; } From webhook-mailer at python.org Tue Aug 15 07:26:46 2023 From: webhook-mailer at python.org (lysnikolaou) Date: Tue, 15 Aug 2023 11:26:46 -0000 Subject: [Python-checkins] gh-107967: Fix infinite recursion on invalid escape sequence warning (#107968) Message-ID: https://github.com/python/cpython/commit/d66bc9e8a7a8d6774d912a4b9d151885c4d8de1d commit: d66bc9e8a7a8d6774d912a4b9d151885c4d8de1d branch: main author: Lysandros Nikolaou committer: lysnikolaou date: 2023-08-15T11:26:42Z summary: gh-107967: Fix infinite recursion on invalid escape sequence warning (#107968) files: M Lib/test/test_fstring.py M Parser/tokenizer.c diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index cb14bba2602de..16f01973f99f3 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1673,5 +1673,15 @@ def test_debug_in_file(self): self.assertEqual(stdout.decode('utf-8').strip().replace('\r\n', '\n').replace('\r', '\n'), "3\n=3") + def test_syntax_warning_infinite_recursion_in_file(self): + with temp_cwd(): + script = 'script.py' + with open(script, 'w') as f: + f.write(r"print(f'\{1}')") + + _, stdout, stderr = assert_python_ok(script) + self.assertIn(rb'\1', stdout) + self.assertEqual(len(stderr.strip().splitlines()), 2) + if __name__ == '__main__': unittest.main() diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 5a42f6f357317..b10c9f1f8ea2c 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1539,6 +1539,9 @@ parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) static int warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char) { + if (!tok->report_warnings) { + return 0; + } PyObject *msg = PyUnicode_FromFormat( "invalid escape sequence '\\%c'", From webhook-mailer at python.org Tue Aug 15 08:41:59 2023 From: webhook-mailer at python.org (pablogsal) Date: Tue, 15 Aug 2023 12:41:59 -0000 Subject: [Python-checkins] [3.12] gh-107967: Fix infinite recursion on invalid escape sequence warning (GH-107968) (#107970) Message-ID: https://github.com/python/cpython/commit/d189480942348aab345500ac9204965f67f30210 commit: d189480942348aab345500ac9204965f67f30210 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: pablogsal date: 2023-08-15T13:41:55+01:00 summary: [3.12] gh-107967: Fix infinite recursion on invalid escape sequence warning (GH-107968) (#107970) gh-107967: Fix infinite recursion on invalid escape sequence warning (GH-107968) (cherry picked from commit d66bc9e8a7a8d6774d912a4b9d151885c4d8de1d) Co-authored-by: Lysandros Nikolaou files: M Lib/test/test_fstring.py M Parser/tokenizer.c diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index cb14bba2602de..16f01973f99f3 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1673,5 +1673,15 @@ def test_debug_in_file(self): self.assertEqual(stdout.decode('utf-8').strip().replace('\r\n', '\n').replace('\r', '\n'), "3\n=3") + def test_syntax_warning_infinite_recursion_in_file(self): + with temp_cwd(): + script = 'script.py' + with open(script, 'w') as f: + f.write(r"print(f'\{1}')") + + _, stdout, stderr = assert_python_ok(script) + self.assertIn(rb'\1', stdout) + self.assertEqual(len(stderr.strip().splitlines()), 2) + if __name__ == '__main__': unittest.main() diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 7e246d2f56481..c4c345e4c358e 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1544,6 +1544,9 @@ parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) static int warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char) { + if (!tok->report_warnings) { + return 0; + } PyObject *msg = PyUnicode_FromFormat( "invalid escape sequence '\\%c'", From webhook-mailer at python.org Tue Aug 15 09:58:16 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 15 Aug 2023 13:58:16 -0000 Subject: [Python-checkins] gh-107963: Fix set_forkserver_preload to check the type of given list (#107965) Message-ID: https://github.com/python/cpython/commit/6515ec3d3d5acd3d0b99c88794bdec09f0831e5b commit: 6515ec3d3d5acd3d0b99c88794bdec09f0831e5b branch: main author: Dong-hee Na committer: Yhg1s date: 2023-08-15T15:58:12+02:00 summary: gh-107963: Fix set_forkserver_preload to check the type of given list (#107965) gh-107963: Fix set_forkserver_preload to check the type of given list files: A Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst M Lib/multiprocessing/forkserver.py M Lib/test/_test_multiprocessing.py diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 22a911a7a29cd..4642707dae2f4 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -61,7 +61,7 @@ def _stop_unlocked(self): def set_forkserver_preload(self, modules_names): '''Set list of module names to try to load in forkserver process.''' - if not all(type(mod) is str for mod in self._preload_modules): + if not all(type(mod) is str for mod in modules_names): raise TypeError('module_names must be a list of strings') self._preload_modules = modules_names diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index f881a5d467469..10754964e73bc 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5369,6 +5369,14 @@ def test_context(self): self.assertRaises(ValueError, ctx.set_start_method, None) self.check_context(ctx) + def test_context_check_module_types(self): + try: + ctx = multiprocessing.get_context('forkserver') + except ValueError: + raise unittest.SkipTest('forkserver should be available') + with self.assertRaisesRegex(TypeError, 'module_names must be a list of strings'): + ctx.set_forkserver_preload([1, 2, 3]) + def test_set_get(self): multiprocessing.set_forkserver_preload(PRELOAD) count = 0 diff --git a/Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst b/Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst new file mode 100644 index 0000000000000..3a73b2da0c433 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst @@ -0,0 +1,2 @@ +Fix :func:`multiprocessing.set_forkserver_preload` to check the given list +of modules names. Patch by Dong-hee Na. From webhook-mailer at python.org Tue Aug 15 10:00:57 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 15 Aug 2023 14:00:57 -0000 Subject: [Python-checkins] gh-107880: Argument Clinic: Fix regression in gh-107885 (#107974) Message-ID: https://github.com/python/cpython/commit/e90036c9bdf25621c15207a9cabd119fb5366c27 commit: e90036c9bdf25621c15207a9cabd119fb5366c27 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-15T14:00:52Z summary: gh-107880: Argument Clinic: Fix regression in gh-107885 (#107974) Co-authored-by: Alex Waygood files: M Lib/test/test_clinic.py M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index f067a26d1fb3a..a67cd301f6eaa 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -609,6 +609,35 @@ def test_directive_output_invalid_command(self): """ self.expect_failure(block, err, lineno=2) + def test_validate_cloned_init(self): + block = """ + /*[clinic input] + class C "void *" "" + C.meth + a: int + [clinic start generated code]*/ + /*[clinic input] + @classmethod + C.__init__ = C.meth + [clinic start generated code]*/ + """ + err = "'__init__' must be a normal method, not a class or static method" + self.expect_failure(block, err, lineno=8) + + def test_validate_cloned_new(self): + block = """ + /*[clinic input] + class C "void *" "" + C.meth + a: int + [clinic start generated code]*/ + /*[clinic input] + C.__new__ = C.meth + [clinic start generated code]*/ + """ + err = "'__new__' must be a class method" + self.expect_failure(block, err, lineno=7) + class ParseFileUnitTest(TestCase): def expect_parsing_failure( @@ -1918,7 +1947,7 @@ class Foo "" "" self.parse_function(block) def test_new_must_be_a_class_method(self): - err = "__new__ must be a class method!" + err = "'__new__' must be a class method!" block = """ module foo class Foo "" "" @@ -1927,7 +1956,7 @@ class Foo "" "" self.expect_failure(block, err, lineno=2) def test_init_must_be_a_normal_method(self): - err = "__init__ must be a normal method, not a class or static method!" + err = "'__init__' must be a normal method, not a class or static method!" block = """ module foo class Foo "" "" @@ -2030,7 +2059,7 @@ def test_illegal_c_identifier(self): self.expect_failure(block, err, lineno=2) def test_cannot_convert_special_method(self): - err = "__len__ is a special method and cannot be converted" + err = "'__len__' is a special method and cannot be converted" block = """ class T "" "" T.__len__ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 1e0303c77087e..11dbfb3fbe858 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4840,6 +4840,21 @@ def state_dsl_start(self, line: str) -> None: self.next(self.state_modulename_name, line) + def update_function_kind(self, fullname: str) -> None: + fields = fullname.split('.') + name = fields.pop() + _, cls = self.clinic._module_and_class(fields) + if name in unsupported_special_methods: + fail(f"{name!r} is a special method and cannot be converted to Argument Clinic!") + if name == '__new__': + if (self.kind is not CLASS_METHOD) or (not cls): + fail("'__new__' must be a class method!") + self.kind = METHOD_NEW + elif name == '__init__': + if (self.kind is not CALLABLE) or (not cls): + fail("'__init__' must be a normal method, not a class or static method!") + self.kind = METHOD_INIT + def state_modulename_name(self, line: str) -> None: # looking for declaration, which establishes the leftmost column # line should be @@ -4888,6 +4903,7 @@ def state_modulename_name(self, line: str) -> None: function_name = fields.pop() module, cls = self.clinic._module_and_class(fields) + self.update_function_kind(full_name) overrides: dict[str, Any] = { "name": function_name, "full_name": full_name, @@ -4948,20 +4964,9 @@ def state_modulename_name(self, line: str) -> None: function_name = fields.pop() module, cls = self.clinic._module_and_class(fields) - fields = full_name.split('.') - if fields[-1] in unsupported_special_methods: - fail(f"{fields[-1]} is a special method and cannot be converted to Argument Clinic! (Yet.)") - - if fields[-1] == '__new__': - if (self.kind is not CLASS_METHOD) or (not cls): - fail("__new__ must be a class method!") - self.kind = METHOD_NEW - elif fields[-1] == '__init__': - if (self.kind is not CALLABLE) or (not cls): - fail("__init__ must be a normal method, not a class or static method!") - self.kind = METHOD_INIT - if not return_converter: - return_converter = init_return_converter() + self.update_function_kind(full_name) + if self.kind is METHOD_INIT and not return_converter: + return_converter = init_return_converter() if not return_converter: return_converter = CReturnConverter() From webhook-mailer at python.org Tue Aug 15 10:41:43 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 15 Aug 2023 14:41:43 -0000 Subject: [Python-checkins] gh-107972: Argument Clinic: Ensure a C basename is provided after 'as' (#107973) Message-ID: https://github.com/python/cpython/commit/607f18c89456cdc9064e27f86a7505e011209757 commit: 607f18c89456cdc9064e27f86a7505e011209757 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-15T14:41:40Z summary: gh-107972: Argument Clinic: Ensure a C basename is provided after 'as' (#107973) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Lib/test/test_clinic.py M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index a67cd301f6eaa..896730571d815 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -28,7 +28,8 @@ def _make_clinic(*, filename='clinic_tests'): return c -def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None): +def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None, + strip=True): """Helper for the parser tests. tc: unittest.TestCase; passed self in the wrapper @@ -38,7 +39,9 @@ def _expect_failure(tc, parser, code, errmsg, *, filename=None, lineno=None): filename: str, optional filename lineno: int, optional line number """ - code = dedent(code).strip() + code = dedent(code) + if strip: + code = code.strip() errmsg = re.escape(errmsg) with tc.assertRaisesRegex(clinic.ClinicError, errmsg) as cm: parser(code) @@ -638,6 +641,19 @@ class C "void *" "" err = "'__new__' must be a class method" self.expect_failure(block, err, lineno=7) + def test_no_c_basename_cloned(self): + block = """ + /*[clinic input] + foo2 + [clinic start generated code]*/ + /*[clinic input] + foo as = foo2 + [clinic start generated code]*/ + """ + err = "No C basename provided after 'as' keyword" + self.expect_failure(block, err, lineno=5) + + class ParseFileUnitTest(TestCase): def expect_parsing_failure( @@ -857,9 +873,10 @@ def parse_function(self, text, signatures_in_block=2, function_index=1): assert isinstance(s[function_index], clinic.Function) return s[function_index] - def expect_failure(self, block, err, *, filename=None, lineno=None): + def expect_failure(self, block, err, *, + filename=None, lineno=None, strip=True): return _expect_failure(self, self.parse_function, block, err, - filename=filename, lineno=lineno) + filename=filename, lineno=lineno, strip=strip) def checkDocstring(self, fn, expected): self.assertTrue(hasattr(fn, "docstring")) @@ -1520,6 +1537,11 @@ def test_illegal_c_basename(self): err = "Illegal C basename: '935'" self.expect_failure(block, err) + def test_no_c_basename(self): + block = "foo as " + err = "No C basename provided after 'as' keyword" + self.expect_failure(block, err, strip=False) + def test_single_star(self): block = """ module foo diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 11dbfb3fbe858..6ff2622d33b38 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4879,9 +4879,11 @@ def state_modulename_name(self, line: str) -> None: before, equals, existing = line.rpartition('=') c_basename: str | None if equals: - full_name, _, c_basename = before.partition(' as ') + full_name, as_, c_basename = before.partition(' as ') full_name = full_name.strip() c_basename = c_basename.strip() + if as_ and not c_basename: + fail("No C basename provided after 'as' keyword") existing = existing.strip() if (is_legal_py_identifier(full_name) and (not c_basename or is_legal_c_identifier(c_basename)) and @@ -4932,10 +4934,11 @@ def state_modulename_name(self, line: str) -> None: line, _, returns = line.partition('->') returns = returns.strip() - full_name, _, c_basename = line.partition(' as ') + full_name, as_, c_basename = line.partition(' as ') full_name = full_name.strip() c_basename = c_basename.strip() or None - + if as_ and not c_basename: + fail("No C basename provided after 'as' keyword") if not is_legal_py_identifier(full_name): fail(f"Illegal function name: {full_name!r}") if c_basename and not is_legal_c_identifier(c_basename): From webhook-mailer at python.org Tue Aug 15 10:53:29 2023 From: webhook-mailer at python.org (corona10) Date: Tue, 15 Aug 2023 14:53:29 -0000 Subject: [Python-checkins] [3.11] gh-107963: Fix set_forkserver_preload to check the type of given list (GH-107965) (gh-107976) Message-ID: https://github.com/python/cpython/commit/db4400b5b28dc17c9eac47780402c0b812703358 commit: db4400b5b28dc17c9eac47780402c0b812703358 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: corona10 date: 2023-08-15T23:53:25+09:00 summary: [3.11] gh-107963: Fix set_forkserver_preload to check the type of given list (GH-107965) (gh-107976) gh-107963: Fix set_forkserver_preload to check the type of given list (GH-107965) (cherry picked from commit 6515ec3d3d5acd3d0b99c88794bdec09f0831e5b) gh-107963: Fix set_forkserver_preload to check the type of given list Co-authored-by: Dong-hee Na files: A Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst M Lib/multiprocessing/forkserver.py M Lib/test/_test_multiprocessing.py diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 22a911a7a29cd..4642707dae2f4 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -61,7 +61,7 @@ def _stop_unlocked(self): def set_forkserver_preload(self, modules_names): '''Set list of module names to try to load in forkserver process.''' - if not all(type(mod) is str for mod in self._preload_modules): + if not all(type(mod) is str for mod in modules_names): raise TypeError('module_names must be a list of strings') self._preload_modules = modules_names diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index aa302c21000dd..29dc386376649 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5293,6 +5293,14 @@ def test_context(self): self.assertRaises(ValueError, ctx.set_start_method, None) self.check_context(ctx) + def test_context_check_module_types(self): + try: + ctx = multiprocessing.get_context('forkserver') + except ValueError: + raise unittest.SkipTest('forkserver should be available') + with self.assertRaisesRegex(TypeError, 'module_names must be a list of strings'): + ctx.set_forkserver_preload([1, 2, 3]) + def test_set_get(self): multiprocessing.set_forkserver_preload(PRELOAD) count = 0 diff --git a/Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst b/Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst new file mode 100644 index 0000000000000..3a73b2da0c433 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst @@ -0,0 +1,2 @@ +Fix :func:`multiprocessing.set_forkserver_preload` to check the given list +of modules names. Patch by Dong-hee Na. From webhook-mailer at python.org Tue Aug 15 10:56:58 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 15 Aug 2023 14:56:58 -0000 Subject: [Python-checkins] [3.12] gh-107963: Fix set_forkserver_preload to check the type of given list (GH-107965) (#107975) Message-ID: https://github.com/python/cpython/commit/f0a583b6fbc9a4b2f9358c8426110a5c1f7948a3 commit: f0a583b6fbc9a4b2f9358c8426110a5c1f7948a3 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-15T16:56:53+02:00 summary: [3.12] gh-107963: Fix set_forkserver_preload to check the type of given list (GH-107965) (#107975) gh-107963: Fix set_forkserver_preload to check the type of given list (GH-107965) (cherry picked from commit 6515ec3d3d5acd3d0b99c88794bdec09f0831e5b) gh-107963: Fix set_forkserver_preload to check the type of given list Co-authored-by: Dong-hee Na files: A Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst M Lib/multiprocessing/forkserver.py M Lib/test/_test_multiprocessing.py diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 22a911a7a29cd..4642707dae2f4 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -61,7 +61,7 @@ def _stop_unlocked(self): def set_forkserver_preload(self, modules_names): '''Set list of module names to try to load in forkserver process.''' - if not all(type(mod) is str for mod in self._preload_modules): + if not all(type(mod) is str for mod in modules_names): raise TypeError('module_names must be a list of strings') self._preload_modules = modules_names diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index c13b049645ac9..4afbe172a1eef 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5331,6 +5331,14 @@ def test_context(self): self.assertRaises(ValueError, ctx.set_start_method, None) self.check_context(ctx) + def test_context_check_module_types(self): + try: + ctx = multiprocessing.get_context('forkserver') + except ValueError: + raise unittest.SkipTest('forkserver should be available') + with self.assertRaisesRegex(TypeError, 'module_names must be a list of strings'): + ctx.set_forkserver_preload([1, 2, 3]) + def test_set_get(self): multiprocessing.set_forkserver_preload(PRELOAD) count = 0 diff --git a/Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst b/Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst new file mode 100644 index 0000000000000..3a73b2da0c433 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-15-18-20-00.gh-issue-107963.20g5BG.rst @@ -0,0 +1,2 @@ +Fix :func:`multiprocessing.set_forkserver_preload` to check the given list +of modules names. Patch by Dong-hee Na. From webhook-mailer at python.org Tue Aug 15 11:33:05 2023 From: webhook-mailer at python.org (zooba) Date: Tue, 15 Aug 2023 15:33:05 -0000 Subject: [Python-checkins] gh-106242: Fix path truncation in os.path.normpath (GH-106816) Message-ID: https://github.com/python/cpython/commit/09322724319d4c23195300b222a1c0ea720af56b commit: 09322724319d4c23195300b222a1c0ea720af56b branch: main author: Finn Womack committer: zooba date: 2023-08-15T16:33:00+01:00 summary: gh-106242: Fix path truncation in os.path.normpath (GH-106816) files: A Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst M Include/internal/pycore_fileutils.h M Lib/test/test_genericpath.py M Modules/posixmodule.c M Python/fileutils.c diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index daa32c0dff609..cb5ac76772de5 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -260,6 +260,7 @@ extern int _Py_add_relfile(wchar_t *dirname, size_t bufsize); extern size_t _Py_find_basename(const wchar_t *filename); PyAPI_FUNC(wchar_t*) _Py_normpath(wchar_t *path, Py_ssize_t size); +extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *length); // The Windows Games API family does not provide these functions // so provide our own implementations. Remove them in case they get added diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 489044f8090d3..4f311c2d498e9 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -460,6 +460,10 @@ def test_normpath_issue5827(self): for path in ('', '.', '/', '\\', '///foo/.//bar//'): self.assertIsInstance(self.pathmodule.normpath(path), str) + def test_normpath_issue106242(self): + for path in ('\x00', 'foo\x00bar', '\x00\x00', '\x00foo', 'foo\x00'): + self.assertEqual(self.pathmodule.normpath(path), path) + def test_abspath_issue3426(self): # Check that abspath returns unicode when the arg is unicode # with both ASCII and non-ASCII cwds. diff --git a/Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst b/Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst new file mode 100644 index 0000000000000..44237a9f15708 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst @@ -0,0 +1 @@ +Fixes :func:`os.path.normpath` to handle embedded null characters without truncating the path. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a42e41c081e5b..c391aab6fdce1 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5274,7 +5274,9 @@ os__path_normpath_impl(PyObject *module, PyObject *path) if (!buffer) { return NULL; } - PyObject *result = PyUnicode_FromWideChar(_Py_normpath(buffer, len), -1); + Py_ssize_t norm_len; + wchar_t *norm_path = _Py_normpath_and_size(buffer, len, &norm_len); + PyObject *result = PyUnicode_FromWideChar(norm_path, norm_len); PyMem_Free(buffer); return result; } diff --git a/Python/fileutils.c b/Python/fileutils.c index 19b23f6bd18b3..0ffd9885fdc84 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2377,12 +2377,14 @@ _Py_find_basename(const wchar_t *filename) path, which will be within the original buffer. Guaranteed to not make the path longer, and will not fail. 'size' is the length of the path, if known. If -1, the first null character will be assumed - to be the end of the path. */ + to be the end of the path. 'normsize' will be set to contain the + length of the resulting normalized path. */ wchar_t * -_Py_normpath(wchar_t *path, Py_ssize_t size) +_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) { assert(path != NULL); - if (!path[0] || size == 0) { + if (!path[0] && size < 0 || size == 0) { + *normsize = 0; return path; } wchar_t *pEnd = size >= 0 ? &path[size] : NULL; @@ -2431,11 +2433,7 @@ _Py_normpath(wchar_t *path, Py_ssize_t size) *p2++ = lastC = *p1; } } - if (sepCount) { - minP2 = p2; // Invalid path - } else { - minP2 = p2 - 1; // Absolute path has SEP at minP2 - } + minP2 = p2 - 1; } #else // Skip past two leading SEPs @@ -2495,13 +2493,28 @@ _Py_normpath(wchar_t *path, Py_ssize_t size) while (--p2 != minP2 && *p2 == SEP) { *p2 = L'\0'; } + } else { + --p2; } + *normsize = p2 - path + 1; #undef SEP_OR_END #undef IS_SEP #undef IS_END return path; } +/* In-place path normalisation. Returns the start of the normalized + path, which will be within the original buffer. Guaranteed to not + make the path longer, and will not fail. 'size' is the length of + the path, if known. If -1, the first null character will be assumed + to be the end of the path. */ +wchar_t * +_Py_normpath(wchar_t *path, Py_ssize_t size) +{ + Py_ssize_t norm_length; + return _Py_normpath_and_size(path, size, &norm_length); +} + /* Get the current directory. buflen is the buffer size in wide characters including the null character. Decode the path from the locale encoding. From webhook-mailer at python.org Tue Aug 15 11:40:09 2023 From: webhook-mailer at python.org (iritkatriel) Date: Tue, 15 Aug 2023 15:40:09 -0000 Subject: [Python-checkins] gh-103082: remove assumption that INSTRUMENTED_LINE is the last instrumented opcode (#107978) Message-ID: https://github.com/python/cpython/commit/971a4c27517e76155a4a489096eba69c53a7b9e9 commit: 971a4c27517e76155a4a489096eba69c53a7b9e9 branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-15T16:40:05+01:00 summary: gh-103082: remove assumption that INSTRUMENTED_LINE is the last instrumented opcode (#107978) files: M Python/instrumentation.c diff --git a/Python/instrumentation.c b/Python/instrumentation.c index b50e8e26476cd..48befed4ea838 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -121,7 +121,7 @@ static inline bool opcode_has_event(int opcode) { return ( - opcode < INSTRUMENTED_LINE && + opcode != INSTRUMENTED_LINE && INSTRUMENTED_OPCODES[opcode] > 0 ); } @@ -1202,7 +1202,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, Py_DECREF(line_obj); done: assert(original_opcode != 0); - assert(original_opcode < INSTRUMENTED_LINE); + assert(original_opcode != INSTRUMENTED_LINE); assert(_PyOpcode_Deopt[original_opcode] == original_opcode); return original_opcode; } From webhook-mailer at python.org Tue Aug 15 13:05:41 2023 From: webhook-mailer at python.org (zooba) Date: Tue, 15 Aug 2023 17:05:41 -0000 Subject: [Python-checkins] gh-106242: Minor fixup to avoid compiler warnings (GH-107983) Message-ID: https://github.com/python/cpython/commit/34e1917912f05e3ab5c9b1e39f678bd36388499e commit: 34e1917912f05e3ab5c9b1e39f678bd36388499e branch: main author: Steve Dower committer: zooba date: 2023-08-15T17:02:32Z summary: gh-106242: Minor fixup to avoid compiler warnings (GH-107983) files: M Python/fileutils.c diff --git a/Python/fileutils.c b/Python/fileutils.c index 0ffd9885fdc84..9c4998397c4ac 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2383,7 +2383,7 @@ wchar_t * _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) { assert(path != NULL); - if (!path[0] && size < 0 || size == 0) { + if ((size < 0 && !path[0]) || size == 0) { *normsize = 0; return path; } From webhook-mailer at python.org Tue Aug 15 13:17:21 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 15 Aug 2023 17:17:21 -0000 Subject: [Python-checkins] [3.11] gh-106242: Fix path truncation in os.path.normpath (GH-106816) (#107982) Message-ID: https://github.com/python/cpython/commit/ccf81e1088c25a9f4464e478dc3b5c03ed7ee63b commit: ccf81e1088c25a9f4464e478dc3b5c03ed7ee63b branch: 3.11 author: Steve Dower committer: ambv date: 2023-08-15T19:07:52+02:00 summary: [3.11] gh-106242: Fix path truncation in os.path.normpath (GH-106816) (#107982) Co-authored-by: Finn Womack files: A Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst M Include/internal/pycore_fileutils.h M Lib/test/test_genericpath.py M Modules/posixmodule.c M Python/fileutils.c diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 3ce8108e4e04f..332cc30365b9e 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -244,7 +244,8 @@ extern int _Py_add_relfile(wchar_t *dirname, const wchar_t *relfile, size_t bufsize); extern size_t _Py_find_basename(const wchar_t *filename); -PyAPI_FUNC(wchar_t *) _Py_normpath(wchar_t *path, Py_ssize_t size); +PyAPI_FUNC(wchar_t*) _Py_normpath(wchar_t *path, Py_ssize_t size); +extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *length); // Macros to protect CRT calls against instant termination when passed an diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 489044f8090d3..4f311c2d498e9 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -460,6 +460,10 @@ def test_normpath_issue5827(self): for path in ('', '.', '/', '\\', '///foo/.//bar//'): self.assertIsInstance(self.pathmodule.normpath(path), str) + def test_normpath_issue106242(self): + for path in ('\x00', 'foo\x00bar', '\x00\x00', '\x00foo', 'foo\x00'): + self.assertEqual(self.pathmodule.normpath(path), path) + def test_abspath_issue3426(self): # Check that abspath returns unicode when the arg is unicode # with both ASCII and non-ASCII cwds. diff --git a/Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst b/Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst new file mode 100644 index 0000000000000..44237a9f15708 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst @@ -0,0 +1 @@ +Fixes :func:`os.path.normpath` to handle embedded null characters without truncating the path. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 9d4228896d230..5ceea7411818a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4552,7 +4552,9 @@ os__path_normpath_impl(PyObject *module, PyObject *path) if (!buffer) { return NULL; } - PyObject *result = PyUnicode_FromWideChar(_Py_normpath(buffer, len), -1); + Py_ssize_t norm_len; + wchar_t *norm_path = _Py_normpath_and_size(buffer, len, &norm_len); + PyObject *result = PyUnicode_FromWideChar(norm_path, norm_len); PyMem_Free(buffer); return result; } diff --git a/Python/fileutils.c b/Python/fileutils.c index c86ed40b19937..b310a6c077d91 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2179,12 +2179,14 @@ _Py_find_basename(const wchar_t *filename) path, which will be within the original buffer. Guaranteed to not make the path longer, and will not fail. 'size' is the length of the path, if known. If -1, the first null character will be assumed - to be the end of the path. */ + to be the end of the path. 'normsize' will be set to contain the + length of the resulting normalized path. */ wchar_t * -_Py_normpath(wchar_t *path, Py_ssize_t size) +_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) { assert(path != NULL); - if (!path[0] || size == 0) { + if ((size < 0 && !path[0]) || size == 0) { + *normsize = 0; return path; } wchar_t *pEnd = size >= 0 ? &path[size] : NULL; @@ -2233,11 +2235,7 @@ _Py_normpath(wchar_t *path, Py_ssize_t size) *p2++ = lastC = *p1; } } - if (sepCount) { - minP2 = p2; // Invalid path - } else { - minP2 = p2 - 1; // Absolute path has SEP at minP2 - } + minP2 = p2 - 1; } #else // Skip past two leading SEPs @@ -2297,13 +2295,28 @@ _Py_normpath(wchar_t *path, Py_ssize_t size) while (--p2 != minP2 && *p2 == SEP) { *p2 = L'\0'; } + } else { + --p2; } + *normsize = p2 - path + 1; #undef SEP_OR_END #undef IS_SEP #undef IS_END return path; } +/* In-place path normalisation. Returns the start of the normalized + path, which will be within the original buffer. Guaranteed to not + make the path longer, and will not fail. 'size' is the length of + the path, if known. If -1, the first null character will be assumed + to be the end of the path. */ +wchar_t * +_Py_normpath(wchar_t *path, Py_ssize_t size) +{ + Py_ssize_t norm_length; + return _Py_normpath_and_size(path, size, &norm_length); +} + /* Get the current directory. buflen is the buffer size in wide characters including the null character. Decode the path from the locale encoding. From webhook-mailer at python.org Tue Aug 15 14:55:35 2023 From: webhook-mailer at python.org (Fidget-Spinner) Date: Tue, 15 Aug 2023 18:55:35 -0000 Subject: [Python-checkins] gh-107557: Setup abstract interpretation (#107847) Message-ID: https://github.com/python/cpython/commit/e28b0dc86dd1d058788b9e1eaa825cdfc2d97067 commit: e28b0dc86dd1d058788b9e1eaa825cdfc2d97067 branch: main author: Ken Jin committer: Fidget-Spinner date: 2023-08-15T18:04:17Z summary: gh-107557: Setup abstract interpretation (#107847) Co-authored-by: Guido van Rossum Co-authored-by: Jules <57632293+juliapoo at users.noreply.github.com> files: A Include/internal/pycore_optimizer.h A Misc/NEWS.d/next/Core and Builtins/2023-08-02-09-55-21.gh-issue-107557.P1z-in.rst A Python/abstract_interp_cases.c.h A Python/optimizer_analysis.c M .gitattributes M Include/cpython/optimizer.h M Include/internal/pycore_opcode_metadata.h M Include/internal/pycore_uops.h M Makefile.pre.in M PCbuild/_freeze_module.vcxproj M PCbuild/_freeze_module.vcxproj.filters M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Python/bytecodes.c M Python/executor_cases.c.h M Python/optimizer.c M Tools/c-analyzer/cpython/_parser.py M Tools/c-analyzer/cpython/ignored.tsv M Tools/cases_generator/generate_cases.py M Tools/cases_generator/instructions.py M Tools/cases_generator/stacking.py diff --git a/.gitattributes b/.gitattributes index 4a072bc990f0f..317a613217a9b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -90,6 +90,7 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated +Python/abstract_interp_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index da34ec1882a53..e3fe0e8f0ffd5 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -22,7 +22,7 @@ typedef struct _PyExecutorObject { typedef struct _PyOptimizerObject _PyOptimizerObject; /* Should return > 0 if a new executor is created. O if no executor is produced and < 0 if an error occurred. */ -typedef int (*optimize_func)(_PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject **); +typedef int (*optimize_func)(_PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, _PyExecutorObject **, int curr_stackentries); typedef struct _PyOptimizerObject { PyObject_HEAD diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index c0a2e9d91c4f4..df9b4184e2a50 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -55,6 +55,7 @@ #define _POP_JUMP_IF_FALSE 331 #define _POP_JUMP_IF_TRUE 332 #define JUMP_TO_TOP 333 +#define INSERT 334 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -118,18 +119,38 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case UNARY_INVERT: return 1; + case _GUARD_BOTH_INT: + return 2; + case _BINARY_OP_MULTIPLY_INT: + return 2; + case _BINARY_OP_ADD_INT: + return 2; + case _BINARY_OP_SUBTRACT_INT: + return 2; case BINARY_OP_MULTIPLY_INT: return 2; case BINARY_OP_ADD_INT: return 2; case BINARY_OP_SUBTRACT_INT: return 2; + case _GUARD_BOTH_FLOAT: + return 2; + case _BINARY_OP_MULTIPLY_FLOAT: + return 2; + case _BINARY_OP_ADD_FLOAT: + return 2; + case _BINARY_OP_SUBTRACT_FLOAT: + return 2; case BINARY_OP_MULTIPLY_FLOAT: return 2; case BINARY_OP_ADD_FLOAT: return 2; case BINARY_OP_SUBTRACT_FLOAT: return 2; + case _GUARD_BOTH_UNICODE: + return 2; + case _BINARY_OP_ADD_UNICODE: + return 2; case BINARY_OP_ADD_UNICODE: return 2; case BINARY_OP_INPLACE_ADD_UNICODE: @@ -226,14 +247,26 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case DELETE_GLOBAL: return 0; + case _LOAD_LOCALS: + return 0; case LOAD_LOCALS: return 0; + case _LOAD_FROM_DICT_OR_GLOBALS: + return 1; case LOAD_NAME: return 0; case LOAD_FROM_DICT_OR_GLOBALS: return 1; case LOAD_GLOBAL: return 0; + case _GUARD_GLOBALS_VERSION: + return 0; + case _GUARD_BUILTINS_VERSION: + return 0; + case _LOAD_GLOBAL_MODULE: + return 0; + case _LOAD_GLOBAL_BUILTINS: + return 0; case LOAD_GLOBAL_MODULE: return 0; case LOAD_GLOBAL_BUILTIN: @@ -294,6 +327,12 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case LOAD_METHOD: return 1; + case _GUARD_TYPE_VERSION: + return 1; + case _CHECK_MANAGED_OBJECT_HAS_VALUES: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE: + return 1; case LOAD_ATTR_INSTANCE_VALUE: return 1; case LOAD_ATTR_MODULE: @@ -348,6 +387,8 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case POP_JUMP_IF_TRUE: return 1; + case IS_NONE: + return 1; case POP_JUMP_IF_NONE: return 1; case POP_JUMP_IF_NOT_NONE: @@ -372,10 +413,28 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case INSTRUMENTED_FOR_ITER: return 0; + case _ITER_CHECK_LIST: + return 1; + case _IS_ITER_EXHAUSTED_LIST: + return 1; + case _ITER_NEXT_LIST: + return 1; case FOR_ITER_LIST: return 1; + case _ITER_CHECK_TUPLE: + return 1; + case _IS_ITER_EXHAUSTED_TUPLE: + return 1; + case _ITER_NEXT_TUPLE: + return 1; case FOR_ITER_TUPLE: return 1; + case _ITER_CHECK_RANGE: + return 1; + case _IS_ITER_EXHAUSTED_RANGE: + return 1; + case _ITER_NEXT_RANGE: + return 1; case FOR_ITER_RANGE: return 1; case FOR_ITER_GEN: @@ -494,6 +553,18 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case RESERVED: return 0; + case _POP_JUMP_IF_FALSE: + return 1; + case _POP_JUMP_IF_TRUE: + return 1; + case JUMP_TO_TOP: + return 0; + case SAVE_IP: + return 0; + case EXIT_TRACE: + return 0; + case INSERT: + return oparg + 1; default: return -1; } @@ -562,18 +633,38 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case UNARY_INVERT: return 1; + case _GUARD_BOTH_INT: + return 2; + case _BINARY_OP_MULTIPLY_INT: + return 1; + case _BINARY_OP_ADD_INT: + return 1; + case _BINARY_OP_SUBTRACT_INT: + return 1; case BINARY_OP_MULTIPLY_INT: return 1; case BINARY_OP_ADD_INT: return 1; case BINARY_OP_SUBTRACT_INT: return 1; + case _GUARD_BOTH_FLOAT: + return 2; + case _BINARY_OP_MULTIPLY_FLOAT: + return 1; + case _BINARY_OP_ADD_FLOAT: + return 1; + case _BINARY_OP_SUBTRACT_FLOAT: + return 1; case BINARY_OP_MULTIPLY_FLOAT: return 1; case BINARY_OP_ADD_FLOAT: return 1; case BINARY_OP_SUBTRACT_FLOAT: return 1; + case _GUARD_BOTH_UNICODE: + return 2; + case _BINARY_OP_ADD_UNICODE: + return 1; case BINARY_OP_ADD_UNICODE: return 1; case BINARY_OP_INPLACE_ADD_UNICODE: @@ -670,14 +761,26 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case DELETE_GLOBAL: return 0; + case _LOAD_LOCALS: + return 1; case LOAD_LOCALS: return 1; + case _LOAD_FROM_DICT_OR_GLOBALS: + return 1; case LOAD_NAME: return 1; case LOAD_FROM_DICT_OR_GLOBALS: return 1; case LOAD_GLOBAL: return ((oparg & 1) ? 1 : 0) + 1; + case _GUARD_GLOBALS_VERSION: + return 0; + case _GUARD_BUILTINS_VERSION: + return 0; + case _LOAD_GLOBAL_MODULE: + return ((oparg & 1) ? 1 : 0) + 1; + case _LOAD_GLOBAL_BUILTINS: + return ((oparg & 1) ? 1 : 0) + 1; case LOAD_GLOBAL_MODULE: return (oparg & 1 ? 1 : 0) + 1; case LOAD_GLOBAL_BUILTIN: @@ -738,6 +841,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return ((oparg & 1) ? 1 : 0) + 1; case LOAD_METHOD: return ((oparg & 1) ? 1 : 0) + 1; + case _GUARD_TYPE_VERSION: + return 1; + case _CHECK_MANAGED_OBJECT_HAS_VALUES: + return 1; + case _LOAD_ATTR_INSTANCE_VALUE: + return ((oparg & 1) ? 1 : 0) + 1; case LOAD_ATTR_INSTANCE_VALUE: return (oparg & 1 ? 1 : 0) + 1; case LOAD_ATTR_MODULE: @@ -792,6 +901,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case POP_JUMP_IF_TRUE: return 0; + case IS_NONE: + return 1; case POP_JUMP_IF_NONE: return 0; case POP_JUMP_IF_NOT_NONE: @@ -816,10 +927,28 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 2; case INSTRUMENTED_FOR_ITER: return 0; + case _ITER_CHECK_LIST: + return 1; + case _IS_ITER_EXHAUSTED_LIST: + return 2; + case _ITER_NEXT_LIST: + return 2; case FOR_ITER_LIST: return 2; + case _ITER_CHECK_TUPLE: + return 1; + case _IS_ITER_EXHAUSTED_TUPLE: + return 2; + case _ITER_NEXT_TUPLE: + return 2; case FOR_ITER_TUPLE: return 2; + case _ITER_CHECK_RANGE: + return 1; + case _IS_ITER_EXHAUSTED_RANGE: + return 2; + case _ITER_NEXT_RANGE: + return 2; case FOR_ITER_RANGE: return 2; case FOR_ITER_GEN: @@ -938,6 +1067,18 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case RESERVED: return 0; + case _POP_JUMP_IF_FALSE: + return 0; + case _POP_JUMP_IF_TRUE: + return 0; + case JUMP_TO_TOP: + return 0; + case SAVE_IP: + return 0; + case EXIT_TRACE: + return 0; + case INSERT: + return oparg + 1; default: return -1; } @@ -1393,5 +1534,6 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", [JUMP_TO_TOP] = "JUMP_TO_TOP", + [INSERT] = "INSERT", }; #endif // NEED_OPCODE_METADATA diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h new file mode 100644 index 0000000000000..2ae657c4e117f --- /dev/null +++ b/Include/internal/pycore_optimizer.h @@ -0,0 +1,20 @@ +#ifndef Py_INTERNAL_OPTIMIZER_H +#define Py_INTERNAL_OPTIMIZER_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_uops.h" + +int _Py_uop_analyze_and_optimize(PyCodeObject *code, + _PyUOpInstruction *trace, int trace_len, int curr_stackentries); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_OPTIMIZER_H */ diff --git a/Include/internal/pycore_uops.h b/Include/internal/pycore_uops.h index 57a5970353b36..254eeca2361be 100644 --- a/Include/internal/pycore_uops.h +++ b/Include/internal/pycore_uops.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#define _Py_UOP_MAX_TRACE_LENGTH 32 +#define _Py_UOP_MAX_TRACE_LENGTH 64 typedef struct { uint32_t opcode; diff --git a/Makefile.pre.in b/Makefile.pre.in index 3a628bf49e97c..32d928316f221 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -405,6 +405,7 @@ PYTHON_OBJS= \ Python/mysnprintf.o \ Python/mystrtoul.o \ Python/optimizer.o \ + Python/optimizer_analysis.o \ Python/pathconfig.o \ Python/preconfig.o \ Python/pyarena.o \ @@ -1552,10 +1553,12 @@ regen-cases: -m $(srcdir)/Include/internal/pycore_opcode_metadata.h.new \ -e $(srcdir)/Python/executor_cases.c.h.new \ -p $(srcdir)/Lib/_opcode_metadata.py.new \ + -a $(srcdir)/Python/abstract_interp_cases.c.h.new \ $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode_metadata.h $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(UPDATE_FILE) $(srcdir)/Python/executor_cases.c.h $(srcdir)/Python/executor_cases.c.h.new + $(UPDATE_FILE) $(srcdir)/Python/abstract_interp_cases.c.h $(srcdir)/Python/abstract_interp_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Lib/_opcode_metadata.py $(srcdir)/Lib/_opcode_metadata.py.new Python/compile.o: $(srcdir)/Include/internal/pycore_opcode_metadata.h @@ -1568,6 +1571,7 @@ Python/ceval.o: \ Python/executor.o: \ $(srcdir)/Include/internal/pycore_opcode_metadata.h \ + $(srcdir)/Include/internal/pycore_optimizer.h \ $(srcdir)/Python/ceval_macros.h \ $(srcdir)/Python/executor_cases.c.h @@ -1576,7 +1580,12 @@ Python/flowgraph.o: \ Python/optimizer.o: \ $(srcdir)/Python/executor_cases.c.h \ - $(srcdir)/Include/internal/pycore_opcode_metadata.h + $(srcdir)/Include/internal/pycore_opcode_metadata.h \ + $(srcdir)/Include/internal/pycore_optimizer.h + +Python/optimizer_analysis.o: \ + $(srcdir)/Include/internal/pycore_opcode_metadata.h \ + $(srcdir)/Include/internal/pycore_optimizer.h Python/frozen.o: $(FROZEN_FILES_OUT) @@ -1786,6 +1795,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_obmalloc_init.h \ $(srcdir)/Include/internal/pycore_opcode.h \ $(srcdir)/Include/internal/pycore_opcode_utils.h \ + $(srcdir)/Include/internal/pycore_optimizer.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ $(srcdir)/Include/internal/pycore_pyarena.h \ $(srcdir)/Include/internal/pycore_pyerrors.h \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-02-09-55-21.gh-issue-107557.P1z-in.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-02-09-55-21.gh-issue-107557.P1z-in.rst new file mode 100644 index 0000000000000..392f59c79e8de --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-02-09-55-21.gh-issue-107557.P1z-in.rst @@ -0,0 +1 @@ +Generate the cases needed for the barebones tier 2 abstract interpreter for optimization passes in CPython. diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index e247637a0dfe5..bdcf29ba44dab 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -218,6 +218,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 2a0e009308022..45333fa97f1c6 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -283,6 +283,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index bfe59acf12a69..b0e62864421e1 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -248,6 +248,7 @@ + @@ -279,6 +280,7 @@ + @@ -548,6 +550,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 0a8b0c3faf51e..d5f61e9c5d7c8 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -648,6 +648,9 @@ Include\internal + + Include\internal + Include\internal @@ -732,6 +735,9 @@ Include\internal + + Include\internal + Modules\zlib @@ -1223,6 +1229,9 @@ Python + + Python + Python diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h new file mode 100644 index 0000000000000..6bfcf534646b1 --- /dev/null +++ b/Python/abstract_interp_cases.c.h @@ -0,0 +1,761 @@ +// This file is generated by Tools/cases_generator/generate_cases.py +// from: +// Python/bytecodes.c +// Do not edit! + + case NOP: { + break; + } + + case POP_TOP: { + STACK_SHRINK(1); + break; + } + + case PUSH_NULL: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case END_SEND: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case UNARY_NEGATIVE: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case UNARY_NOT: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case TO_BOOL: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case TO_BOOL_BOOL: { + break; + } + + case TO_BOOL_INT: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case TO_BOOL_LIST: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case TO_BOOL_NONE: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case TO_BOOL_STR: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case TO_BOOL_ALWAYS_TRUE: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case UNARY_INVERT: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _GUARD_BOTH_INT: { + break; + } + + case _GUARD_BOTH_FLOAT: { + break; + } + + case _BINARY_OP_MULTIPLY_FLOAT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _BINARY_OP_ADD_FLOAT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _BINARY_OP_SUBTRACT_FLOAT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _GUARD_BOTH_UNICODE: { + break; + } + + case _BINARY_OP_ADD_UNICODE: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BINARY_SUBSCR: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BINARY_SLICE: { + STACK_SHRINK(2); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case STORE_SLICE: { + STACK_SHRINK(4); + break; + } + + case BINARY_SUBSCR_LIST_INT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BINARY_SUBSCR_STR_INT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BINARY_SUBSCR_TUPLE_INT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BINARY_SUBSCR_DICT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case LIST_APPEND: { + STACK_SHRINK(1); + break; + } + + case SET_ADD: { + STACK_SHRINK(1); + break; + } + + case STORE_SUBSCR: { + STACK_SHRINK(3); + break; + } + + case STORE_SUBSCR_LIST_INT: { + STACK_SHRINK(3); + break; + } + + case STORE_SUBSCR_DICT: { + STACK_SHRINK(3); + break; + } + + case DELETE_SUBSCR: { + STACK_SHRINK(2); + break; + } + + case CALL_INTRINSIC_1: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_INTRINSIC_2: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case GET_AITER: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case GET_ANEXT: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case GET_AWAITABLE: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case POP_EXCEPT: { + STACK_SHRINK(1); + break; + } + + case LOAD_ASSERTION_ERROR: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case LOAD_BUILD_CLASS: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case STORE_NAME: { + STACK_SHRINK(1); + break; + } + + case DELETE_NAME: { + break; + } + + case UNPACK_SEQUENCE: { + STACK_SHRINK(1); + STACK_GROW(oparg); + break; + } + + case UNPACK_SEQUENCE_TWO_TUPLE: { + STACK_SHRINK(1); + STACK_GROW(oparg); + break; + } + + case UNPACK_SEQUENCE_TUPLE: { + STACK_SHRINK(1); + STACK_GROW(oparg); + break; + } + + case UNPACK_SEQUENCE_LIST: { + STACK_SHRINK(1); + STACK_GROW(oparg); + break; + } + + case UNPACK_EX: { + STACK_GROW((oparg & 0xFF) + (oparg >> 8)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg >> 8))), true); + break; + } + + case STORE_ATTR: { + STACK_SHRINK(2); + break; + } + + case DELETE_ATTR: { + STACK_SHRINK(1); + break; + } + + case STORE_GLOBAL: { + STACK_SHRINK(1); + break; + } + + case DELETE_GLOBAL: { + break; + } + + case _LOAD_LOCALS: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _LOAD_FROM_DICT_OR_GLOBALS: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case LOAD_GLOBAL: { + STACK_GROW(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); + break; + } + + case _GUARD_GLOBALS_VERSION: { + break; + } + + case _GUARD_BUILTINS_VERSION: { + break; + } + + case _LOAD_GLOBAL_MODULE: { + STACK_GROW(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); + break; + } + + case _LOAD_GLOBAL_BUILTINS: { + STACK_GROW(1); + STACK_GROW(((oparg & 1) ? 1 : 0)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); + break; + } + + case DELETE_FAST: { + break; + } + + case DELETE_DEREF: { + break; + } + + case LOAD_FROM_DICT_OR_DEREF: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case LOAD_DEREF: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case STORE_DEREF: { + STACK_SHRINK(1); + break; + } + + case COPY_FREE_VARS: { + break; + } + + case BUILD_STRING: { + STACK_SHRINK(oparg); + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BUILD_TUPLE: { + STACK_SHRINK(oparg); + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BUILD_LIST: { + STACK_SHRINK(oparg); + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case LIST_EXTEND: { + STACK_SHRINK(1); + break; + } + + case SET_UPDATE: { + STACK_SHRINK(1); + break; + } + + case BUILD_SET: { + STACK_SHRINK(oparg); + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BUILD_MAP: { + STACK_SHRINK(oparg*2); + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case SETUP_ANNOTATIONS: { + break; + } + + case BUILD_CONST_KEY_MAP: { + STACK_SHRINK(oparg); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case DICT_UPDATE: { + STACK_SHRINK(1); + break; + } + + case DICT_MERGE: { + STACK_SHRINK(1); + break; + } + + case MAP_ADD: { + STACK_SHRINK(2); + break; + } + + case LOAD_SUPER_ATTR_ATTR: { + STACK_SHRINK(2); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(0)), true); + break; + } + + case LOAD_SUPER_ATTR_METHOD: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case LOAD_ATTR: { + STACK_GROW(((oparg & 1) ? 1 : 0)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); + break; + } + + case _GUARD_TYPE_VERSION: { + break; + } + + case _CHECK_MANAGED_OBJECT_HAS_VALUES: { + break; + } + + case _LOAD_ATTR_INSTANCE_VALUE: { + STACK_GROW(((oparg & 1) ? 1 : 0)); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); + break; + } + + case COMPARE_OP: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case COMPARE_OP_FLOAT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case COMPARE_OP_INT: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case COMPARE_OP_STR: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case IS_OP: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CONTAINS_OP: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CHECK_EG_MATCH: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CHECK_EXC_MATCH: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case IS_NONE: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case GET_LEN: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case MATCH_CLASS: { + STACK_SHRINK(2); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case MATCH_MAPPING: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case MATCH_SEQUENCE: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case MATCH_KEYS: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case GET_ITER: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case GET_YIELD_FROM_ITER: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _ITER_CHECK_LIST: { + break; + } + + case _IS_ITER_EXHAUSTED_LIST: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _ITER_NEXT_LIST: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _ITER_CHECK_TUPLE: { + break; + } + + case _IS_ITER_EXHAUSTED_TUPLE: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _ITER_NEXT_TUPLE: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _ITER_CHECK_RANGE: { + break; + } + + case _IS_ITER_EXHAUSTED_RANGE: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _ITER_NEXT_RANGE: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case WITH_EXCEPT_START: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case PUSH_EXC_INFO: { + STACK_GROW(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2)), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_TYPE_1: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_STR_1: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_TUPLE_1: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case EXIT_INIT_CHECK: { + STACK_SHRINK(1); + break; + } + + case CALL_NO_KW_BUILTIN_O: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_BUILTIN_FAST: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_LEN: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_ISINSTANCE: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_METHOD_DESCRIPTOR_O: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CALL_NO_KW_METHOD_DESCRIPTOR_FAST: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case MAKE_FUNCTION: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case SET_FUNCTION_ATTRIBUTE: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BUILD_SLICE: { + STACK_SHRINK(((oparg == 3) ? 1 : 0)); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case CONVERT_VALUE: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case FORMAT_SIMPLE: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case FORMAT_WITH_SPEC: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case BINARY_OP: { + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case SWAP: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-2 - (oparg-2))), true); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _POP_JUMP_IF_FALSE: { + STACK_SHRINK(1); + break; + } + + case _POP_JUMP_IF_TRUE: { + STACK_SHRINK(1); + break; + } + + case JUMP_TO_TOP: { + break; + } + + case SAVE_IP: { + break; + } + + case EXIT_TRACE: { + break; + } + + case INSERT: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - oparg)), true); + break; + } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5efa36fcf5c62..e9a5cf59e7d68 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3743,6 +3743,11 @@ dummy_func( return frame; } + op(INSERT, (unused[oparg], top -- top, unused[oparg])) { + // Inserts TOS at position specified by oparg; + memmove(&stack_pointer[-1 - oparg], &stack_pointer[-oparg], oparg * sizeof(stack_pointer[0])); + } + // END BYTECODES // diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 5e3c84b2b4b61..85d27777423ab 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2733,3 +2733,12 @@ return frame; break; } + + case INSERT: { + PyObject *top; + top = stack_pointer[-1]; + // Inserts TOS at position specified by oparg; + memmove(&stack_pointer[-1 - oparg], &stack_pointer[-oparg], oparg * sizeof(stack_pointer[0])); + stack_pointer[-1 - oparg] = top; + break; + } diff --git a/Python/optimizer.c b/Python/optimizer.c index 6c730aa14b9a4..d3ac2424038ef 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -4,6 +4,7 @@ #include "pycore_opcode.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" +#include "pycore_optimizer.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_uops.h" #include "cpython/optimizer.h" @@ -103,7 +104,8 @@ error_optimize( _PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, - _PyExecutorObject **exec) + _PyExecutorObject **exec, + int Py_UNUSED(stack_entries)) { PyErr_Format(PyExc_SystemError, "Should never call error_optimize"); return -1; @@ -164,7 +166,7 @@ _PyOptimizer_BackEdge(_PyInterpreterFrame *frame, _Py_CODEUNIT *src, _Py_CODEUNI } _PyOptimizerObject *opt = interp->optimizer; _PyExecutorObject *executor = NULL; - int err = opt->optimize(opt, code, dest, &executor); + int err = opt->optimize(opt, code, dest, &executor, (int)(stack_pointer - _PyFrame_Stackbase(frame))); if (err <= 0) { assert(executor == NULL); if (err < 0) { @@ -254,7 +256,9 @@ counter_optimize( _PyOptimizerObject* self, PyCodeObject *code, _Py_CODEUNIT *instr, - _PyExecutorObject **exec_ptr) + _PyExecutorObject **exec_ptr, + int Py_UNUSED(curr_stackentries) +) { _PyCounterExecutorObject *executor = (_PyCounterExecutorObject *)_PyObject_New(&CounterExecutor_Type); if (executor == NULL) { @@ -684,7 +688,8 @@ uop_optimize( _PyOptimizerObject *self, PyCodeObject *code, _Py_CODEUNIT *instr, - _PyExecutorObject **exec_ptr) + _PyExecutorObject **exec_ptr, + int curr_stackentries) { _PyUOpInstruction trace[_Py_UOP_MAX_TRACE_LENGTH]; int trace_length = translate_bytecode_to_trace(code, instr, trace, _Py_UOP_MAX_TRACE_LENGTH); @@ -693,6 +698,10 @@ uop_optimize( return trace_length; } OBJECT_STAT_INC(optimization_traces_created); + char *uop_optimize = Py_GETENV("PYTHONUOPSOPTIMIZE"); + if (uop_optimize != NULL && *uop_optimize > '0') { + trace_length = _Py_uop_analyze_and_optimize(code, trace, trace_length, curr_stackentries); + } _PyUOpExecutorObject *executor = PyObject_NewVar(_PyUOpExecutorObject, &UOpExecutor_Type, trace_length); if (executor == NULL) { return -1; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c new file mode 100644 index 0000000000000..e48e018052c71 --- /dev/null +++ b/Python/optimizer_analysis.c @@ -0,0 +1,26 @@ +#include "Python.h" +#include "opcode.h" +#include "pycore_interp.h" +#include "pycore_opcode.h" +#include "pycore_opcode_metadata.h" +#include "pycore_opcode_utils.h" +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_uops.h" +#include "pycore_long.h" +#include "cpython/optimizer.h" +#include +#include +#include +#include "pycore_optimizer.h" + + +int +_Py_uop_analyze_and_optimize( + PyCodeObject *co, + _PyUOpInstruction *trace, + int trace_len, + int curr_stacklen +) +{ + return trace_len; +} diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 9bc7285e18b2f..90334d0e79da8 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -84,6 +84,7 @@ def clean_lines(text): Python/frozen_modules/*.h Python/generated_cases.c.h Python/executor_cases.c.h +Python/abstract_interp_cases.c.h # not actually source Python/bytecodes.c diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index c64d391bae13b..d1ac0410619c9 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -716,3 +716,5 @@ Modules/expat/xmlrole.c - error - ## other Modules/_io/_iomodule.c - _PyIO_Module - Modules/_sqlite/module.c - _sqlite3module - +Python/optimizer_analysis.c - _Py_PartitionRootNode_Type - +Python/optimizer_analysis.c - _Py_UOpsAbstractInterpContext_Type - diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index e78b0edff9a0c..e170e110f80cf 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -16,6 +16,7 @@ from flags import InstructionFlags, variable_used from instructions import ( AnyInstruction, + AbstractInstruction, Component, Instruction, MacroInstruction, @@ -44,6 +45,9 @@ DEFAULT_EXECUTOR_OUTPUT = os.path.relpath( os.path.join(ROOT, "Python/executor_cases.c.h") ) +DEFAULT_ABSTRACT_INTERPRETER_OUTPUT = os.path.relpath( + os.path.join(ROOT, "Python/abstract_interp_cases.c.h") +) # Constants used instead of size for macro expansions. # Note: 1, 2, 4 must match actual cache entry sizes. @@ -58,6 +62,23 @@ INSTR_FMT_PREFIX = "INSTR_FMT_" +# TODO: generate all these after updating the DSL +SPECIALLY_HANDLED_ABSTRACT_INSTR = { + "LOAD_FAST", + "LOAD_FAST_CHECK", + "LOAD_FAST_AND_CLEAR", + "LOAD_CONST", + "STORE_FAST", + "STORE_FAST_MAYBE_NULL", + "COPY", + + # Arithmetic + "_BINARY_OP_MULTIPLY_INT", + "_BINARY_OP_ADD_INT", + "_BINARY_OP_SUBTRACT_INT", + +} + arg_parser = argparse.ArgumentParser( description="Generate the code for the interpreter switch.", formatter_class=argparse.ArgumentDefaultsHelpFormatter, @@ -92,7 +113,13 @@ help="Write executor cases to this file", default=DEFAULT_EXECUTOR_OUTPUT, ) - +arg_parser.add_argument( + "-a", + "--abstract-interpreter-cases", + type=str, + help="Write abstract interpreter cases to this file", + default=DEFAULT_ABSTRACT_INTERPRETER_OUTPUT, +) class Generator(Analyzer): def get_stack_effect_info( @@ -109,7 +136,7 @@ def effect_str(effects: list[StackEffect]) -> str: pushed: str | None match thing: case parsing.InstDef(): - if thing.kind != "op": + if thing.kind != "op" or self.instrs[thing.name].is_viable_uop(): instr = self.instrs[thing.name] popped = effect_str(instr.input_effects) pushed = effect_str(instr.output_effects) @@ -588,6 +615,35 @@ def write_executor_instructions( file=sys.stderr, ) + def write_abstract_interpreter_instructions( + self, abstract_interpreter_filename: str, emit_line_directives: bool + ) -> None: + """Generate cases for the Tier 2 abstract interpreter/analzyer.""" + with open(abstract_interpreter_filename, "w") as f: + self.out = Formatter(f, 8, emit_line_directives) + self.write_provenance_header() + for thing in self.everything: + match thing: + case OverriddenInstructionPlaceHolder(): + pass + case parsing.InstDef(): + instr = AbstractInstruction(self.instrs[thing.name].inst) + if instr.is_viable_uop() and instr.name not in SPECIALLY_HANDLED_ABSTRACT_INSTR: + self.out.emit("") + with self.out.block(f"case {thing.name}:"): + instr.write(self.out, tier=TIER_TWO) + self.out.emit("break;") + case parsing.Macro(): + pass + case parsing.Pseudo(): + pass + case _: + typing.assert_never(thing) + print( + f"Wrote some stuff to {abstract_interpreter_filename}", + file=sys.stderr, + ) + def write_overridden_instr_place_holder( self, place_holder: OverriddenInstructionPlaceHolder ) -> None: @@ -629,6 +685,8 @@ def main(): a.write_instructions(args.output, args.emit_line_directives) a.write_metadata(args.metadata, args.pymetadata) a.write_executor_instructions(args.executor_cases, args.emit_line_directives) + a.write_abstract_interpreter_instructions(args.abstract_interpreter_cases, + args.emit_line_directives) if __name__ == "__main__": diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index aa94dbb07ea1c..a505df08fa265 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -248,6 +248,25 @@ def write_body( InstructionOrCacheEffect = Instruction | parsing.CacheEffect +# Instruction used for abstract interpretation. +class AbstractInstruction(Instruction): + def __init__(self, inst: parsing.InstDef): + super().__init__(inst) + + def write(self, out: Formatter, tier: Tiers = TIER_ONE) -> None: + """Write one abstract instruction, sans prologue and epilogue.""" + stacking.write_single_instr_for_abstract_interp(self, out) + + def write_body( + self, + out: Formatter, + dedent: int, + active_caches: list[ActiveCacheEffect], + tier: Tiers = TIER_ONE, + ) -> None: + pass + + @dataclasses.dataclass class Component: instr: Instruction diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 9bb7f46844224..31a21e026cb49 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -391,3 +391,32 @@ def write_components( poke.as_stack_effect(), poke.effect, ) + + +def write_single_instr_for_abstract_interp( + instr: Instruction, out: Formatter +): + try: + _write_components_for_abstract_interp( + [Component(instr, instr.active_caches)], + out, + ) + except AssertionError as err: + raise AssertionError(f"Error writing abstract instruction {instr.name}") from err + + +def _write_components_for_abstract_interp( + parts: list[Component], + out: Formatter, +): + managers = get_managers(parts) + for mgr in managers: + if mgr is managers[-1]: + out.stack_adjust(mgr.final_offset.deep, mgr.final_offset.high) + # Use clone() since adjust_inverse() mutates final_offset + mgr.adjust_inverse(mgr.final_offset.clone()) + # NULL out the output stack effects + for poke in mgr.pokes: + if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: + out.emit(f"PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)" + f"PARTITIONNODE_NULLROOT, PEEK(-({poke.offset.as_index()})), true);") From webhook-mailer at python.org Tue Aug 15 16:41:38 2023 From: webhook-mailer at python.org (carljm) Date: Tue, 15 Aug 2023 20:41:38 -0000 Subject: [Python-checkins] Note that lnotab_notes.txt is only valid before 3.11 (#107961) Message-ID: https://github.com/python/cpython/commit/99b6ce56f8185bb936cae3b2b67ae24b11853e30 commit: 99b6ce56f8185bb936cae3b2b67ae24b11853e30 branch: main author: Martin DeMello committer: carljm date: 2023-08-15T20:27:35Z summary: Note that lnotab_notes.txt is only valid before 3.11 (#107961) files: M Objects/lnotab_notes.txt diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index 362b87a86a481..d45d09d4ab9a5 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -1,4 +1,7 @@ -Description of the internal format of the line number table +Description of the internal format of the line number table in Python 3.10 +and earlier. + +(For 3.11 onwards, see Objects/locations.md) Conceptually, the line number table consists of a sequence of triples: start-offset (inclusive), end-offset (exclusive), line-number. From webhook-mailer at python.org Tue Aug 15 16:55:53 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 15 Aug 2023 20:55:53 -0000 Subject: [Python-checkins] gh-106368: Argument Clinic: Add tests for cloned functions with custom C base names (#107977) Message-ID: https://github.com/python/cpython/commit/bb456a08a3db851e6feaefc3328f39096919ec8d commit: bb456a08a3db851e6feaefc3328f39096919ec8d branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-15T14:45:53-06:00 summary: gh-106368: Argument Clinic: Add tests for cloned functions with custom C base names (#107977) files: M Lib/test/test_clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 896730571d815..76729c8c3e4f7 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -653,6 +653,37 @@ def test_no_c_basename_cloned(self): err = "No C basename provided after 'as' keyword" self.expect_failure(block, err, lineno=5) + def test_cloned_with_custom_c_basename(self): + raw = dedent(""" + /*[clinic input] + # Make sure we don't create spurious clinic/ directories. + output everything suppress + foo2 + [clinic start generated code]*/ + + /*[clinic input] + foo as foo1 = foo2 + [clinic start generated code]*/ + """) + self.clinic.parse(raw) + funcs = self.clinic.functions + self.assertEqual(len(funcs), 2) + self.assertEqual(funcs[1].name, "foo") + self.assertEqual(funcs[1].c_basename, "foo1") + + def test_cloned_with_illegal_c_basename(self): + block = """ + /*[clinic input] + class C "void *" "" + foo1 + [clinic start generated code]*/ + + /*[clinic input] + foo2 as .illegal. = foo1 + [clinic start generated code]*/ + """ + err = "Illegal C basename: '.illegal. = foo1'" + self.expect_failure(block, err, lineno=7) class ParseFileUnitTest(TestCase): From webhook-mailer at python.org Tue Aug 15 21:03:49 2023 From: webhook-mailer at python.org (gpshead) Date: Wed, 16 Aug 2023 01:03:49 -0000 Subject: [Python-checkins] More actionable error message when spawn is incorrectly used. (#102203) Message-ID: https://github.com/python/cpython/commit/a794ebeb028f7ef287c780d3890f816db9c21c51 commit: a794ebeb028f7ef287c780d3890f816db9c21c51 branch: main author: Yuxin Wu committer: gpshead date: 2023-08-15T18:03:45-07:00 summary: More actionable error message when spawn is incorrectly used. (#102203) Co-authored-by: Yuxin Wu Co-authored-by: Oleg Iarygin files: M Lib/multiprocessing/spawn.py diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index f1af770910471..daac1ecc34b55 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -150,7 +150,11 @@ def _check_not_importing_main(): ... The "freeze_support()" line can be omitted if the program - is not going to be frozen to produce an executable.''') + is not going to be frozen to produce an executable. + + To fix this issue, refer to the "Safe importing of main module" + section in https://docs.python.org/3/library/multiprocessing.html + ''') def get_preparation_data(name): From webhook-mailer at python.org Tue Aug 15 21:43:32 2023 From: webhook-mailer at python.org (gpshead) Date: Wed, 16 Aug 2023 01:43:32 -0000 Subject: [Python-checkins] [3.11] More actionable error message when spawn is incorrectly used. (GH-102203) (#107991) Message-ID: https://github.com/python/cpython/commit/3f7dfb6d392f6278e6aee9233003dcaa45e2e736 commit: 3f7dfb6d392f6278e6aee9233003dcaa45e2e736 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: gpshead date: 2023-08-16T01:43:28Z summary: [3.11] More actionable error message when spawn is incorrectly used. (GH-102203) (#107991) More actionable error message when spawn is incorrectly used. (GH-102203) (cherry picked from commit a794ebeb028f7ef287c780d3890f816db9c21c51) Co-authored-by: Yuxin Wu Co-authored-by: Yuxin Wu Co-authored-by: Oleg Iarygin files: M Lib/multiprocessing/spawn.py diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index f1af770910471..daac1ecc34b55 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -150,7 +150,11 @@ def _check_not_importing_main(): ... The "freeze_support()" line can be omitted if the program - is not going to be frozen to produce an executable.''') + is not going to be frozen to produce an executable. + + To fix this issue, refer to the "Safe importing of main module" + section in https://docs.python.org/3/library/multiprocessing.html + ''') def get_preparation_data(name): From webhook-mailer at python.org Wed Aug 16 03:00:07 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 16 Aug 2023 07:00:07 -0000 Subject: [Python-checkins] gh-99203: shutil.make_archive(): restore select CPython <= 3.10.5 behavior (GH-99802) Message-ID: https://github.com/python/cpython/commit/a86df298df5b02e2d69ea6879e9ed10a7adb85d0 commit: a86df298df5b02e2d69ea6879e9ed10a7adb85d0 branch: main author: 6t8k <58048945+6t8k at users.noreply.github.com> committer: serhiy-storchaka date: 2023-08-16T10:00:03+03:00 summary: gh-99203: shutil.make_archive(): restore select CPython <= 3.10.5 behavior (GH-99802) Restore following CPython <= 3.10.5 behavior of shutil.make_archive() that went away as part of gh-93160: Do not create an empty archive if root_dir is not a directory, and, in that case, raise FileNotFoundError or NotADirectoryError regardless of format choice. Beyond the brought-back behavior, the function may now also raise these exceptions in dry_run mode. files: A Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst M Lib/shutil.py M Lib/test/test_shutil.py diff --git a/Lib/shutil.py b/Lib/shutil.py index 3f2864af517e7..b37bd082eee0c 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -1156,6 +1156,10 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, supports_root_dir = getattr(func, 'supports_root_dir', False) save_cwd = None if root_dir is not None: + stmd = os.stat(root_dir).st_mode + if not stat.S_ISDIR(stmd): + raise NotADirectoryError(errno.ENOTDIR, 'Not a directory', root_dir) + if supports_root_dir: # Support path-like base_name here for backwards-compatibility. base_name = os.fspath(base_name) diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 93f20a6ff4133..a2ca4df135846 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1839,6 +1839,49 @@ def test_register_archive_format(self): formats = [name for name, params in get_archive_formats()] self.assertNotIn('xxx', formats) + def test_make_tarfile_rootdir_nodir(self): + # GH-99203 + self.addCleanup(os_helper.unlink, f'{TESTFN}.tar') + for dry_run in (False, True): + with self.subTest(dry_run=dry_run): + tmp_dir = self.mkdtemp() + nonexisting_file = os.path.join(tmp_dir, 'nonexisting') + with self.assertRaises(FileNotFoundError) as cm: + make_archive(TESTFN, 'tar', nonexisting_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOENT) + self.assertEqual(cm.exception.filename, nonexisting_file) + self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + + tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) + os.close(tmp_fd) + with self.assertRaises(NotADirectoryError) as cm: + make_archive(TESTFN, 'tar', tmp_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + self.assertEqual(cm.exception.filename, tmp_file) + self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + + @support.requires_zlib() + def test_make_zipfile_rootdir_nodir(self): + # GH-99203 + self.addCleanup(os_helper.unlink, f'{TESTFN}.zip') + for dry_run in (False, True): + with self.subTest(dry_run=dry_run): + tmp_dir = self.mkdtemp() + nonexisting_file = os.path.join(tmp_dir, 'nonexisting') + with self.assertRaises(FileNotFoundError) as cm: + make_archive(TESTFN, 'zip', nonexisting_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOENT) + self.assertEqual(cm.exception.filename, nonexisting_file) + self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + + tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) + os.close(tmp_fd) + with self.assertRaises(NotADirectoryError) as cm: + make_archive(TESTFN, 'zip', tmp_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + self.assertEqual(cm.exception.filename, tmp_file) + self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + ### shutil.unpack_archive def check_unpack_archive(self, format, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst b/Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst new file mode 100644 index 0000000000000..fcfb044d476ac --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst @@ -0,0 +1,5 @@ +Restore following CPython <= 3.10.5 behavior of :func:`shutil.make_archive`: +do not create an empty archive if ``root_dir`` is not a directory, and, in that +case, raise :class:`FileNotFoundError` or :class:`NotADirectoryError` +regardless of ``format`` choice. Beyond the brought-back behavior, the function +may now also raise these exceptions in ``dry_run`` mode. From webhook-mailer at python.org Wed Aug 16 03:43:49 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 16 Aug 2023 07:43:49 -0000 Subject: [Python-checkins] gh-100061: Proper fix of the bug in the matching of possessive quantifiers (GH-102612) Message-ID: https://github.com/python/cpython/commit/abd9cc52d94b8e2835322b62c29f09bb0e6fcfe9 commit: abd9cc52d94b8e2835322b62c29f09bb0e6fcfe9 branch: main author: SKO <41810398+uyw4687 at users.noreply.github.com> committer: serhiy-storchaka date: 2023-08-16T10:43:45+03:00 summary: gh-100061: Proper fix of the bug in the matching of possessive quantifiers (GH-102612) Restore the global Input Stream pointer after trying to match a sub-pattern. Co-authored-by: Ma Lin files: A Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst M Lib/re/_compiler.py M Lib/test/test_re.py M Modules/_sre/sre_lib.h diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index f5fd160ba0043..d0a4c55caf6e4 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -100,13 +100,6 @@ def _compile(code, pattern, flags): emit(ANY_ALL) else: emit(ANY) - elif op is POSSESSIVE_REPEAT: - # gh-106052: Possessive quantifiers do not work when the - # subpattern contains backtracking, i.e. "(?:ab?c)*+". - # Implement it as equivalent greedy qualifier in atomic group. - p = [(MAX_REPEAT, av)] - p = [(ATOMIC_GROUP, p)] - _compile(code, p, flags) elif op in REPEATING_CODES: if _simple(av[2]): emit(REPEATING_CODES[op][2]) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index bf3698ac78a88..042f97f57ecf1 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2342,7 +2342,17 @@ def test_bug_gh91616(self): self.assertTrue(re.fullmatch(r'(?s:(?>.*?\.).*)\Z', "a.txt")) # reproducer self.assertTrue(re.fullmatch(r'(?s:(?=(?P.*?\.))(?P=g0).*)\Z', "a.txt")) - def test_bug_gh106052(self): + def test_bug_gh100061(self): + # gh-100061 + self.assertEqual(re.match('(?>(?:.(?!D))+)', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?:.(?!D))++', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?>(?:.(?!D))*)', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?:.(?!D))*+', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?>(?:.(?!D))?)', 'CDE').span(), (0, 0)) + self.assertEqual(re.match('(?:.(?!D))?+', 'CDE').span(), (0, 0)) + self.assertEqual(re.match('(?>(?:.(?!D)){1,3})', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?:.(?!D)){1,3}+', 'ABCDE').span(), (0, 2)) + # gh-106052 self.assertEqual(re.match("(?>(?:ab?c)+)", "aca").span(), (0, 2)) self.assertEqual(re.match("(?:ab?c)++", "aca").span(), (0, 2)) self.assertEqual(re.match("(?>(?:ab?c)*)", "aca").span(), (0, 2)) @@ -2451,7 +2461,6 @@ def test_atomic_group(self): 17: SUCCESS ''') - @unittest.expectedFailure # gh-106052 def test_possesive_repeat_one(self): self.assertEqual(get_debug_out(r'a?+'), '''\ POSSESSIVE_REPEAT 0 1 @@ -2464,7 +2473,6 @@ def test_possesive_repeat_one(self): 12: SUCCESS ''') - @unittest.expectedFailure # gh-106052 def test_possesive_repeat(self): self.assertEqual(get_debug_out(r'(?:ab)?+'), '''\ POSSESSIVE_REPEAT 0 1 diff --git a/Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst b/Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst new file mode 100644 index 0000000000000..dfed34f6ae976 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst @@ -0,0 +1,2 @@ +Fix a bug that causes wrong matches for regular expressions with possessive +qualifier. diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index c1a774f69090b..ae80009fd63bb 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -1336,6 +1336,10 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) MARK_POP(ctx->lastmark); LASTMARK_RESTORE(); + /* Restore the global Input Stream pointer + since it can change after jumps. */ + state->ptr = ptr; + /* We have sufficient matches, so exit loop. */ break; } From webhook-mailer at python.org Wed Aug 16 03:44:25 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 16 Aug 2023 07:44:25 -0000 Subject: [Python-checkins] [3.11] gh-99203: shutil.make_archive(): restore select CPython <= 3.10.5 behavior (GH-99802) (GH-107999) Message-ID: https://github.com/python/cpython/commit/4cfbcffd3d0bbaab1610896ad9ff9c588467681d commit: 4cfbcffd3d0bbaab1610896ad9ff9c588467681d branch: 3.11 author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-16T07:44:21Z summary: [3.11] gh-99203: shutil.make_archive(): restore select CPython <= 3.10.5 behavior (GH-99802) (GH-107999) Restore following CPython <= 3.10.5 behavior of shutil.make_archive() that went away as part of gh-93160: Do not create an empty archive if root_dir is not a directory, and, in that case, raise FileNotFoundError or NotADirectoryError regardless of format choice. Beyond the brought-back behavior, the function may now also raise these exceptions in dry_run mode. (cherry picked from commit a86df298df5b02e2d69ea6879e9ed10a7adb85d0) Co-authored-by: 6t8k <58048945+6t8k at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst M Lib/shutil.py M Lib/test/test_shutil.py diff --git a/Lib/shutil.py b/Lib/shutil.py index f4efaef38933d..1c3a75da55ba5 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -1114,10 +1114,14 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, if base_dir is None: base_dir = os.curdir - support_root_dir = format_info[3] + supports_root_dir = format_info[3] save_cwd = None if root_dir is not None: - if support_root_dir: + stmd = os.stat(root_dir).st_mode + if not stat.S_ISDIR(stmd): + raise NotADirectoryError(errno.ENOTDIR, 'Not a directory', root_dir) + + if supports_root_dir: # Support path-like base_name here for backwards-compatibility. base_name = os.fspath(base_name) kwargs['root_dir'] = root_dir diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index bd82aa54368eb..274a8b7377e3b 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1629,6 +1629,49 @@ def test_register_archive_format(self): formats = [name for name, params in get_archive_formats()] self.assertNotIn('xxx', formats) + def test_make_tarfile_rootdir_nodir(self): + # GH-99203 + self.addCleanup(os_helper.unlink, f'{TESTFN}.tar') + for dry_run in (False, True): + with self.subTest(dry_run=dry_run): + tmp_dir = self.mkdtemp() + nonexisting_file = os.path.join(tmp_dir, 'nonexisting') + with self.assertRaises(FileNotFoundError) as cm: + make_archive(TESTFN, 'tar', nonexisting_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOENT) + self.assertEqual(cm.exception.filename, nonexisting_file) + self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + + tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) + os.close(tmp_fd) + with self.assertRaises(NotADirectoryError) as cm: + make_archive(TESTFN, 'tar', tmp_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + self.assertEqual(cm.exception.filename, tmp_file) + self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + + @support.requires_zlib() + def test_make_zipfile_rootdir_nodir(self): + # GH-99203 + self.addCleanup(os_helper.unlink, f'{TESTFN}.zip') + for dry_run in (False, True): + with self.subTest(dry_run=dry_run): + tmp_dir = self.mkdtemp() + nonexisting_file = os.path.join(tmp_dir, 'nonexisting') + with self.assertRaises(FileNotFoundError) as cm: + make_archive(TESTFN, 'zip', nonexisting_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOENT) + self.assertEqual(cm.exception.filename, nonexisting_file) + self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + + tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) + os.close(tmp_fd) + with self.assertRaises(NotADirectoryError) as cm: + make_archive(TESTFN, 'zip', tmp_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + self.assertEqual(cm.exception.filename, tmp_file) + self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + ### shutil.unpack_archive def check_unpack_archive(self, format, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst b/Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst new file mode 100644 index 0000000000000..fcfb044d476ac --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst @@ -0,0 +1,5 @@ +Restore following CPython <= 3.10.5 behavior of :func:`shutil.make_archive`: +do not create an empty archive if ``root_dir`` is not a directory, and, in that +case, raise :class:`FileNotFoundError` or :class:`NotADirectoryError` +regardless of ``format`` choice. Beyond the brought-back behavior, the function +may now also raise these exceptions in ``dry_run`` mode. From webhook-mailer at python.org Wed Aug 16 04:07:06 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 16 Aug 2023 08:07:06 -0000 Subject: [Python-checkins] [3.11] gh-100814: Fix exception for invalid callable value of Tkinter image option (GH-107692) (GH-107723) Message-ID: https://github.com/python/cpython/commit/af08bcab8f7e0067de54a91b11ca5e4b99ec11b7 commit: af08bcab8f7e0067de54a91b11ca5e4b99ec11b7 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: serhiy-storchaka date: 2023-08-16T11:07:02+03:00 summary: [3.11] gh-100814: Fix exception for invalid callable value of Tkinter image option (GH-107692) (GH-107723) Passing a callable object as an option value to a Tkinter image now raises the expected TclError instead of an AttributeError. (cherry picked from commit 50e3cc9748eb2103eb7ed6cc5a74d177df3cfb13) Co-authored-by: Serhiy Storchaka files: A Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst M Lib/tkinter/__init__.py M Lib/tkinter/test/test_tkinter/test_images.py diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 054034098880f..6ae9839055382 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -4068,8 +4068,6 @@ def __init__(self, imgtype, name=None, cnf={}, master=None, **kw): elif kw: cnf = kw options = () for k, v in cnf.items(): - if callable(v): - v = self._register(v) options = options + ('-'+k, v) self.tk.call(('image', 'create', imgtype, name,) + options) self.name = name @@ -4096,8 +4094,6 @@ def configure(self, **kw): for k, v in _cnfmerge(kw).items(): if v is not None: if k[-1] == '_': k = k[:-1] - if callable(v): - v = self._register(v) res = res + ('-'+k, v) self.tk.call((self.name, 'config') + res) diff --git a/Lib/tkinter/test/test_tkinter/test_images.py b/Lib/tkinter/test/test_tkinter/test_images.py index 8b473cce5f994..5b3566a3611eb 100644 --- a/Lib/tkinter/test/test_tkinter/test_images.py +++ b/Lib/tkinter/test/test_tkinter/test_images.py @@ -144,6 +144,14 @@ def test_configure_foreground(self): self.assertEqual(image['foreground'], '-foreground {} {} #000000 yellow') + def test_bug_100814(self): + # gh-100814: Passing a callable option value causes AttributeError. + with self.assertRaises(tkinter.TclError): + tkinter.BitmapImage('::img::test', master=self.root, spam=print) + image = tkinter.BitmapImage('::img::test', master=self.root) + with self.assertRaises(tkinter.TclError): + image.configure(spam=print) + class PhotoImageTest(AbstractTkTest, unittest.TestCase): @@ -274,6 +282,14 @@ def test_configure_palette(self): image.configure(palette='3/4/2') self.assertEqual(image['palette'], '3/4/2') + def test_bug_100814(self): + # gh-100814: Passing a callable option value causes AttributeError. + with self.assertRaises(tkinter.TclError): + tkinter.PhotoImage('::img::test', master=self.root, spam=print) + image = tkinter.PhotoImage('::img::test', master=self.root) + with self.assertRaises(tkinter.TclError): + image.configure(spam=print) + def test_blank(self): image = self.create() image.blank() diff --git a/Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst b/Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst new file mode 100644 index 0000000000000..86cb7bf79f307 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst @@ -0,0 +1,2 @@ +Passing a callable object as an option value to a Tkinter image now raises +the expected TclError instead of an AttributeError. From webhook-mailer at python.org Wed Aug 16 04:15:42 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 16 Aug 2023 08:15:42 -0000 Subject: [Python-checkins] [3.11] gh-91795: Update build optimization part of PCbuild/readme.txt (GH-91849) (GH-107777) Message-ID: https://github.com/python/cpython/commit/9e87e07aa422f95a42073b7ce750fbe19842f960 commit: 9e87e07aa422f95a42073b7ce750fbe19842f960 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: serhiy-storchaka date: 2023-08-16T11:15:38+03:00 summary: [3.11] gh-91795: Update build optimization part of PCbuild/readme.txt (GH-91849) (GH-107777) (cherry picked from commit 906b73be5eada1995bd667a02c59f7a11998310f) Co-authored-by: Fatih <77548106+fatihkabakk at users.noreply.github.com> files: M PCbuild/readme.txt diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index e9a0917227467..092b2cc8c1a7f 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -250,9 +250,11 @@ against a profiling library and contain extra debug information. The PGUpdate configuration takes the profiling data and generates optimized binaries. -The build_pgo.bat script automates the creation of optimized binaries. -It creates the PGI files, runs the unit test suite or PyBench with the -PGI python, and finally creates the optimized files. +The build.bat script has an argument `--pgo` that automate the creation +of optimized binaries. +It creates the PGI files, runs the unit test suite with the PGI python, +and finally creates the optimized files. +You can customize the job for profiling with `--pgo-job ` option. See https://docs.microsoft.com/en-us/cpp/build/profile-guided-optimizations From webhook-mailer at python.org Wed Aug 16 04:34:41 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 16 Aug 2023 08:34:41 -0000 Subject: [Python-checkins] [3.11] gh-107237: Fix test_udp_reconnection() of test_logging (GH-107238) (GH-107243) Message-ID: https://github.com/python/cpython/commit/4f35d4f6f258da5fed468cd5561c1171a3735f55 commit: 4f35d4f6f258da5fed468cd5561c1171a3735f55 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: serhiy-storchaka date: 2023-08-16T11:34:36+03:00 summary: [3.11] gh-107237: Fix test_udp_reconnection() of test_logging (GH-107238) (GH-107243) test_logging: Fix test_udp_reconnection() by increasing the timeout from 100 ms to 5 minutes (LONG_TIMEOUT). Replace also blocking wait() with wait(LONG_TIMEOUT) in test_output() to prevent the test to hang. (cherry picked from commit ed082383272c2c238e364e9cc83229234aee23cc) Co-authored-by: Victor Stinner files: From webhook-mailer at python.org Wed Aug 16 04:36:39 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 16 Aug 2023 08:36:39 -0000 Subject: [Python-checkins] [3.11] gh-100061: Proper fix of the bug in the matching of possessive quantifiers (GH-102612) (GH-108004) Message-ID: https://github.com/python/cpython/commit/26137e2cf70b31e93b8bb26287647915bcaa5c99 commit: 26137e2cf70b31e93b8bb26287647915bcaa5c99 branch: 3.11 author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-16T08:36:36Z summary: [3.11] gh-100061: Proper fix of the bug in the matching of possessive quantifiers (GH-102612) (GH-108004) Restore the global Input Stream pointer after trying to match a sub-pattern. Co-authored-by: Ma Lin (cherry picked from commit abd9cc52d94b8e2835322b62c29f09bb0e6fcfe9) Co-authored-by: SKO <41810398+uyw4687 at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst M Lib/re/_compiler.py M Lib/test/test_re.py M Modules/_sre/sre_lib.h diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index e30740b9c30b0..d8e0d2fdefdcc 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -100,13 +100,6 @@ def _compile(code, pattern, flags): emit(ANY_ALL) else: emit(ANY) - elif op is POSSESSIVE_REPEAT: - # gh-106052: Possessive quantifiers do not work when the - # subpattern contains backtracking, i.e. "(?:ab?c)*+". - # Implement it as equivalent greedy qualifier in atomic group. - p = [(MAX_REPEAT, av)] - p = [(ATOMIC_GROUP, p)] - _compile(code, p, flags) elif op in REPEATING_CODES: if flags & SRE_FLAG_TEMPLATE: raise error("internal: unsupported template operator %r" % (op,)) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index d2736dae20b1b..f4d64dc9fcf2a 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2397,6 +2397,16 @@ def test_template_function_and_flag_is_deprecated(self): self.assertFalse(template_re1.match('nope')) def test_bug_gh106052(self): + # gh-100061 + self.assertEqual(re.match('(?>(?:.(?!D))+)', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?:.(?!D))++', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?>(?:.(?!D))*)', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?:.(?!D))*+', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?>(?:.(?!D))?)', 'CDE').span(), (0, 0)) + self.assertEqual(re.match('(?:.(?!D))?+', 'CDE').span(), (0, 0)) + self.assertEqual(re.match('(?>(?:.(?!D)){1,3})', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?:.(?!D)){1,3}+', 'ABCDE').span(), (0, 2)) + # gh-106052 self.assertEqual(re.match("(?>(?:ab?c)+)", "aca").span(), (0, 2)) self.assertEqual(re.match("(?:ab?c)++", "aca").span(), (0, 2)) self.assertEqual(re.match("(?>(?:ab?c)*)", "aca").span(), (0, 2)) @@ -2502,7 +2512,6 @@ def test_atomic_group(self): 17: SUCCESS ''') - @unittest.expectedFailure # gh-106052 def test_possesive_repeat_one(self): self.assertEqual(get_debug_out(r'a?+'), '''\ POSSESSIVE_REPEAT 0 1 @@ -2515,7 +2524,6 @@ def test_possesive_repeat_one(self): 12: SUCCESS ''') - @unittest.expectedFailure # gh-106052 def test_possesive_repeat(self): self.assertEqual(get_debug_out(r'(?:ab)?+'), '''\ POSSESSIVE_REPEAT 0 1 diff --git a/Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst b/Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst new file mode 100644 index 0000000000000..dfed34f6ae976 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst @@ -0,0 +1,2 @@ +Fix a bug that causes wrong matches for regular expressions with possessive +qualifier. diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index fb4c18b63d643..e83149825e2cd 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -1334,6 +1334,10 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) MARK_POP(ctx->lastmark); LASTMARK_RESTORE(); + /* Restore the global Input Stream pointer + since it can change after jumps. */ + state->ptr = ptr; + /* We have sufficient matches, so exit loop. */ break; } From webhook-mailer at python.org Wed Aug 16 05:14:18 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Wed, 16 Aug 2023 09:14:18 -0000 Subject: [Python-checkins] gh-107955 Remove old comment about increasing the reference count in usage of Py_None (#107993) Message-ID: https://github.com/python/cpython/commit/f6099871fac0581ed4d9bcd9b15ce136fb0de8d6 commit: f6099871fac0581ed4d9bcd9b15ce136fb0de8d6 branch: main author: brandonardenwalli <142186236+brandonardenwalli at users.noreply.github.com> committer: kumaraditya303 date: 2023-08-16T09:14:14Z summary: gh-107955 Remove old comment about increasing the reference count in usage of Py_None (#107993) files: M Include/object.h diff --git a/Include/object.h b/Include/object.h index e26cedf8ca3c9..be9a0cedb7205 100644 --- a/Include/object.h +++ b/Include/object.h @@ -845,8 +845,6 @@ static inline PyObject* _Py_XNewRef(PyObject *obj) /* _Py_NoneStruct is an object of undefined type which can be used in contexts where NULL (nil) is not suitable (since NULL often means 'error'). - -Don't forget to apply Py_INCREF() when returning this value!!! */ PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */ #define Py_None (&_Py_NoneStruct) From webhook-mailer at python.org Wed Aug 16 05:15:05 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 16 Aug 2023 09:15:05 -0000 Subject: [Python-checkins] [3.11] bpo-18319: gettext() can retrieve a message even if a plural form exists (GH-19869) (GH-107107) Message-ID: https://github.com/python/cpython/commit/c1c3f0bfec9de0fab35027cdec627f18e6c05c86 commit: c1c3f0bfec9de0fab35027cdec627f18e6c05c86 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: serhiy-storchaka date: 2023-08-16T09:15:01Z summary: [3.11] bpo-18319: gettext() can retrieve a message even if a plural form exists (GH-19869) (GH-107107) (cherry picked from commit 54632528eeba841e4a8cc95ecbd84c9aca8eef57) Co-authored-by: Gilles Bassi?re files: A Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst M Lib/gettext.py M Lib/test/test_gettext.py diff --git a/Lib/gettext.py b/Lib/gettext.py index 57f1a449c28a4..b72b15f82d435 100644 --- a/Lib/gettext.py +++ b/Lib/gettext.py @@ -422,10 +422,12 @@ def gettext(self, message): missing = object() tmsg = self._catalog.get(message, missing) if tmsg is missing: - if self._fallback: - return self._fallback.gettext(message) - return message - return tmsg + tmsg = self._catalog.get((message, self.plural(1)), missing) + if tmsg is not missing: + return tmsg + if self._fallback: + return self._fallback.gettext(message) + return message def ngettext(self, msgid1, msgid2, n): try: diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index 7f7b51c19f35c..8430fc234d00e 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -320,6 +320,8 @@ def test_plural_forms1(self): eq(x, 'Hay %s fichero') x = gettext.ngettext('There is %s file', 'There are %s files', 2) eq(x, 'Hay %s ficheros') + x = gettext.gettext('There is %s file') + eq(x, 'Hay %s fichero') def test_plural_context_forms1(self): eq = self.assertEqual @@ -340,6 +342,8 @@ def test_plural_forms2(self): eq(x, 'Hay %s fichero') x = t.ngettext('There is %s file', 'There are %s files', 2) eq(x, 'Hay %s ficheros') + x = t.gettext('There is %s file') + eq(x, 'Hay %s fichero') def test_plural_context_forms2(self): eq = self.assertEqual diff --git a/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst b/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst new file mode 100644 index 0000000000000..94d7cc9deadbb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst @@ -0,0 +1,2 @@ +Ensure ``gettext(msg)`` retrieve translations even if a plural form exists. In +other words: ``gettext(msg) == ngettext(msg, '', 1)``. From webhook-mailer at python.org Wed Aug 16 05:20:46 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 16 Aug 2023 09:20:46 -0000 Subject: [Python-checkins] gh-106300: Improve errors testing in test_unittest.test_runner (GH-106737) Message-ID: https://github.com/python/cpython/commit/fd9d70a94de5b0756b52b9ae21e236e25545db4f commit: fd9d70a94de5b0756b52b9ae21e236e25545db4f branch: main author: Nikita Sobolev committer: serhiy-storchaka date: 2023-08-16T12:20:42+03:00 summary: gh-106300: Improve errors testing in test_unittest.test_runner (GH-106737) Use a custom exception to prevent unintentional silence of actual errors. files: M Lib/test/test_unittest/test_runner.py diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index f3b2c0cffd451..1b9cef43e3f9c 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -24,6 +24,13 @@ def getRunner(): stream=io.StringIO()) +class CustomError(Exception): + pass + +# For test output compat: +CustomErrorRepr = f"{__name__ + '.' if __name__ != '__main__' else ''}CustomError" + + def runTests(*cases): suite = unittest.TestSuite() for case in cases: @@ -46,7 +53,7 @@ def cleanup(ordering, blowUp=False): ordering.append('cleanup_good') else: ordering.append('cleanup_exc') - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class TestCM: @@ -108,8 +115,8 @@ def testNothing(self): result = unittest.TestResult() outcome = test._outcome = _Outcome(result=result) - CleanUpExc = Exception('foo') - exc2 = Exception('bar') + CleanUpExc = CustomError('foo') + exc2 = CustomError('bar') def cleanup1(): raise CleanUpExc @@ -125,10 +132,10 @@ def cleanup2(): (_, msg2), (_, msg1) = result.errors self.assertIn('in cleanup1', msg1) self.assertIn('raise CleanUpExc', msg1) - self.assertIn('Exception: foo', msg1) + self.assertIn(f'{CustomErrorRepr}: foo', msg1) self.assertIn('in cleanup2', msg2) self.assertIn('raise exc2', msg2) - self.assertIn('Exception: bar', msg2) + self.assertIn(f'{CustomErrorRepr}: bar', msg2) def testCleanupInRun(self): blowUp = False @@ -139,7 +146,7 @@ def setUp(self): ordering.append('setUp') test.addCleanup(cleanup2) if blowUp: - raise Exception('foo') + raise CustomError('foo') def testNothing(self): ordering.append('test') @@ -280,7 +287,7 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() def testNothing(self): ordering.append('test') @classmethod @@ -306,7 +313,7 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() def testNothing(self): ordering.append('test') @classmethod @@ -346,7 +353,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'CleanUpExc') self.assertEqual(ordering, @@ -366,10 +373,10 @@ def testNothing(self): @classmethod def tearDownClass(cls): ordering.append('tearDownClass') - raise Exception('TearDownClassExc') + raise CustomError('TearDownClassExc') suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) @@ -379,7 +386,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) @@ -392,16 +399,22 @@ def testNothing(self): pass def cleanup1(): - raise Exception('cleanup1') + raise CustomError('cleanup1') def cleanup2(): - raise Exception('cleanup2') + raise CustomError('cleanup2') TestableTest.addClassCleanup(cleanup1) TestableTest.addClassCleanup(cleanup2) - with self.assertRaises(Exception) as e: - TestableTest.doClassCleanups() - self.assertEqual(e, 'cleanup1') + TestableTest.doClassCleanups() + + self.assertEqual(len(TestableTest.tearDown_exceptions), 2) + + e1, e2 = TestableTest.tearDown_exceptions + self.assertIsInstance(e1[1], CustomError) + self.assertEqual(str(e1[1]), 'cleanup2') + self.assertIsInstance(e2[1], CustomError) + self.assertEqual(str(e2[1]), 'cleanup1') def test_with_errors_addCleanUp(self): ordering = [] @@ -421,7 +434,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'cleanup_exc', 'tearDownClass', 'cleanup_good']) @@ -444,7 +457,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'cleanup_good', 'tearDownClass', 'cleanup_exc']) @@ -460,11 +473,11 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering, blowUp=True) if class_blow_up: - raise Exception('ClassExc') + raise CustomError('ClassExc') def setUp(self): ordering.append('setUp') if method_blow_up: - raise Exception('MethodExc') + raise CustomError('MethodExc') def testNothing(self): ordering.append('test') @classmethod @@ -473,7 +486,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'tearDownClass', 'cleanup_exc']) @@ -483,9 +496,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ClassExc') + f'{CustomErrorRepr}: ClassExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'cleanup_exc']) @@ -494,9 +507,9 @@ def tearDownClass(cls): method_blow_up = True result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: MethodExc') + f'{CustomErrorRepr}: MethodExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'tearDownClass', 'cleanup_exc']) @@ -513,11 +526,11 @@ def testNothing(self): @classmethod def tearDownClass(cls): ordering.append('tearDownClass') - raise Exception('TearDownExc') + raise CustomError('TearDownExc') result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: TearDownExc') + f'{CustomErrorRepr}: TearDownExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass', 'cleanup_good']) @@ -620,7 +633,7 @@ def module_cleanup_good(*args, **kwargs): module_cleanups.append((3, args, kwargs)) def module_cleanup_bad(*args, **kwargs): - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class Module(object): unittest.addModuleCleanup(module_cleanup_good, 1, 2, 3, @@ -630,7 +643,7 @@ class Module(object): [(module_cleanup_good, (1, 2, 3), dict(four='hello', five='goodbye')), (module_cleanup_bad, (), {})]) - with self.assertRaises(Exception) as e: + with self.assertRaises(CustomError) as e: unittest.case.doModuleCleanups() self.assertEqual(str(e.exception), 'CleanUpExc') self.assertEqual(unittest.case._module_cleanups, []) @@ -659,7 +672,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) if blowUp: - raise Exception('setUpModule Exc') + raise CustomError('setUpModule Exc') @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -679,7 +692,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(ordering, ['setUpModule', 'cleanup_good']) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: setUpModule Exc') + f'{CustomErrorRepr}: setUpModule Exc') ordering = [] blowUp = False @@ -699,7 +712,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -710,7 +723,7 @@ def setUpModule(): ordering.append('setUpModule2') unittest.addModuleCleanup(cleanup, ordering) if blowUp2: - raise Exception() + raise CustomError() @staticmethod def tearDownModule(): ordering.append('tearDownModule2') @@ -799,7 +812,7 @@ def setUpModule(): @staticmethod def tearDownModule(): ordering.append('tearDownModule') - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class TestableTest(unittest.TestCase): @classmethod @@ -815,7 +828,7 @@ def tearDownClass(cls): sys.modules['Module'] = Module result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'tearDownModule', 'cleanup_good']) @@ -855,7 +868,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -873,7 +886,7 @@ def setUpModule(): @staticmethod def tearDownModule(): ordering.append('tearDownModule') - raise Exception('TearDownModuleExc') + raise CustomError('TearDownModuleExc') class TestableTest(unittest.TestCase): @classmethod @@ -888,7 +901,7 @@ def tearDownClass(cls): TestableTest.__module__ = 'Module' sys.modules['Module'] = Module suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -899,7 +912,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -978,7 +991,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) @@ -1008,7 +1021,7 @@ def tearDown(self): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUp', 'test', 'tearDown', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) @@ -1024,7 +1037,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering, blowUp=True) if module_blow_up: - raise Exception('ModuleExc') + raise CustomError('ModuleExc') @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -1034,11 +1047,11 @@ class TestableTest(unittest.TestCase): def setUpClass(cls): ordering.append('setUpClass') if class_blow_up: - raise Exception('ClassExc') + raise CustomError('ClassExc') def setUp(self): ordering.append('setUp') if method_blow_up: - raise Exception('MethodExc') + raise CustomError('MethodExc') def testNothing(self): ordering.append('test') @classmethod @@ -1050,7 +1063,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'setUp', 'test', 'tearDownClass', 'tearDownModule', @@ -1062,9 +1075,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ModuleExc') + f'{CustomErrorRepr}: ModuleExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'cleanup_exc']) ordering = [] @@ -1073,9 +1086,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ClassExc') + f'{CustomErrorRepr}: ClassExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'tearDownModule', 'cleanup_exc']) @@ -1085,9 +1098,9 @@ def tearDownClass(cls): method_blow_up = True result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: MethodExc') + f'{CustomErrorRepr}: MethodExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'setUp', 'tearDownClass', 'tearDownModule', 'cleanup_exc']) From webhook-mailer at python.org Wed Aug 16 05:59:06 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 09:59:06 -0000 Subject: [Python-checkins] [3.12] gh-91051: fix segfault when using all 8 type watchers (GH-107853) (#107876) Message-ID: https://github.com/python/cpython/commit/00bfed7cba3f2d66a1534f9f088ddf68abbf0d3f commit: 00bfed7cba3f2d66a1534f9f088ddf68abbf0d3f branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-16T11:58:54+02:00 summary: [3.12] gh-91051: fix segfault when using all 8 type watchers (GH-107853) (#107876) * gh-91051: fix segfault when using all 8 type watchers (GH-107853) (cherry picked from commit 66e4edd7346b1cd65ddff6da890a0d725e325116) Co-authored-by: Carl Meyer files: A Misc/NEWS.d/next/Core and Builtins/2023-08-10-17-36-27.gh-issue-91051.LfaeNW.rst M Doc/c-api/typeobj.rst M Doc/data/python3.12.abi M Doc/includes/typestruct.h M Include/cpython/object.h M Lib/test/test_capi/test_watchers.py diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 221a05b192240..faa183e27fcfa 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -147,7 +147,7 @@ Quick Reference +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_vectorcall` | :c:type:`vectorcallfunc` | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_watched`] | char | | | | | | + | [:c:member:`~PyTypeObject.tp_watched`] | unsigned char | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ .. [#slots] @@ -2141,7 +2141,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. versionadded:: 3.9 (the field exists since 3.8 but it's only used since 3.9) -.. c:member:: char PyTypeObject.tp_watched +.. c:member:: unsigned char PyTypeObject.tp_watched Internal. Do not use. diff --git a/Doc/data/python3.12.abi b/Doc/data/python3.12.abi index 5eec83faf1679..cdefda782b9c6 100644 --- a/Doc/data/python3.12.abi +++ b/Doc/data/python3.12.abi @@ -1,26464 +1,26457 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h index f0ad1e47cb0d8..ec939c28831c3 100644 --- a/Doc/includes/typestruct.h +++ b/Doc/includes/typestruct.h @@ -82,5 +82,5 @@ typedef struct _typeobject { vectorcallfunc tp_vectorcall; /* bitset of which type-watchers care about this type */ - char tp_watched; + unsigned char tp_watched; } PyTypeObject; diff --git a/Include/cpython/object.h b/Include/cpython/object.h index c5d0851a4b11b..ae7f780a93182 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -227,7 +227,7 @@ struct _typeobject { vectorcallfunc tp_vectorcall; /* bitset of which type-watchers care about this type */ - char tp_watched; + unsigned char tp_watched; }; /* This struct is used by the specializer diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 93f6ef752d066..10b76e163bfb2 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -294,6 +294,18 @@ class C2: pass C2.hmm = "baz" self.assert_events([C1, [C2]]) + def test_all_watchers(self): + class C: pass + with ExitStack() as stack: + last_wid = -1 + # don't make assumptions about how many watchers are already + # registered, just go until we reach the max ID + while last_wid < self.TYPE_MAX_WATCHERS - 1: + last_wid = stack.enter_context(self.watcher()) + self.watch(last_wid, C) + C.foo = "bar" + self.assert_events([C]) + def test_watch_non_type(self): with self.watcher() as wid: with self.assertRaisesRegex(ValueError, r"Cannot watch non-type"): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-10-17-36-27.gh-issue-91051.LfaeNW.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-10-17-36-27.gh-issue-91051.LfaeNW.rst new file mode 100644 index 0000000000000..b4b90ad4ea0ec --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-10-17-36-27.gh-issue-91051.LfaeNW.rst @@ -0,0 +1,2 @@ +Fix abort / segfault when using all eight type watcher slots, on platforms +where ``char`` is signed by default. From webhook-mailer at python.org Wed Aug 16 05:59:48 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 16 Aug 2023 09:59:48 -0000 Subject: [Python-checkins] [3.11] gh-106300: Improve errors testing in test_unittest.test_runner (GH-106737) (GH-108007) Message-ID: https://github.com/python/cpython/commit/6b825c1b8a14460641ca6f1647d83005c68199aa commit: 6b825c1b8a14460641ca6f1647d83005c68199aa branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: serhiy-storchaka date: 2023-08-16T09:59:44Z summary: [3.11] gh-106300: Improve errors testing in test_unittest.test_runner (GH-106737) (GH-108007) Use a custom exception to prevent unintentional silence of actual errors. (cherry picked from commit fd9d70a94de5b0756b52b9ae21e236e25545db4f) Co-authored-by: Nikita Sobolev files: M Lib/unittest/test/test_runner.py diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py index e02b2dadc8fdb..3aefc1e0a5fdd 100644 --- a/Lib/unittest/test/test_runner.py +++ b/Lib/unittest/test/test_runner.py @@ -21,6 +21,13 @@ def getRunner(): stream=io.StringIO()) +class CustomError(Exception): + pass + +# For test output compat: +CustomErrorRepr = f"{__name__ + '.' if __name__ != '__main__' else ''}CustomError" + + def runTests(*cases): suite = unittest.TestSuite() for case in cases: @@ -43,7 +50,7 @@ def cleanup(ordering, blowUp=False): ordering.append('cleanup_good') else: ordering.append('cleanup_exc') - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class TestCM: @@ -105,8 +112,8 @@ def testNothing(self): result = unittest.TestResult() outcome = test._outcome = _Outcome(result=result) - CleanUpExc = Exception('foo') - exc2 = Exception('bar') + CleanUpExc = CustomError('foo') + exc2 = CustomError('bar') def cleanup1(): raise CleanUpExc @@ -122,10 +129,10 @@ def cleanup2(): (_, msg2), (_, msg1) = result.errors self.assertIn('in cleanup1', msg1) self.assertIn('raise CleanUpExc', msg1) - self.assertIn('Exception: foo', msg1) + self.assertIn(f'{CustomErrorRepr}: foo', msg1) self.assertIn('in cleanup2', msg2) self.assertIn('raise exc2', msg2) - self.assertIn('Exception: bar', msg2) + self.assertIn(f'{CustomErrorRepr}: bar', msg2) def testCleanupInRun(self): blowUp = False @@ -136,7 +143,7 @@ def setUp(self): ordering.append('setUp') test.addCleanup(cleanup2) if blowUp: - raise Exception('foo') + raise CustomError('foo') def testNothing(self): ordering.append('test') @@ -277,7 +284,7 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() def testNothing(self): ordering.append('test') @classmethod @@ -303,7 +310,7 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() def testNothing(self): ordering.append('test') @classmethod @@ -343,7 +350,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'CleanUpExc') self.assertEqual(ordering, @@ -363,10 +370,10 @@ def testNothing(self): @classmethod def tearDownClass(cls): ordering.append('tearDownClass') - raise Exception('TearDownClassExc') + raise CustomError('TearDownClassExc') suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) @@ -376,7 +383,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) @@ -389,16 +396,22 @@ def testNothing(self): pass def cleanup1(): - raise Exception('cleanup1') + raise CustomError('cleanup1') def cleanup2(): - raise Exception('cleanup2') + raise CustomError('cleanup2') TestableTest.addClassCleanup(cleanup1) TestableTest.addClassCleanup(cleanup2) - with self.assertRaises(Exception) as e: - TestableTest.doClassCleanups() - self.assertEqual(e, 'cleanup1') + TestableTest.doClassCleanups() + + self.assertEqual(len(TestableTest.tearDown_exceptions), 2) + + e1, e2 = TestableTest.tearDown_exceptions + self.assertIsInstance(e1[1], CustomError) + self.assertEqual(str(e1[1]), 'cleanup2') + self.assertIsInstance(e2[1], CustomError) + self.assertEqual(str(e2[1]), 'cleanup1') def test_with_errors_addCleanUp(self): ordering = [] @@ -418,7 +431,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'cleanup_exc', 'tearDownClass', 'cleanup_good']) @@ -441,7 +454,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'cleanup_good', 'tearDownClass', 'cleanup_exc']) @@ -457,11 +470,11 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering, blowUp=True) if class_blow_up: - raise Exception('ClassExc') + raise CustomError('ClassExc') def setUp(self): ordering.append('setUp') if method_blow_up: - raise Exception('MethodExc') + raise CustomError('MethodExc') def testNothing(self): ordering.append('test') @classmethod @@ -470,7 +483,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'tearDownClass', 'cleanup_exc']) @@ -480,9 +493,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ClassExc') + f'{CustomErrorRepr}: ClassExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'cleanup_exc']) @@ -491,9 +504,9 @@ def tearDownClass(cls): method_blow_up = True result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: MethodExc') + f'{CustomErrorRepr}: MethodExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'tearDownClass', 'cleanup_exc']) @@ -510,11 +523,11 @@ def testNothing(self): @classmethod def tearDownClass(cls): ordering.append('tearDownClass') - raise Exception('TearDownExc') + raise CustomError('TearDownExc') result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: TearDownExc') + f'{CustomErrorRepr}: TearDownExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass', 'cleanup_good']) @@ -607,7 +620,7 @@ def module_cleanup_good(*args, **kwargs): module_cleanups.append((3, args, kwargs)) def module_cleanup_bad(*args, **kwargs): - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class Module(object): unittest.addModuleCleanup(module_cleanup_good, 1, 2, 3, @@ -617,7 +630,7 @@ class Module(object): [(module_cleanup_good, (1, 2, 3), dict(four='hello', five='goodbye')), (module_cleanup_bad, (), {})]) - with self.assertRaises(Exception) as e: + with self.assertRaises(CustomError) as e: unittest.case.doModuleCleanups() self.assertEqual(str(e.exception), 'CleanUpExc') self.assertEqual(unittest.case._module_cleanups, []) @@ -646,7 +659,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) if blowUp: - raise Exception('setUpModule Exc') + raise CustomError('setUpModule Exc') @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -666,7 +679,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(ordering, ['setUpModule', 'cleanup_good']) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: setUpModule Exc') + f'{CustomErrorRepr}: setUpModule Exc') ordering = [] blowUp = False @@ -686,7 +699,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -697,7 +710,7 @@ def setUpModule(): ordering.append('setUpModule2') unittest.addModuleCleanup(cleanup, ordering) if blowUp2: - raise Exception() + raise CustomError() @staticmethod def tearDownModule(): ordering.append('tearDownModule2') @@ -786,7 +799,7 @@ def setUpModule(): @staticmethod def tearDownModule(): ordering.append('tearDownModule') - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class TestableTest(unittest.TestCase): @classmethod @@ -802,7 +815,7 @@ def tearDownClass(cls): sys.modules['Module'] = Module result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'tearDownModule', 'cleanup_good']) @@ -842,7 +855,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -860,7 +873,7 @@ def setUpModule(): @staticmethod def tearDownModule(): ordering.append('tearDownModule') - raise Exception('TearDownModuleExc') + raise CustomError('TearDownModuleExc') class TestableTest(unittest.TestCase): @classmethod @@ -875,7 +888,7 @@ def tearDownClass(cls): TestableTest.__module__ = 'Module' sys.modules['Module'] = Module suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -886,7 +899,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -965,7 +978,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) @@ -995,7 +1008,7 @@ def tearDown(self): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUp', 'test', 'tearDown', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) @@ -1011,7 +1024,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering, blowUp=True) if module_blow_up: - raise Exception('ModuleExc') + raise CustomError('ModuleExc') @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -1021,11 +1034,11 @@ class TestableTest(unittest.TestCase): def setUpClass(cls): ordering.append('setUpClass') if class_blow_up: - raise Exception('ClassExc') + raise CustomError('ClassExc') def setUp(self): ordering.append('setUp') if method_blow_up: - raise Exception('MethodExc') + raise CustomError('MethodExc') def testNothing(self): ordering.append('test') @classmethod @@ -1037,7 +1050,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'setUp', 'test', 'tearDownClass', 'tearDownModule', @@ -1049,9 +1062,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ModuleExc') + f'{CustomErrorRepr}: ModuleExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'cleanup_exc']) ordering = [] @@ -1060,9 +1073,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ClassExc') + f'{CustomErrorRepr}: ClassExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'tearDownModule', 'cleanup_exc']) @@ -1072,9 +1085,9 @@ def tearDownClass(cls): method_blow_up = True result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: MethodExc') + f'{CustomErrorRepr}: MethodExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'setUp', 'tearDownClass', 'tearDownModule', 'cleanup_exc']) From webhook-mailer at python.org Wed Aug 16 06:00:59 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 10:00:59 -0000 Subject: [Python-checkins] [3.12] gh-100061: Proper fix of the bug in the matching of possessive quantifiers (GH-102612) (#108003) Message-ID: https://github.com/python/cpython/commit/bd2ef82a5010985abdeef2ca71bcbcc9a366993b commit: bd2ef82a5010985abdeef2ca71bcbcc9a366993b branch: 3.12 author: Serhiy Storchaka committer: Yhg1s date: 2023-08-16T12:00:55+02:00 summary: [3.12] gh-100061: Proper fix of the bug in the matching of possessive quantifiers (GH-102612) (#108003) Restore the global Input Stream pointer after trying to match a sub-pattern. . (cherry picked from commit abd9cc52d94b8e2835322b62c29f09bb0e6fcfe9) Co-authored-by: SKO <41810398+uyw4687 at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst M Lib/re/_compiler.py M Lib/test/test_re.py M Modules/_sre/sre_lib.h diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index e30740b9c30b0..d8e0d2fdefdcc 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -100,13 +100,6 @@ def _compile(code, pattern, flags): emit(ANY_ALL) else: emit(ANY) - elif op is POSSESSIVE_REPEAT: - # gh-106052: Possessive quantifiers do not work when the - # subpattern contains backtracking, i.e. "(?:ab?c)*+". - # Implement it as equivalent greedy qualifier in atomic group. - p = [(MAX_REPEAT, av)] - p = [(ATOMIC_GROUP, p)] - _compile(code, p, flags) elif op in REPEATING_CODES: if flags & SRE_FLAG_TEMPLATE: raise error("internal: unsupported template operator %r" % (op,)) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 85541f4451d03..5a5de523eba05 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2366,6 +2366,16 @@ def test_template_function_and_flag_is_deprecated(self): self.assertFalse(template_re1.match('nope')) def test_bug_gh106052(self): + # gh-100061 + self.assertEqual(re.match('(?>(?:.(?!D))+)', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?:.(?!D))++', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?>(?:.(?!D))*)', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?:.(?!D))*+', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?>(?:.(?!D))?)', 'CDE').span(), (0, 0)) + self.assertEqual(re.match('(?:.(?!D))?+', 'CDE').span(), (0, 0)) + self.assertEqual(re.match('(?>(?:.(?!D)){1,3})', 'ABCDE').span(), (0, 2)) + self.assertEqual(re.match('(?:.(?!D)){1,3}+', 'ABCDE').span(), (0, 2)) + # gh-106052 self.assertEqual(re.match("(?>(?:ab?c)+)", "aca").span(), (0, 2)) self.assertEqual(re.match("(?:ab?c)++", "aca").span(), (0, 2)) self.assertEqual(re.match("(?>(?:ab?c)*)", "aca").span(), (0, 2)) @@ -2471,7 +2481,6 @@ def test_atomic_group(self): 17: SUCCESS ''') - @unittest.expectedFailure # gh-106052 def test_possesive_repeat_one(self): self.assertEqual(get_debug_out(r'a?+'), '''\ POSSESSIVE_REPEAT 0 1 @@ -2484,7 +2493,6 @@ def test_possesive_repeat_one(self): 12: SUCCESS ''') - @unittest.expectedFailure # gh-106052 def test_possesive_repeat(self): self.assertEqual(get_debug_out(r'(?:ab)?+'), '''\ POSSESSIVE_REPEAT 0 1 diff --git a/Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst b/Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst new file mode 100644 index 0000000000000..dfed34f6ae976 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-14-01-19-57.gh-issue-100061.CiXJYn.rst @@ -0,0 +1,2 @@ +Fix a bug that causes wrong matches for regular expressions with possessive +qualifier. diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index fb4c18b63d643..e83149825e2cd 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -1334,6 +1334,10 @@ SRE(match)(SRE_STATE* state, const SRE_CODE* pattern, int toplevel) MARK_POP(ctx->lastmark); LASTMARK_RESTORE(); + /* Restore the global Input Stream pointer + since it can change after jumps. */ + state->ptr = ptr; + /* We have sufficient matches, so exit loop. */ break; } From webhook-mailer at python.org Wed Aug 16 06:03:09 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 10:03:09 -0000 Subject: [Python-checkins] [3.12] gh-107080: Fix Py_TRACE_REFS Crashes Under Isolated Subinterpreters (#107751) Message-ID: https://github.com/python/cpython/commit/aa9707dda9f8dcbe2ada8d8a7280f0f6a8229c59 commit: aa9707dda9f8dcbe2ada8d8a7280f0f6a8229c59 branch: 3.12 author: Eric Snow committer: Yhg1s date: 2023-08-16T12:03:05+02:00 summary: [3.12] gh-107080: Fix Py_TRACE_REFS Crashes Under Isolated Subinterpreters (#107751) * Unrevert "[3.12] gh-107080: Fix Py_TRACE_REFS Crashes Under Isolated Subinterpreters (gh-107567) (#107599)". This reverts commit 6e4eec760648a71e1cd8f8f551997b1823b4bb9f (gh-107648). * Initialize each interpreter's refchain properly. * Skip test_basic_multiple_interpreters_deleted_no_reset on tracerefs builds. files: A Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst M Include/internal/pycore_object.h M Include/internal/pycore_object_state.h M Include/internal/pycore_runtime_init.h M Lib/test/test_import/__init__.py M Objects/object.c M Python/pylifecycle.c M Python/pystate.c diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 0981d1122fec5..7a2f13a21bda7 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -152,6 +152,7 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { extern void _PyType_InitCache(PyInterpreterState *interp); +extern void _PyObject_InitState(PyInterpreterState *interp); /* Inline functions trading binary compatibility for speed: _PyObject_Init() is the fast version of PyObject_Init(), and @@ -271,8 +272,8 @@ extern void _PyDebug_PrintTotalRefs(void); #ifdef Py_TRACE_REFS extern void _Py_AddToAllObjects(PyObject *op, int force); -extern void _Py_PrintReferences(FILE *); -extern void _Py_PrintReferenceAddresses(FILE *); +extern void _Py_PrintReferences(PyInterpreterState *, FILE *); +extern void _Py_PrintReferenceAddresses(PyInterpreterState *, FILE *); #endif diff --git a/Include/internal/pycore_object_state.h b/Include/internal/pycore_object_state.h index 94005d7788143..65feb5af969f8 100644 --- a/Include/internal/pycore_object_state.h +++ b/Include/internal/pycore_object_state.h @@ -11,17 +11,22 @@ extern "C" { struct _py_object_runtime_state { #ifdef Py_REF_DEBUG Py_ssize_t interpreter_leaks; -#else - int _not_used; #endif + int _not_used; }; struct _py_object_state { #ifdef Py_REF_DEBUG Py_ssize_t reftotal; -#else - int _not_used; #endif +#ifdef Py_TRACE_REFS + /* Head of circular doubly-linked list of all objects. These are linked + * together via the _ob_prev and _ob_next members of a PyObject, which + * exist only in a Py_TRACE_REFS build. + */ + PyObject refchain; +#endif + int _not_used; }; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 4130188079cff..7aace9f86119c 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -101,6 +101,7 @@ extern PyTypeObject _PyExc_MemoryError; { .threshold = 10, }, \ }, \ }, \ + .object_state = _py_object_state_INIT(INTERP), \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ .dict_state = _dict_state_INIT, \ .func_state = { \ @@ -130,6 +131,16 @@ extern PyTypeObject _PyExc_MemoryError; .context_ver = 1, \ } +#ifdef Py_TRACE_REFS +# define _py_object_state_INIT(INTERP) \ + { \ + .refchain = {&INTERP.object_state.refchain, &INTERP.object_state.refchain}, \ + } +#else +# define _py_object_state_INIT(INTERP) \ + { 0 } +#endif + // global objects diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 06adf01f18c0a..f7df6d7e42319 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2539,6 +2539,12 @@ def test_basic_multiple_interpreters_main_no_reset(self): def test_basic_multiple_interpreters_deleted_no_reset(self): # without resetting; already loaded in a deleted interpreter + if hasattr(sys, 'getobjects'): + # It's a Py_TRACE_REFS build. + # This test breaks interpreter isolation a little, + # which causes problems on Py_TRACE_REF builds. + raise unittest.SkipTest('crashes on Py_TRACE_REFS builds') + # At this point: # * alive in 0 interpreters # * module def may or may not be loaded already diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst new file mode 100644 index 0000000000000..5084c854360e3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-02-12-24-51.gh-issue-107080.PNolFU.rst @@ -0,0 +1,4 @@ +Trace refs builds (``--with-trace-refs``) were crashing when used with +isolated subinterpreters. The problematic global state has been isolated to +each interpreter. Other fixing the crashes, this change does not affect +users. diff --git a/Objects/object.c b/Objects/object.c index bd0fa40bd2a85..8a4010fb13669 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -158,11 +158,16 @@ _PyDebug_PrintTotalRefs(void) { Do not call them otherwise, they do not initialize the object! */ #ifdef Py_TRACE_REFS -/* Head of circular doubly-linked list of all objects. These are linked - * together via the _ob_prev and _ob_next members of a PyObject, which - * exist only in a Py_TRACE_REFS build. - */ -static PyObject refchain = {&refchain, &refchain}; + +#define REFCHAIN(interp) &interp->object_state.refchain + +static inline void +init_refchain(PyInterpreterState *interp) +{ + PyObject *refchain = REFCHAIN(interp); + refchain->_ob_prev = refchain; + refchain->_ob_next = refchain; +} /* Insert op at the front of the list of all objects. If force is true, * op is added even if _ob_prev and _ob_next are non-NULL already. If @@ -187,10 +192,11 @@ _Py_AddToAllObjects(PyObject *op, int force) } #endif if (force || op->_ob_prev == NULL) { - op->_ob_next = refchain._ob_next; - op->_ob_prev = &refchain; - refchain._ob_next->_ob_prev = op; - refchain._ob_next = op; + PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); + op->_ob_next = refchain->_ob_next; + op->_ob_prev = refchain; + refchain->_ob_next->_ob_prev = op; + refchain->_ob_next = op; } } #endif /* Py_TRACE_REFS */ @@ -1998,6 +2004,18 @@ PyObject _Py_NotImplementedStruct = { &_PyNotImplemented_Type }; + +void +_PyObject_InitState(PyInterpreterState *interp) +{ +#ifdef Py_TRACE_REFS + if (!_Py_IsMainInterpreter(interp)) { + init_refchain(interp); + } +#endif +} + + extern PyTypeObject _Py_GenericAliasIterType; extern PyTypeObject _PyMemoryIter_Type; extern PyTypeObject _PyLineIterator; @@ -2206,7 +2224,8 @@ _Py_ForgetReference(PyObject *op) _PyObject_ASSERT_FAILED_MSG(op, "negative refcnt"); } - if (op == &refchain || + PyObject *refchain = REFCHAIN(_PyInterpreterState_GET()); + if (op == refchain || op->_ob_prev->_ob_next != op || op->_ob_next->_ob_prev != op) { _PyObject_ASSERT_FAILED_MSG(op, "invalid object chain"); @@ -2214,12 +2233,12 @@ _Py_ForgetReference(PyObject *op) #ifdef SLOW_UNREF_CHECK PyObject *p; - for (p = refchain._ob_next; p != &refchain; p = p->_ob_next) { + for (p = refchain->_ob_next; p != refchain; p = p->_ob_next) { if (p == op) { break; } } - if (p == &refchain) { + if (p == refchain) { /* Not found */ _PyObject_ASSERT_FAILED_MSG(op, "object not found in the objects list"); @@ -2235,11 +2254,15 @@ _Py_ForgetReference(PyObject *op) * interpreter must be in a healthy state. */ void -_Py_PrintReferences(FILE *fp) +_Py_PrintReferences(PyInterpreterState *interp, FILE *fp) { PyObject *op; + if (interp == NULL) { + interp = _PyInterpreterState_Main(); + } fprintf(fp, "Remaining objects:\n"); - for (op = refchain._ob_next; op != &refchain; op = op->_ob_next) { + PyObject *refchain = REFCHAIN(interp); + for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) { fprintf(fp, "%p [%zd] ", (void *)op, Py_REFCNT(op)); if (PyObject_Print(op, fp, 0) != 0) { PyErr_Clear(); @@ -2251,34 +2274,42 @@ _Py_PrintReferences(FILE *fp) /* Print the addresses of all live objects. Unlike _Py_PrintReferences, this * doesn't make any calls to the Python C API, so is always safe to call. */ +// XXX This function is not safe to use if the interpreter has been +// freed or is in an unhealthy state (e.g. late in finalization). +// The call in Py_FinalizeEx() is okay since the main interpreter +// is statically allocated. void -_Py_PrintReferenceAddresses(FILE *fp) +_Py_PrintReferenceAddresses(PyInterpreterState *interp, FILE *fp) { PyObject *op; + PyObject *refchain = REFCHAIN(interp); fprintf(fp, "Remaining object addresses:\n"); - for (op = refchain._ob_next; op != &refchain; op = op->_ob_next) + for (op = refchain->_ob_next; op != refchain; op = op->_ob_next) fprintf(fp, "%p [%zd] %s\n", (void *)op, Py_REFCNT(op), Py_TYPE(op)->tp_name); } +/* The implementation of sys.getobjects(). */ PyObject * _Py_GetObjects(PyObject *self, PyObject *args) { int i, n; PyObject *t = NULL; PyObject *res, *op; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (!PyArg_ParseTuple(args, "i|O", &n, &t)) return NULL; - op = refchain._ob_next; + PyObject *refchain = REFCHAIN(interp); + op = refchain->_ob_next; res = PyList_New(0); if (res == NULL) return NULL; - for (i = 0; (n == 0 || i < n) && op != &refchain; i++) { + for (i = 0; (n == 0 || i < n) && op != refchain; i++) { while (op == self || op == args || op == res || op == t || (t != NULL && !Py_IS_TYPE(op, (PyTypeObject *) t))) { op = op->_ob_next; - if (op == &refchain) + if (op == refchain) return res; } if (PyList_Append(res, op) < 0) { @@ -2290,7 +2321,9 @@ _Py_GetObjects(PyObject *self, PyObject *args) return res; } -#endif +#undef REFCHAIN + +#endif /* Py_TRACE_REFS */ /* Hack to force loading of abstract.o */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index a67fa26b37227..29771e07ae6a2 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1920,11 +1920,11 @@ Py_FinalizeEx(void) } if (dump_refs) { - _Py_PrintReferences(stderr); + _Py_PrintReferences(tstate->interp, stderr); } if (dump_refs_fp != NULL) { - _Py_PrintReferences(dump_refs_fp); + _Py_PrintReferences(tstate->interp, dump_refs_fp); } #endif /* Py_TRACE_REFS */ @@ -1960,11 +1960,11 @@ Py_FinalizeEx(void) */ if (dump_refs) { - _Py_PrintReferenceAddresses(stderr); + _Py_PrintReferenceAddresses(tstate->interp, stderr); } if (dump_refs_fp != NULL) { - _Py_PrintReferenceAddresses(dump_refs_fp); + _Py_PrintReferenceAddresses(tstate->interp, dump_refs_fp); fclose(dump_refs_fp); } #endif /* Py_TRACE_REFS */ @@ -2074,6 +2074,8 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) } has_gil = 1; + /* No objects have been created yet. */ + status = pycore_interp_init(tstate); if (_PyStatus_EXCEPTION(status)) { goto error; diff --git a/Python/pystate.c b/Python/pystate.c index 8097124965cb7..2ee16e3de25da 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -673,6 +673,7 @@ init_interpreter(PyInterpreterState *interp, _obmalloc_pools_INIT(interp->obmalloc.pools); memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp)); } + _PyObject_InitState(interp); _PyEval_InitState(interp, pending_lock); _PyGC_InitState(&interp->gc); From webhook-mailer at python.org Wed Aug 16 06:10:46 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 10:10:46 -0000 Subject: [Python-checkins] [3.12] gh-106844: Fix issues in _winapi.LCMapStringEx (GH-107832) (#107874) Message-ID: https://github.com/python/cpython/commit/b5176a86bd2587856f88aec3958ad8caef7b11fc commit: b5176a86bd2587856f88aec3958ad8caef7b11fc branch: 3.12 author: Serhiy Storchaka committer: Yhg1s date: 2023-08-16T12:10:42+02:00 summary: [3.12] gh-106844: Fix issues in _winapi.LCMapStringEx (GH-107832) (#107874) * Strings with length from 2**31-1 to 2**32-2 always caused MemoryError, it doesn't matter how much memory is available. * Strings with length exactly 2**32-1 caused OSError. * Strings longer than 2**32-1 characters were truncated due to integer overflow bug. * Strings containing the null character were truncated at the first null character. Now strings longer than 2**31-1 characters caused OverflowError and the null character is allowed.. (cherry picked from commit 04cc01453db2f0af72a06440831637f8bf512daf) files: A Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst M Lib/test/test_ntpath.py M Modules/_winapi.c M Modules/clinic/_winapi.c.h diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 538d758624c9d..78e1cb582512b 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1036,6 +1036,7 @@ def test_path_normcase(self): self._check_function(self.path.normcase) if sys.platform == 'win32': self.assertEqual(ntpath.normcase('\u03a9\u2126'), '??') + self.assertEqual(ntpath.normcase('abc\x00def'), 'abc\x00def') def test_path_isabs(self): self._check_function(self.path.isabs) diff --git a/Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst b/Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst new file mode 100644 index 0000000000000..1fdf162ef4ecd --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-07-18-13-01-26.gh-issue-106844.mci4xO.rst @@ -0,0 +1 @@ +Fix integer overflow and truncating by the null character in :func:`!_winapi.LCMapStringEx` which affects :func:`ntpath.normcase`. diff --git a/Modules/_winapi.c b/Modules/_winapi.c index d6d2f4a6a9b10..77275408aed86 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1538,40 +1538,56 @@ _winapi.LCMapStringEx locale: LPCWSTR flags: DWORD - src: LPCWSTR + src: unicode [clinic start generated code]*/ static PyObject * _winapi_LCMapStringEx_impl(PyObject *module, LPCWSTR locale, DWORD flags, - LPCWSTR src) -/*[clinic end generated code: output=cf4713d80e2b47c9 input=9fe26f95d5ab0001]*/ + PyObject *src) +/*[clinic end generated code: output=b90e6b26e028ff0a input=3e3dcd9b8164012f]*/ { if (flags & (LCMAP_SORTHANDLE | LCMAP_HASH | LCMAP_BYTEREV | LCMAP_SORTKEY)) { return PyErr_Format(PyExc_ValueError, "unsupported flags"); } - int dest_size = LCMapStringEx(locale, flags, src, -1, NULL, 0, + Py_ssize_t src_size; + wchar_t *src_ = PyUnicode_AsWideCharString(src, &src_size); + if (!src_) { + return NULL; + } + if (src_size > INT_MAX) { + PyMem_Free(src_); + PyErr_SetString(PyExc_OverflowError, "input string is too long"); + return NULL; + } + + int dest_size = LCMapStringEx(locale, flags, src_, (int)src_size, NULL, 0, NULL, NULL, 0); - if (dest_size == 0) { - return PyErr_SetFromWindowsErr(0); + if (dest_size <= 0) { + DWORD error = GetLastError(); + PyMem_Free(src_); + return PyErr_SetFromWindowsErr(error); } wchar_t* dest = PyMem_NEW(wchar_t, dest_size); if (dest == NULL) { + PyMem_Free(src_); return PyErr_NoMemory(); } - int nmapped = LCMapStringEx(locale, flags, src, -1, dest, dest_size, + int nmapped = LCMapStringEx(locale, flags, src_, (int)src_size, dest, dest_size, NULL, NULL, 0); - if (nmapped == 0) { + if (nmapped <= 0) { DWORD error = GetLastError(); + PyMem_Free(src_); PyMem_DEL(dest); return PyErr_SetFromWindowsErr(error); } - PyObject *ret = PyUnicode_FromWideChar(dest, dest_size - 1); + PyMem_Free(src_); + PyObject *ret = PyUnicode_FromWideChar(dest, nmapped); PyMem_DEL(dest); return ret; diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 3767b19d76db0..1394a8fcffd9f 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -885,7 +885,7 @@ PyDoc_STRVAR(_winapi_LCMapStringEx__doc__, static PyObject * _winapi_LCMapStringEx_impl(PyObject *module, LPCWSTR locale, DWORD flags, - LPCWSTR src); + PyObject *src); static PyObject * _winapi_LCMapStringEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -912,16 +912,16 @@ _winapi_LCMapStringEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static const char * const _keywords[] = {"locale", "flags", "src", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .format = "O&kO&:LCMapStringEx", + .format = "O&kU:LCMapStringEx", .kwtuple = KWTUPLE, }; #undef KWTUPLE LPCWSTR locale = NULL; DWORD flags; - LPCWSTR src = NULL; + PyObject *src; if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - _PyUnicode_WideCharString_Converter, &locale, &flags, _PyUnicode_WideCharString_Converter, &src)) { + _PyUnicode_WideCharString_Converter, &locale, &flags, &src)) { goto exit; } return_value = _winapi_LCMapStringEx_impl(module, locale, flags, src); @@ -929,8 +929,6 @@ _winapi_LCMapStringEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: /* Cleanup for locale */ PyMem_Free((void *)locale); - /* Cleanup for src */ - PyMem_Free((void *)src); return return_value; } @@ -1481,4 +1479,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=be1343b3759e0c96 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9d43ae4bdbe1126a input=a9049054013a1b77]*/ From webhook-mailer at python.org Wed Aug 16 06:13:39 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 10:13:39 -0000 Subject: [Python-checkins] [3.12] gh-106300: Improve errors testing in test_unittest.test_runner (GH-106737) (#108006) Message-ID: https://github.com/python/cpython/commit/4421c65f08228b203a1bed3b30ab3f5189b62427 commit: 4421c65f08228b203a1bed3b30ab3f5189b62427 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-16T12:13:35+02:00 summary: [3.12] gh-106300: Improve errors testing in test_unittest.test_runner (GH-106737) (#108006) gh-106300: Improve errors testing in test_unittest.test_runner (GH-106737) Use a custom exception to prevent unintentional silence of actual errors. (cherry picked from commit fd9d70a94de5b0756b52b9ae21e236e25545db4f) Co-authored-by: Nikita Sobolev files: M Lib/test/test_unittest/test_runner.py diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index f3b2c0cffd451..1b9cef43e3f9c 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -24,6 +24,13 @@ def getRunner(): stream=io.StringIO()) +class CustomError(Exception): + pass + +# For test output compat: +CustomErrorRepr = f"{__name__ + '.' if __name__ != '__main__' else ''}CustomError" + + def runTests(*cases): suite = unittest.TestSuite() for case in cases: @@ -46,7 +53,7 @@ def cleanup(ordering, blowUp=False): ordering.append('cleanup_good') else: ordering.append('cleanup_exc') - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class TestCM: @@ -108,8 +115,8 @@ def testNothing(self): result = unittest.TestResult() outcome = test._outcome = _Outcome(result=result) - CleanUpExc = Exception('foo') - exc2 = Exception('bar') + CleanUpExc = CustomError('foo') + exc2 = CustomError('bar') def cleanup1(): raise CleanUpExc @@ -125,10 +132,10 @@ def cleanup2(): (_, msg2), (_, msg1) = result.errors self.assertIn('in cleanup1', msg1) self.assertIn('raise CleanUpExc', msg1) - self.assertIn('Exception: foo', msg1) + self.assertIn(f'{CustomErrorRepr}: foo', msg1) self.assertIn('in cleanup2', msg2) self.assertIn('raise exc2', msg2) - self.assertIn('Exception: bar', msg2) + self.assertIn(f'{CustomErrorRepr}: bar', msg2) def testCleanupInRun(self): blowUp = False @@ -139,7 +146,7 @@ def setUp(self): ordering.append('setUp') test.addCleanup(cleanup2) if blowUp: - raise Exception('foo') + raise CustomError('foo') def testNothing(self): ordering.append('test') @@ -280,7 +287,7 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() def testNothing(self): ordering.append('test') @classmethod @@ -306,7 +313,7 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() def testNothing(self): ordering.append('test') @classmethod @@ -346,7 +353,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'CleanUpExc') self.assertEqual(ordering, @@ -366,10 +373,10 @@ def testNothing(self): @classmethod def tearDownClass(cls): ordering.append('tearDownClass') - raise Exception('TearDownClassExc') + raise CustomError('TearDownClassExc') suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) @@ -379,7 +386,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) @@ -392,16 +399,22 @@ def testNothing(self): pass def cleanup1(): - raise Exception('cleanup1') + raise CustomError('cleanup1') def cleanup2(): - raise Exception('cleanup2') + raise CustomError('cleanup2') TestableTest.addClassCleanup(cleanup1) TestableTest.addClassCleanup(cleanup2) - with self.assertRaises(Exception) as e: - TestableTest.doClassCleanups() - self.assertEqual(e, 'cleanup1') + TestableTest.doClassCleanups() + + self.assertEqual(len(TestableTest.tearDown_exceptions), 2) + + e1, e2 = TestableTest.tearDown_exceptions + self.assertIsInstance(e1[1], CustomError) + self.assertEqual(str(e1[1]), 'cleanup2') + self.assertIsInstance(e2[1], CustomError) + self.assertEqual(str(e2[1]), 'cleanup1') def test_with_errors_addCleanUp(self): ordering = [] @@ -421,7 +434,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'cleanup_exc', 'tearDownClass', 'cleanup_good']) @@ -444,7 +457,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'cleanup_good', 'tearDownClass', 'cleanup_exc']) @@ -460,11 +473,11 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering, blowUp=True) if class_blow_up: - raise Exception('ClassExc') + raise CustomError('ClassExc') def setUp(self): ordering.append('setUp') if method_blow_up: - raise Exception('MethodExc') + raise CustomError('MethodExc') def testNothing(self): ordering.append('test') @classmethod @@ -473,7 +486,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'tearDownClass', 'cleanup_exc']) @@ -483,9 +496,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ClassExc') + f'{CustomErrorRepr}: ClassExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'cleanup_exc']) @@ -494,9 +507,9 @@ def tearDownClass(cls): method_blow_up = True result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: MethodExc') + f'{CustomErrorRepr}: MethodExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'tearDownClass', 'cleanup_exc']) @@ -513,11 +526,11 @@ def testNothing(self): @classmethod def tearDownClass(cls): ordering.append('tearDownClass') - raise Exception('TearDownExc') + raise CustomError('TearDownExc') result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: TearDownExc') + f'{CustomErrorRepr}: TearDownExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass', 'cleanup_good']) @@ -620,7 +633,7 @@ def module_cleanup_good(*args, **kwargs): module_cleanups.append((3, args, kwargs)) def module_cleanup_bad(*args, **kwargs): - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class Module(object): unittest.addModuleCleanup(module_cleanup_good, 1, 2, 3, @@ -630,7 +643,7 @@ class Module(object): [(module_cleanup_good, (1, 2, 3), dict(four='hello', five='goodbye')), (module_cleanup_bad, (), {})]) - with self.assertRaises(Exception) as e: + with self.assertRaises(CustomError) as e: unittest.case.doModuleCleanups() self.assertEqual(str(e.exception), 'CleanUpExc') self.assertEqual(unittest.case._module_cleanups, []) @@ -659,7 +672,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) if blowUp: - raise Exception('setUpModule Exc') + raise CustomError('setUpModule Exc') @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -679,7 +692,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(ordering, ['setUpModule', 'cleanup_good']) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: setUpModule Exc') + f'{CustomErrorRepr}: setUpModule Exc') ordering = [] blowUp = False @@ -699,7 +712,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -710,7 +723,7 @@ def setUpModule(): ordering.append('setUpModule2') unittest.addModuleCleanup(cleanup, ordering) if blowUp2: - raise Exception() + raise CustomError() @staticmethod def tearDownModule(): ordering.append('tearDownModule2') @@ -799,7 +812,7 @@ def setUpModule(): @staticmethod def tearDownModule(): ordering.append('tearDownModule') - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class TestableTest(unittest.TestCase): @classmethod @@ -815,7 +828,7 @@ def tearDownClass(cls): sys.modules['Module'] = Module result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'tearDownModule', 'cleanup_good']) @@ -855,7 +868,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -873,7 +886,7 @@ def setUpModule(): @staticmethod def tearDownModule(): ordering.append('tearDownModule') - raise Exception('TearDownModuleExc') + raise CustomError('TearDownModuleExc') class TestableTest(unittest.TestCase): @classmethod @@ -888,7 +901,7 @@ def tearDownClass(cls): TestableTest.__module__ = 'Module' sys.modules['Module'] = Module suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -899,7 +912,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -978,7 +991,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) @@ -1008,7 +1021,7 @@ def tearDown(self): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUp', 'test', 'tearDown', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) @@ -1024,7 +1037,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering, blowUp=True) if module_blow_up: - raise Exception('ModuleExc') + raise CustomError('ModuleExc') @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -1034,11 +1047,11 @@ class TestableTest(unittest.TestCase): def setUpClass(cls): ordering.append('setUpClass') if class_blow_up: - raise Exception('ClassExc') + raise CustomError('ClassExc') def setUp(self): ordering.append('setUp') if method_blow_up: - raise Exception('MethodExc') + raise CustomError('MethodExc') def testNothing(self): ordering.append('test') @classmethod @@ -1050,7 +1063,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'setUp', 'test', 'tearDownClass', 'tearDownModule', @@ -1062,9 +1075,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ModuleExc') + f'{CustomErrorRepr}: ModuleExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'cleanup_exc']) ordering = [] @@ -1073,9 +1086,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ClassExc') + f'{CustomErrorRepr}: ClassExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'tearDownModule', 'cleanup_exc']) @@ -1085,9 +1098,9 @@ def tearDownClass(cls): method_blow_up = True result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: MethodExc') + f'{CustomErrorRepr}: MethodExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'setUp', 'tearDownClass', 'tearDownModule', 'cleanup_exc']) From webhook-mailer at python.org Wed Aug 16 06:35:42 2023 From: webhook-mailer at python.org (markshannon) Date: Wed, 16 Aug 2023 10:35:42 -0000 Subject: [Python-checkins] gh-105724: Add location information to `assert` errors (GH-105935) Message-ID: https://github.com/python/cpython/commit/bdd8ddfda166d1ed49744d61dcc486d62a9ac890 commit: bdd8ddfda166d1ed49744d61dcc486d62a9ac890 branch: main author: Nikita Sobolev committer: markshannon date: 2023-08-16T11:35:38+01:00 summary: gh-105724: Add location information to `assert` errors (GH-105935) files: A Misc/NEWS.d/next/Core and Builtins/2023-06-20-10-53-17.gh-issue-105724.d23L4M.rst M Lib/test/test_compile.py M Lib/test/test_exceptions.py M Python/compile.c diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 770184c5ef9a9..2ed8ae0496137 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1315,18 +1315,18 @@ def test_multiline_assert(self): snippet = textwrap.dedent("""\ assert (a > 0 and bb > 0 and - ccc == 4), "error msg" + ccc == 1000000), "error msg" """) compiled_code, _ = self.check_positions_against_ast(snippet) self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_ASSERTION_ERROR', - line=1, end_line=3, column=0, end_column=30, occurrence=1) + line=1, end_line=3, column=0, end_column=36, occurrence=1) # The "error msg": self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_CONST', - line=3, end_line=3, column=19, end_column=30, occurrence=4) + line=3, end_line=3, column=25, end_column=36, occurrence=4) self.assertOpcodeSourcePositionIs(compiled_code, 'CALL', - line=1, end_line=3, column=0, end_column=30, occurrence=1) + line=1, end_line=3, column=0, end_column=36, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'RAISE_VARARGS', - line=1, end_line=3, column=0, end_column=30, occurrence=1) + line=1, end_line=3, column=8, end_column=22, occurrence=1) def test_multiline_generator_expression(self): snippet = textwrap.dedent("""\ diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index f3554f1c4bb3f..764122ed4ef78 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1931,6 +1931,123 @@ def test_copy_pickle(self): self.assertEqual(exc.name, orig.name) self.assertEqual(exc.path, orig.path) + +class AssertionErrorTests(unittest.TestCase): + def tearDown(self): + unlink(TESTFN) + + def write_source(self, source): + with open(TESTFN, 'w') as testfile: + testfile.write(dedent(source)) + _rc, _out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) + return err.decode('utf-8').splitlines() + + def test_assertion_error_location(self): + cases = [ + ('assert None', + [ + ' assert None', + ' ^^^^', + 'AssertionError', + ], + ), + ('assert 0', + [ + ' assert 0', + ' ^', + 'AssertionError', + ], + ), + ('assert 1 > 2', + [ + ' assert 1 > 2', + ' ^^^^^', + 'AssertionError', + ], + ), + ('assert 1 > 2 and 3 > 2', + [ + ' assert 1 > 2 and 3 > 2', + ' ^^^^^^^^^^^^^^^', + 'AssertionError', + ], + ), + ('assert 1 > 2, "message"', + [ + ' assert 1 > 2, "message"', + ' ^^^^^', + 'AssertionError: message', + ], + ), + + # Multiline: + (""" + assert ( + 1 > 2) + """, + [ + ' 1 > 2)', + ' ^^^^^', + 'AssertionError', + ], + ), + (""" + assert ( + 1 > 2), "Message" + """, + [ + ' 1 > 2), "Message"', + ' ^^^^^', + 'AssertionError: Message', + ], + ), + (""" + assert ( + 1 > 2), \\ + "Message" + """, + [ + ' 1 > 2), \\', + ' ^^^^^', + 'AssertionError: Message', + ], + ), + ] + for source, expected in cases: + with self.subTest(source): + result = self.write_source(source) + self.assertEqual(result[-3:], expected) + + def test_multiline_not_highlighted(self): + cases = [ + (""" + assert ( + 1 > 2 + ) + """, + [ + ' 1 > 2', + 'AssertionError', + ], + ), + (""" + assert ( + 1 < 2 and + 3 > 4 + ) + """, + [ + ' 1 < 2 and', + 'AssertionError', + ], + ), + ] + for source, expected in cases: + with self.subTest(source): + result = self.write_source(source) + self.assertEqual(result[-2:], expected) + + class SyntaxErrorTests(unittest.TestCase): def test_range_of_offsets(self): cases = [ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-20-10-53-17.gh-issue-105724.d23L4M.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-20-10-53-17.gh-issue-105724.d23L4M.rst new file mode 100644 index 0000000000000..281c139d1a8d1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-20-10-53-17.gh-issue-105724.d23L4M.rst @@ -0,0 +1 @@ +Improve ``assert`` error messages by providing exact error range. diff --git a/Python/compile.c b/Python/compile.c index 83cf45550e258..3260dba57eac8 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -3952,7 +3952,7 @@ compiler_assert(struct compiler *c, stmt_ty s) VISIT(c, expr, s->v.Assert.msg); ADDOP_I(c, LOC(s), CALL, 0); } - ADDOP_I(c, LOC(s), RAISE_VARARGS, 1); + ADDOP_I(c, LOC(s->v.Assert.test), RAISE_VARARGS, 1); USE_LABEL(c, end); return SUCCESS; From webhook-mailer at python.org Wed Aug 16 06:40:00 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 16 Aug 2023 10:40:00 -0000 Subject: [Python-checkins] gh-106368: Argument Clinic: Test that keyword params are disallowed in groups (#107985) Message-ID: https://github.com/python/cpython/commit/57a20b0960f5c087a476b34c72f608580746cab5 commit: 57a20b0960f5c087a476b34c72f608580746cab5 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-16T12:39:56+02:00 summary: gh-106368: Argument Clinic: Test that keyword params are disallowed in groups (#107985) files: M Lib/test/test_clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 76729c8c3e4f7..32aac407a028c 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1524,6 +1524,27 @@ def test_disallowed_grouping__no_matching_bracket(self): err = "Function 'empty_group' has a ']' without a matching '['" self.expect_failure(block, err) + def test_disallowed_grouping__must_be_position_only(self): + dataset = (""" + with_kwds + [ + * + a: object + ] + """, """ + with_kwds + [ + a: object + ] + """) + err = ( + "You cannot use optional groups ('[' and ']') unless all " + "parameters are positional-only ('/')" + ) + for block in dataset: + with self.subTest(block=block): + self.expect_failure(block, err) + def test_no_parameters(self): function = self.parse_function(""" module foo From webhook-mailer at python.org Wed Aug 16 07:57:03 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 16 Aug 2023 11:57:03 -0000 Subject: [Python-checkins] gh-106659: Fix test_embed.test_forced_io_encoding() on Windows (#108010) Message-ID: https://github.com/python/cpython/commit/e35c722d22cae605b485e75a69238dc44aab4c96 commit: e35c722d22cae605b485e75a69238dc44aab4c96 branch: main author: Victor Stinner committer: vstinner date: 2023-08-16T11:56:56Z summary: gh-106659: Fix test_embed.test_forced_io_encoding() on Windows (#108010) Use config.legacy_windows_stdio=1 to avoid _io._WindowsConsoleIO. files: M Doc/whatsnew/3.13.rst M Programs/_testembed.c diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index a65a98bae1357..84ffd84b9f0bb 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -941,7 +941,9 @@ Removed * ``Py_SetPath()``: set :c:member:`PyConfig.module_search_paths` instead. * ``Py_SetProgramName()``: set :c:member:`PyConfig.program_name` instead. * ``Py_SetPythonHome()``: set :c:member:`PyConfig.home` instead. - * ``Py_SetStandardStreamEncoding()``: set :c:member:`PyConfig.stdio_encoding` instead. + * ``Py_SetStandardStreamEncoding()``: set :c:member:`PyConfig.stdio_encoding` + instead, and set also maybe :c:member:`PyConfig.legacy_windows_stdio` (on + Windows). * ``_Py_SetProgramFullPath()``: set :c:member:`PyConfig.executable` instead. Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 351cdc302e8cb..7ee64b22925f0 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -237,6 +237,11 @@ static void check_stdio_details(const wchar_t *encoding, const wchar_t *errors) if (errors) { config_set_string(&config, &config.stdio_errors, errors); } +#ifdef MS_WINDOWS + // gh-106659: On Windows, don't use _io._WindowsConsoleIO which always + // announce UTF-8 for sys.stdin.encoding. + config.legacy_windows_stdio = 1; +#endif config_set_program_name(&config); init_from_config_clear(&config); From webhook-mailer at python.org Wed Aug 16 08:22:01 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 12:22:01 -0000 Subject: [Python-checkins] [3.12] gh-99203: shutil.make_archive(): restore select CPython <= 3.10.5 behavior (GH-99802) (#107998) Message-ID: https://github.com/python/cpython/commit/5d9f20a06c6ad3f1d7812e098717b1c6f7c673fa commit: 5d9f20a06c6ad3f1d7812e098717b1c6f7c673fa branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-16T14:21:57+02:00 summary: [3.12] gh-99203: shutil.make_archive(): restore select CPython <= 3.10.5 behavior (GH-99802) (#107998) gh-99203: shutil.make_archive(): restore select CPython <= 3.10.5 behavior (GH-99802) Restore following CPython <= 3.10.5 behavior of shutil.make_archive() that went away as part of gh-93160: Do not create an empty archive if root_dir is not a directory, and, in that case, raise FileNotFoundError or NotADirectoryError regardless of format choice. Beyond the brought-back behavior, the function may now also raise these exceptions in dry_run mode. (cherry picked from commit a86df298df5b02e2d69ea6879e9ed10a7adb85d0) Co-authored-by: 6t8k <58048945+6t8k at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst M Lib/shutil.py M Lib/test/test_shutil.py diff --git a/Lib/shutil.py b/Lib/shutil.py index 3f2864af517e7..b37bd082eee0c 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -1156,6 +1156,10 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, supports_root_dir = getattr(func, 'supports_root_dir', False) save_cwd = None if root_dir is not None: + stmd = os.stat(root_dir).st_mode + if not stat.S_ISDIR(stmd): + raise NotADirectoryError(errno.ENOTDIR, 'Not a directory', root_dir) + if supports_root_dir: # Support path-like base_name here for backwards-compatibility. base_name = os.fspath(base_name) diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index d74eee767c991..cd1c3d8cfbc38 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1840,6 +1840,49 @@ def test_register_archive_format(self): formats = [name for name, params in get_archive_formats()] self.assertNotIn('xxx', formats) + def test_make_tarfile_rootdir_nodir(self): + # GH-99203 + self.addCleanup(os_helper.unlink, f'{TESTFN}.tar') + for dry_run in (False, True): + with self.subTest(dry_run=dry_run): + tmp_dir = self.mkdtemp() + nonexisting_file = os.path.join(tmp_dir, 'nonexisting') + with self.assertRaises(FileNotFoundError) as cm: + make_archive(TESTFN, 'tar', nonexisting_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOENT) + self.assertEqual(cm.exception.filename, nonexisting_file) + self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + + tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) + os.close(tmp_fd) + with self.assertRaises(NotADirectoryError) as cm: + make_archive(TESTFN, 'tar', tmp_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + self.assertEqual(cm.exception.filename, tmp_file) + self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + + @support.requires_zlib() + def test_make_zipfile_rootdir_nodir(self): + # GH-99203 + self.addCleanup(os_helper.unlink, f'{TESTFN}.zip') + for dry_run in (False, True): + with self.subTest(dry_run=dry_run): + tmp_dir = self.mkdtemp() + nonexisting_file = os.path.join(tmp_dir, 'nonexisting') + with self.assertRaises(FileNotFoundError) as cm: + make_archive(TESTFN, 'zip', nonexisting_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOENT) + self.assertEqual(cm.exception.filename, nonexisting_file) + self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + + tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) + os.close(tmp_fd) + with self.assertRaises(NotADirectoryError) as cm: + make_archive(TESTFN, 'zip', tmp_file, dry_run=dry_run) + self.assertEqual(cm.exception.errno, errno.ENOTDIR) + self.assertEqual(cm.exception.filename, tmp_file) + self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + ### shutil.unpack_archive def check_unpack_archive(self, format, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst b/Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst new file mode 100644 index 0000000000000..fcfb044d476ac --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-26-22-05-22.gh-issue-99203.j0DUae.rst @@ -0,0 +1,5 @@ +Restore following CPython <= 3.10.5 behavior of :func:`shutil.make_archive`: +do not create an empty archive if ``root_dir`` is not a directory, and, in that +case, raise :class:`FileNotFoundError` or :class:`NotADirectoryError` +regardless of ``format`` choice. Beyond the brought-back behavior, the function +may now also raise these exceptions in ``dry_run`` mode. From webhook-mailer at python.org Wed Aug 16 08:22:26 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 12:22:26 -0000 Subject: [Python-checkins] [3.12] More actionable error message when spawn is incorrectly used. (GH-102203) (#107990) Message-ID: https://github.com/python/cpython/commit/c2fb25a7145cbb6bbe92de28a230e963fe4d696f commit: c2fb25a7145cbb6bbe92de28a230e963fe4d696f branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-16T14:22:22+02:00 summary: [3.12] More actionable error message when spawn is incorrectly used. (GH-102203) (#107990) More actionable error message when spawn is incorrectly used. (GH-102203) (cherry picked from commit a794ebeb028f7ef287c780d3890f816db9c21c51) Co-authored-by: Yuxin Wu Co-authored-by: Yuxin Wu Co-authored-by: Oleg Iarygin files: M Lib/multiprocessing/spawn.py diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index f1af770910471..daac1ecc34b55 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -150,7 +150,11 @@ def _check_not_importing_main(): ... The "freeze_support()" line can be omitted if the program - is not going to be frozen to produce an executable.''') + is not going to be frozen to produce an executable. + + To fix this issue, refer to the "Safe importing of main module" + section in https://docs.python.org/3/library/multiprocessing.html + ''') def get_preparation_data(name): From webhook-mailer at python.org Wed Aug 16 08:24:50 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 12:24:50 -0000 Subject: [Python-checkins] [3.12] gh-91795: Update build optimization part of PCbuild/readme.txt (GH-91849) (#107776) Message-ID: https://github.com/python/cpython/commit/8cb750df13c3d508726b6d507d35b807f1826e32 commit: 8cb750df13c3d508726b6d507d35b807f1826e32 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-16T14:24:47+02:00 summary: [3.12] gh-91795: Update build optimization part of PCbuild/readme.txt (GH-91849) (#107776) gh-91795: Update build optimization part of PCbuild/readme.txt (GH-91849) (cherry picked from commit 906b73be5eada1995bd667a02c59f7a11998310f) Co-authored-by: Fatih <77548106+fatihkabakk at users.noreply.github.com> files: M PCbuild/readme.txt diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index fe44ca40f4a0e..5c682a2826088 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -252,9 +252,11 @@ against a profiling library and contain extra debug information. The PGUpdate configuration takes the profiling data and generates optimized binaries. -The build_pgo.bat script automates the creation of optimized binaries. -It creates the PGI files, runs the unit test suite or PyBench with the -PGI python, and finally creates the optimized files. +The build.bat script has an argument `--pgo` that automate the creation +of optimized binaries. +It creates the PGI files, runs the unit test suite with the PGI python, +and finally creates the optimized files. +You can customize the job for profiling with `--pgo-job ` option. See https://docs.microsoft.com/en-us/cpp/build/profile-guided-optimizations From webhook-mailer at python.org Wed Aug 16 08:25:22 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 12:25:22 -0000 Subject: [Python-checkins] [3.12] README: remove unmaintained sections (GH-107703) (#107762) Message-ID: https://github.com/python/cpython/commit/2598a1a835e5bdf0532c08d867cf54f94aeb85a0 commit: 2598a1a835e5bdf0532c08d867cf54f94aeb85a0 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-16T14:25:18+02:00 summary: [3.12] README: remove unmaintained sections (GH-107703) (#107762) README: remove unmaintained sections (GH-107703) (cherry picked from commit 7a250fdc16bb6f1fe0a6b0df8bb502870405b5d6) Co-authored-by: Inada Naoki files: M README.rst diff --git a/README.rst b/README.rst index 7742de68d5a2f..b97bd3c8bd667 100644 --- a/README.rst +++ b/README.rst @@ -211,30 +211,6 @@ primary version, you would execute ``make install`` in your 3.12 build directory and ``make altinstall`` in the others. -Issue Tracker and Mailing List ------------------------------- - -Bug reports are welcome! You can use Github to `report bugs -`_, and/or `submit pull requests -`_. - -You can also follow development discussion on the `python-dev mailing list -`_. - - -Proposals for enhancement -------------------------- - -If you have a proposal to change Python, you may want to send an email to the -`comp.lang.python`_ or `python-ideas`_ mailing lists for initial feedback. A -Python Enhancement Proposal (PEP) may be submitted if your idea gains ground. -All current PEPs, as well as guidelines for submitting a new PEP, are listed at -`peps.python.org `_. - -.. _python-ideas: https://mail.python.org/mailman/listinfo/python-ideas/ -.. _comp.lang.python: https://mail.python.org/mailman/listinfo/python-list - - Release Schedule ---------------- From webhook-mailer at python.org Wed Aug 16 08:27:37 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 12:27:37 -0000 Subject: [Python-checkins] [3.12] gh-86457: Fix signature for code.replace() (GH-23199) (#107744) Message-ID: https://github.com/python/cpython/commit/9864f9a7c7046d6a0174c8142ddfbb5f07de5925 commit: 9864f9a7c7046d6a0174c8142ddfbb5f07de5925 branch: 3.12 author: Serhiy Storchaka committer: Yhg1s date: 2023-08-16T14:27:32+02:00 summary: [3.12] gh-86457: Fix signature for code.replace() (GH-23199) (#107744) * [3.12] gh-86457: Fix signature for code.replace() (GH-23199) Also add support of @text_signature in Argument Clinic.. (cherry picked from commit 0e6e32fb84b2f7cb668e0b9927637587081e38cd) Co-authored-by: Serhiy Storchaka * Update 2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst files: A Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst M Objects/clinic/codeobject.c.h M Objects/codeobject.c M Tools/clinic/clinic.py diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst new file mode 100644 index 0000000000000..4768e6767574d --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-07-16-30-48.gh-issue-95065.-im4R5.rst @@ -0,0 +1,2 @@ +Argument Clinic now supports overriding automatically generated signature by +using directive ``@text_signature``. diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index 5ad4b1fed7341..1034627edd757 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -163,12 +163,7 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(code_replace__doc__, -"replace($self, /, *, co_argcount=-1, co_posonlyargcount=-1,\n" -" co_kwonlyargcount=-1, co_nlocals=-1, co_stacksize=-1,\n" -" co_flags=-1, co_firstlineno=-1, co_code=None, co_consts=None,\n" -" co_names=None, co_varnames=None, co_freevars=None,\n" -" co_cellvars=None, co_filename=None, co_name=None,\n" -" co_qualname=None, co_linetable=None, co_exceptiontable=None)\n" +"replace($self, /, **changes)\n" "--\n" "\n" "Return a copy of the code object with new values for the specified fields."); @@ -180,13 +175,12 @@ static PyObject * code_replace_impl(PyCodeObject *self, int co_argcount, int co_posonlyargcount, int co_kwonlyargcount, int co_nlocals, int co_stacksize, int co_flags, - int co_firstlineno, PyBytesObject *co_code, - PyObject *co_consts, PyObject *co_names, - PyObject *co_varnames, PyObject *co_freevars, - PyObject *co_cellvars, PyObject *co_filename, - PyObject *co_name, PyObject *co_qualname, - PyBytesObject *co_linetable, - PyBytesObject *co_exceptiontable); + int co_firstlineno, PyObject *co_code, PyObject *co_consts, + PyObject *co_names, PyObject *co_varnames, + PyObject *co_freevars, PyObject *co_cellvars, + PyObject *co_filename, PyObject *co_name, + PyObject *co_qualname, PyObject *co_linetable, + PyObject *co_exceptiontable); static PyObject * code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -226,7 +220,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje int co_stacksize = self->co_stacksize; int co_flags = self->co_flags; int co_firstlineno = self->co_firstlineno; - PyBytesObject *co_code = NULL; + PyObject *co_code = NULL; PyObject *co_consts = self->co_consts; PyObject *co_names = self->co_names; PyObject *co_varnames = NULL; @@ -235,8 +229,8 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje PyObject *co_filename = self->co_filename; PyObject *co_name = self->co_name; PyObject *co_qualname = self->co_qualname; - PyBytesObject *co_linetable = (PyBytesObject *)self->co_linetable; - PyBytesObject *co_exceptiontable = (PyBytesObject *)self->co_exceptiontable; + PyObject *co_linetable = self->co_linetable; + PyObject *co_exceptiontable = self->co_exceptiontable; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf); if (!args) { @@ -313,7 +307,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("replace", "argument 'co_code'", "bytes", args[7]); goto exit; } - co_code = (PyBytesObject *)args[7]; + co_code = args[7]; if (!--noptargs) { goto skip_optional_kwonly; } @@ -412,7 +406,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("replace", "argument 'co_linetable'", "bytes", args[16]); goto exit; } - co_linetable = (PyBytesObject *)args[16]; + co_linetable = args[16]; if (!--noptargs) { goto skip_optional_kwonly; } @@ -421,7 +415,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje _PyArg_BadArgument("replace", "argument 'co_exceptiontable'", "bytes", args[17]); goto exit; } - co_exceptiontable = (PyBytesObject *)args[17]; + co_exceptiontable = args[17]; skip_optional_kwonly: return_value = code_replace_impl(self, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, co_stacksize, co_flags, co_firstlineno, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_qualname, co_linetable, co_exceptiontable); @@ -488,4 +482,4 @@ code__varname_from_oparg(PyCodeObject *self, PyObject *const *args, Py_ssize_t n exit: return return_value; } -/*[clinic end generated code: output=f1fab6e71c785182 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ff40f7bdd3851de3 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index d47ca731a10c8..aee1213632e49 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1962,27 +1962,28 @@ code_linesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args)) } /*[clinic input] + at text_signature "($self, /, **changes)" code.replace * - co_argcount: int(c_default="self->co_argcount") = -1 - co_posonlyargcount: int(c_default="self->co_posonlyargcount") = -1 - co_kwonlyargcount: int(c_default="self->co_kwonlyargcount") = -1 - co_nlocals: int(c_default="self->co_nlocals") = -1 - co_stacksize: int(c_default="self->co_stacksize") = -1 - co_flags: int(c_default="self->co_flags") = -1 - co_firstlineno: int(c_default="self->co_firstlineno") = -1 - co_code: PyBytesObject(c_default="NULL") = None - co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = None - co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = None - co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = None - co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None - co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = None - co_filename: unicode(c_default="self->co_filename") = None - co_name: unicode(c_default="self->co_name") = None - co_qualname: unicode(c_default="self->co_qualname") = None - co_linetable: PyBytesObject(c_default="(PyBytesObject *)self->co_linetable") = None - co_exceptiontable: PyBytesObject(c_default="(PyBytesObject *)self->co_exceptiontable") = None + co_argcount: int(c_default="self->co_argcount") = unchanged + co_posonlyargcount: int(c_default="self->co_posonlyargcount") = unchanged + co_kwonlyargcount: int(c_default="self->co_kwonlyargcount") = unchanged + co_nlocals: int(c_default="self->co_nlocals") = unchanged + co_stacksize: int(c_default="self->co_stacksize") = unchanged + co_flags: int(c_default="self->co_flags") = unchanged + co_firstlineno: int(c_default="self->co_firstlineno") = unchanged + co_code: object(subclass_of="&PyBytes_Type", c_default="NULL") = unchanged + co_consts: object(subclass_of="&PyTuple_Type", c_default="self->co_consts") = unchanged + co_names: object(subclass_of="&PyTuple_Type", c_default="self->co_names") = unchanged + co_varnames: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged + co_freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged + co_cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = unchanged + co_filename: unicode(c_default="self->co_filename") = unchanged + co_name: unicode(c_default="self->co_name") = unchanged + co_qualname: unicode(c_default="self->co_qualname") = unchanged + co_linetable: object(subclass_of="&PyBytes_Type", c_default="self->co_linetable") = unchanged + co_exceptiontable: object(subclass_of="&PyBytes_Type", c_default="self->co_exceptiontable") = unchanged Return a copy of the code object with new values for the specified fields. [clinic start generated code]*/ @@ -1991,14 +1992,13 @@ static PyObject * code_replace_impl(PyCodeObject *self, int co_argcount, int co_posonlyargcount, int co_kwonlyargcount, int co_nlocals, int co_stacksize, int co_flags, - int co_firstlineno, PyBytesObject *co_code, - PyObject *co_consts, PyObject *co_names, - PyObject *co_varnames, PyObject *co_freevars, - PyObject *co_cellvars, PyObject *co_filename, - PyObject *co_name, PyObject *co_qualname, - PyBytesObject *co_linetable, - PyBytesObject *co_exceptiontable) -/*[clinic end generated code: output=b6cd9988391d5711 input=f6f68e03571f8d7c]*/ + int co_firstlineno, PyObject *co_code, PyObject *co_consts, + PyObject *co_names, PyObject *co_varnames, + PyObject *co_freevars, PyObject *co_cellvars, + PyObject *co_filename, PyObject *co_name, + PyObject *co_qualname, PyObject *co_linetable, + PyObject *co_exceptiontable) +/*[clinic end generated code: output=e75c48a15def18b9 input=18e280e07846c122]*/ { #define CHECK_INT_ARG(ARG) \ if (ARG < 0) { \ @@ -2023,7 +2023,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount, if (code == NULL) { return NULL; } - co_code = (PyBytesObject *)code; + co_code = code; } if (PySys_Audit("code.__new__", "OOOiiiiii", @@ -2062,10 +2062,10 @@ code_replace_impl(PyCodeObject *self, int co_argcount, co = PyCode_NewWithPosOnlyArgs( co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, - co_stacksize, co_flags, (PyObject*)co_code, co_consts, co_names, + co_stacksize, co_flags, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_qualname, co_firstlineno, - (PyObject*)co_linetable, (PyObject*)co_exceptiontable); + co_linetable, co_exceptiontable); error: Py_XDECREF(code); diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index e775b012d0b6d..fd394c92cad9c 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -4310,6 +4310,7 @@ def reset(self) -> None: self.indent = IndentStack() self.kind = CALLABLE self.coexist = False + self.forced_text_signature: str | None = None self.parameter_continuation = '' self.preserve_output = False @@ -4444,6 +4445,11 @@ def at_coexist(self) -> None: fail("Called @coexist twice!") self.coexist = True + def at_text_signature(self, text_signature: str) -> None: + if self.forced_text_signature: + fail("Called @text_signature twice!") + self.forced_text_signature = text_signature + def parse(self, block: Block) -> None: self.reset() self.block = block @@ -5148,142 +5154,145 @@ def format_docstring(self): add(f.cls.name) else: add(f.name) - add('(') + if self.forced_text_signature: + add(self.forced_text_signature) + else: + add('(') + + # populate "right_bracket_count" field for every parameter + assert parameters, "We should always have a self parameter. " + repr(f) + assert isinstance(parameters[0].converter, self_converter) + # self is always positional-only. + assert parameters[0].is_positional_only() + parameters[0].right_bracket_count = 0 + positional_only = True + for p in parameters[1:]: + if not p.is_positional_only(): + positional_only = False + else: + assert positional_only + if positional_only: + p.right_bracket_count = abs(p.group) + else: + # don't put any right brackets around non-positional-only parameters, ever. + p.right_bracket_count = 0 + + right_bracket_count = 0 + + def fix_right_bracket_count(desired): + nonlocal right_bracket_count + s = '' + while right_bracket_count < desired: + s += '[' + right_bracket_count += 1 + while right_bracket_count > desired: + s += ']' + right_bracket_count -= 1 + return s + + need_slash = False + added_slash = False + need_a_trailing_slash = False + + # we only need a trailing slash: + # * if this is not a "docstring_only" signature + # * and if the last *shown* parameter is + # positional only + if not f.docstring_only: + for p in reversed(parameters): + if not p.converter.show_in_signature: + continue + if p.is_positional_only(): + need_a_trailing_slash = True + break - # populate "right_bracket_count" field for every parameter - assert parameters, "We should always have a self parameter. " + repr(f) - assert isinstance(parameters[0].converter, self_converter) - # self is always positional-only. - assert parameters[0].is_positional_only() - parameters[0].right_bracket_count = 0 - positional_only = True - for p in parameters[1:]: - if not p.is_positional_only(): - positional_only = False - else: - assert positional_only - if positional_only: - p.right_bracket_count = abs(p.group) - else: - # don't put any right brackets around non-positional-only parameters, ever. - p.right_bracket_count = 0 - - right_bracket_count = 0 - - def fix_right_bracket_count(desired): - nonlocal right_bracket_count - s = '' - while right_bracket_count < desired: - s += '[' - right_bracket_count += 1 - while right_bracket_count > desired: - s += ']' - right_bracket_count -= 1 - return s - need_slash = False - added_slash = False - need_a_trailing_slash = False + added_star = False - # we only need a trailing slash: - # * if this is not a "docstring_only" signature - # * and if the last *shown* parameter is - # positional only - if not f.docstring_only: - for p in reversed(parameters): + first_parameter = True + last_p = parameters[-1] + line_length = len(''.join(text)) + indent = " " * line_length + def add_parameter(text): + nonlocal line_length + nonlocal first_parameter + if first_parameter: + s = text + first_parameter = False + else: + s = ' ' + text + if line_length + len(s) >= 72: + add('\n') + add(indent) + line_length = len(indent) + s = text + line_length += len(s) + add(s) + + for p in parameters: if not p.converter.show_in_signature: continue - if p.is_positional_only(): - need_a_trailing_slash = True - break + assert p.name + is_self = isinstance(p.converter, self_converter) + if is_self and f.docstring_only: + # this isn't a real machine-parsable signature, + # so let's not print the "self" parameter + continue - added_star = False - - first_parameter = True - last_p = parameters[-1] - line_length = len(''.join(text)) - indent = " " * line_length - def add_parameter(text): - nonlocal line_length - nonlocal first_parameter - if first_parameter: - s = text - first_parameter = False - else: - s = ' ' + text - if line_length + len(s) >= 72: - add('\n') - add(indent) - line_length = len(indent) - s = text - line_length += len(s) - add(s) - - for p in parameters: - if not p.converter.show_in_signature: - continue - assert p.name - - is_self = isinstance(p.converter, self_converter) - if is_self and f.docstring_only: - # this isn't a real machine-parsable signature, - # so let's not print the "self" parameter - continue - - if p.is_positional_only(): - need_slash = not f.docstring_only - elif need_slash and not (added_slash or p.is_positional_only()): - added_slash = True - add_parameter('/,') - - if p.is_keyword_only() and not added_star: - added_star = True - add_parameter('*,') - - p_add, p_output = text_accumulator() - p_add(fix_right_bracket_count(p.right_bracket_count)) - - if isinstance(p.converter, self_converter): - # annotate first parameter as being a "self". - # - # if inspect.Signature gets this function, - # and it's already bound, the self parameter - # will be stripped off. - # - # if it's not bound, it should be marked - # as positional-only. - # - # note: we don't print "self" for __init__, - # because this isn't actually the signature - # for __init__. (it can't be, __init__ doesn't - # have a docstring.) if this is an __init__ - # (or __new__), then this signature is for - # calling the class to construct a new instance. - p_add('$') + if p.is_positional_only(): + need_slash = not f.docstring_only + elif need_slash and not (added_slash or p.is_positional_only()): + added_slash = True + add_parameter('/,') + + if p.is_keyword_only() and not added_star: + added_star = True + add_parameter('*,') + + p_add, p_output = text_accumulator() + p_add(fix_right_bracket_count(p.right_bracket_count)) + + if isinstance(p.converter, self_converter): + # annotate first parameter as being a "self". + # + # if inspect.Signature gets this function, + # and it's already bound, the self parameter + # will be stripped off. + # + # if it's not bound, it should be marked + # as positional-only. + # + # note: we don't print "self" for __init__, + # because this isn't actually the signature + # for __init__. (it can't be, __init__ doesn't + # have a docstring.) if this is an __init__ + # (or __new__), then this signature is for + # calling the class to construct a new instance. + p_add('$') - if p.is_vararg(): - p_add("*") + if p.is_vararg(): + p_add("*") - name = p.converter.signature_name or p.name - p_add(name) + name = p.converter.signature_name or p.name + p_add(name) - if not p.is_vararg() and p.converter.is_optional(): - p_add('=') - value = p.converter.py_default - if not value: - value = repr(p.converter.default) - p_add(value) + if not p.is_vararg() and p.converter.is_optional(): + p_add('=') + value = p.converter.py_default + if not value: + value = repr(p.converter.default) + p_add(value) - if (p != last_p) or need_a_trailing_slash: - p_add(',') + if (p != last_p) or need_a_trailing_slash: + p_add(',') - add_parameter(p_output()) + add_parameter(p_output()) - add(fix_right_bracket_count(0)) - if need_a_trailing_slash: - add_parameter('/') - add(')') + add(fix_right_bracket_count(0)) + if need_a_trailing_slash: + add_parameter('/') + add(')') # PEP 8 says: # From webhook-mailer at python.org Wed Aug 16 08:28:17 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 12:28:17 -0000 Subject: [Python-checkins] [3.12] gh-107735: Add C API tests for PySys_GetObject() and PySys_SetObject() (GH-107736) (#107740) Message-ID: https://github.com/python/cpython/commit/72534ca85c64d4d33a9cf30492e037be05140fd2 commit: 72534ca85c64d4d33a9cf30492e037be05140fd2 branch: 3.12 author: Serhiy Storchaka committer: Yhg1s date: 2023-08-16T14:28:14+02:00 summary: [3.12] gh-107735: Add C API tests for PySys_GetObject() and PySys_SetObject() (GH-107736) (#107740) [3.12] gh-107735: Add C API tests for PySys_GetObject() and PySys_SetObject() (GH-107736). (cherry picked from commit bea5f93196d213d6fbf4ba8984caf4c3cd1da882) files: M Lib/test/test_capi/test_misc.py M Modules/_testcapimodule.c diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 3f2736da7d93a..3d494975c71e2 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -51,6 +51,8 @@ import _testinternalcapi +NULL = None + def decode_stderr(err): return err.decode('utf-8', 'replace').replace('\r', '') @@ -1121,6 +1123,46 @@ class Data(_testcapi.ObjExtraData): del d.extra self.assertIsNone(d.extra) + def test_sys_getobject(self): + getobject = _testcapi.sys_getobject + + self.assertIs(getobject(b'stdout'), sys.stdout) + with support.swap_attr(sys, '\U0001f40d', 42): + self.assertEqual(getobject('\U0001f40d'.encode()), 42) + + self.assertIs(getobject(b'nonexisting'), AttributeError) + self.assertIs(getobject(b'\xff'), AttributeError) + # CRASHES getobject(NULL) + + def test_sys_setobject(self): + setobject = _testcapi.sys_setobject + + value = ['value'] + value2 = ['value2'] + try: + self.assertEqual(setobject(b'newattr', value), 0) + self.assertIs(sys.newattr, value) + self.assertEqual(setobject(b'newattr', value2), 0) + self.assertIs(sys.newattr, value2) + self.assertEqual(setobject(b'newattr', NULL), 0) + self.assertFalse(hasattr(sys, 'newattr')) + self.assertEqual(setobject(b'newattr', NULL), 0) + finally: + with contextlib.suppress(AttributeError): + del sys.newattr + try: + self.assertEqual(setobject('\U0001f40d'.encode(), value), 0) + self.assertIs(getattr(sys, '\U0001f40d'), value) + self.assertEqual(setobject('\U0001f40d'.encode(), NULL), 0) + self.assertFalse(hasattr(sys, '\U0001f40d')) + finally: + with contextlib.suppress(AttributeError): + delattr(sys, '\U0001f40d') + + with self.assertRaises(UnicodeDecodeError): + setobject(b'\xff', value) + # CRASHES setobject(NULL, value) + @requires_limited_api class TestHeapTypeRelative(unittest.TestCase): diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 6523734131c2d..098d2cfa6d8ad 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -45,6 +45,16 @@ // Include definitions from there. #include "_testcapi/parts.h" +#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromLong(_ret); \ + } while (0) + // Forward declarations static struct PyModuleDef _testcapimodule; static PyObject *TestError; /* set to exception object in init */ @@ -3336,6 +3346,35 @@ test_atexit(PyObject *self, PyObject *Py_UNUSED(args)) } +static PyObject * +sys_getobject(PyObject *Py_UNUSED(module), PyObject *arg) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_Parse(arg, "z#", &name, &size)) { + return NULL; + } + PyObject *result = PySys_GetObject(name); + if (result == NULL) { + result = PyExc_AttributeError; + } + return Py_NewRef(result); +} + +static PyObject * +sys_setobject(PyObject *Py_UNUSED(module), PyObject *args) +{ + const char *name; + Py_ssize_t size; + PyObject *value; + if (!PyArg_ParseTuple(args, "z#O", &name, &size, &value)) { + return NULL; + } + NULLABLE(value); + RETURN_INT(PySys_SetObject(name, value)); +} + + static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyMethodDef TestMethods[] = { @@ -3482,6 +3521,8 @@ static PyMethodDef TestMethods[] = { {"function_get_kw_defaults", function_get_kw_defaults, METH_O, NULL}, {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, {"test_atexit", test_atexit, METH_NOARGS}, + {"sys_getobject", sys_getobject, METH_O}, + {"sys_setobject", sys_setobject, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; From webhook-mailer at python.org Wed Aug 16 08:29:12 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 12:29:12 -0000 Subject: [Python-checkins] [3.12] gh-107178: Add the C API tests for the Abstract Objects Layer (GH-107179) (#107728) Message-ID: https://github.com/python/cpython/commit/9c8dfcec645b233dd6f36c286a7c5525c5d713e2 commit: 9c8dfcec645b233dd6f36c286a7c5525c5d713e2 branch: 3.12 author: Serhiy Storchaka committer: Yhg1s date: 2023-08-16T14:29:08+02:00 summary: [3.12] gh-107178: Add the C API tests for the Abstract Objects Layer (GH-107179) (#107728) Cover all the Mapping Protocol, almost all the Sequence Protocol (except PySequence_Fast) and a part of the Object Protocol. Move existing tests to Lib/test/test_capi/test_abstract.py and Modules/_testcapi/abstract.c. Add also tests for PyDict C API.. (cherry picked from commit 16c9415fba4972743f1944ebc44946e475e68bc4) files: A Lib/test/test_capi/test_abstract.py A Lib/test/test_capi/test_dict.py A Misc/NEWS.d/next/Tests/2023-07-24-16-56-59.gh-issue-107178.Gq1usE.rst A Modules/_testcapi/abstract.c A Modules/_testcapi/dict.c M Lib/test/test_bytes.py M Lib/test/test_capi/test_misc.py M Lib/test/test_class.py M Modules/Setup.stdlib.in M Modules/_testcapi/parts.h M Modules/_testcapimodule.c M PCbuild/_testcapi.vcxproj M PCbuild/_testcapi.vcxproj.filters diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 7c62b722059d1..afd506f07520d 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1354,7 +1354,7 @@ def do_tests(setitem): except ValueError: pass try: - setitem(b, 0, None) + setitem(b, 0, object()) self.fail("Didn't raise TypeError") except TypeError: pass diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py new file mode 100644 index 0000000000000..116edd8d4d2fd --- /dev/null +++ b/Lib/test/test_capi/test_abstract.py @@ -0,0 +1,694 @@ +import unittest +import sys +from collections import OrderedDict +from test import support +from test.support import import_helper +import _testcapi + + +NULL = None + +class TestObject: + @property + def evil(self): + raise RuntimeError('do not get evil') + @evil.setter + def evil(self, value): + raise RuntimeError('do not set evil') + @evil.deleter + def evil(self): + raise RuntimeError('do not del evil') + +class ProxyGetItem: + def __init__(self, obj): + self.obj = obj + def __getitem__(self, key): + return self.obj[key] + +class ProxySetItem: + def __init__(self, obj): + self.obj = obj + def __setitem__(self, key, value): + self.obj[key] = value + +class ProxyDelItem: + def __init__(self, obj): + self.obj = obj + def __delitem__(self, key): + del self.obj[key] + +def gen(): + yield 'a' + yield 'b' + yield 'c' + + +class CAPITest(unittest.TestCase): + + def test_object_getattr(self): + xgetattr = _testcapi.object_getattr + obj = TestObject() + obj.a = 11 + setattr(obj, '\U0001f40d', 22) + self.assertEqual(xgetattr(obj, 'a'), 11) + self.assertRaises(AttributeError, xgetattr, obj, 'b') + self.assertEqual(xgetattr(obj, '\U0001f40d'), 22) + + self.assertRaises(RuntimeError, xgetattr, obj, 'evil') + self.assertRaises(TypeError, xgetattr, obj, 1) + # CRASHES xgetattr(obj, NULL) + # CRASHES xgetattr(NULL, 'a') + + def test_object_getattrstring(self): + getattrstring = _testcapi.object_getattrstring + obj = TestObject() + obj.a = 11 + setattr(obj, '\U0001f40d', 22) + self.assertEqual(getattrstring(obj, b'a'), 11) + self.assertRaises(AttributeError, getattrstring, obj, b'b') + self.assertEqual(getattrstring(obj, '\U0001f40d'.encode()), 22) + + self.assertRaises(RuntimeError, getattrstring, obj, b'evil') + self.assertRaises(UnicodeDecodeError, getattrstring, obj, b'\xff') + # CRASHES getattrstring(obj, NULL) + # CRASHES getattrstring(NULL, b'a') + + def test_object_hasattr(self): + xhasattr = _testcapi.object_hasattr + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + self.assertTrue(xhasattr(obj, 'a')) + self.assertFalse(xhasattr(obj, 'b')) + self.assertTrue(xhasattr(obj, '\U0001f40d')) + + self.assertFalse(xhasattr(obj, 'evil')) + self.assertFalse(xhasattr(obj, 1)) + # CRASHES xhasattr(obj, NULL) + # CRASHES xhasattr(NULL, 'a') + + def test_object_hasattrstring(self): + hasattrstring = _testcapi.object_hasattrstring + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + self.assertTrue(hasattrstring(obj, b'a')) + self.assertFalse(hasattrstring(obj, b'b')) + self.assertTrue(hasattrstring(obj, '\U0001f40d'.encode())) + + self.assertFalse(hasattrstring(obj, b'evil')) + self.assertFalse(hasattrstring(obj, b'\xff')) + # CRASHES hasattrstring(obj, NULL) + # CRASHES hasattrstring(NULL, b'a') + + def test_object_setattr(self): + xsetattr = _testcapi.object_setattr + obj = TestObject() + xsetattr(obj, 'a', 5) + self.assertEqual(obj.a, 5) + xsetattr(obj, '\U0001f40d', 8) + self.assertEqual(getattr(obj, '\U0001f40d'), 8) + + # PyObject_SetAttr(obj, attr_name, NULL) removes the attribute + xsetattr(obj, 'a', NULL) + self.assertFalse(hasattr(obj, 'a')) + self.assertRaises(AttributeError, xsetattr, obj, 'b', NULL) + self.assertRaises(RuntimeError, xsetattr, obj, 'evil', NULL) + + self.assertRaises(RuntimeError, xsetattr, obj, 'evil', 'good') + self.assertRaises(AttributeError, xsetattr, 42, 'a', 5) + self.assertRaises(TypeError, xsetattr, obj, 1, 5) + # CRASHES xsetattr(obj, NULL, 5) + # CRASHES xsetattr(NULL, 'a', 5) + + def test_object_setattrstring(self): + setattrstring = _testcapi.object_setattrstring + obj = TestObject() + setattrstring(obj, b'a', 5) + self.assertEqual(obj.a, 5) + setattrstring(obj, '\U0001f40d'.encode(), 8) + self.assertEqual(getattr(obj, '\U0001f40d'), 8) + + # PyObject_SetAttrString(obj, attr_name, NULL) removes the attribute + setattrstring(obj, b'a', NULL) + self.assertFalse(hasattr(obj, 'a')) + self.assertRaises(AttributeError, setattrstring, obj, b'b', NULL) + self.assertRaises(RuntimeError, setattrstring, obj, b'evil', NULL) + + self.assertRaises(RuntimeError, setattrstring, obj, b'evil', 'good') + self.assertRaises(AttributeError, setattrstring, 42, b'a', 5) + self.assertRaises(TypeError, setattrstring, obj, 1, 5) + self.assertRaises(UnicodeDecodeError, setattrstring, obj, b'\xff', 5) + # CRASHES setattrstring(obj, NULL, 5) + # CRASHES setattrstring(NULL, b'a', 5) + + def test_object_delattr(self): + xdelattr = _testcapi.object_delattr + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + xdelattr(obj, 'a') + self.assertFalse(hasattr(obj, 'a')) + self.assertRaises(AttributeError, xdelattr, obj, 'b') + xdelattr(obj, '\U0001f40d') + self.assertFalse(hasattr(obj, '\U0001f40d')) + + self.assertRaises(AttributeError, xdelattr, 42, 'numerator') + self.assertRaises(RuntimeError, xdelattr, obj, 'evil') + self.assertRaises(TypeError, xdelattr, obj, 1) + # CRASHES xdelattr(obj, NULL) + # CRASHES xdelattr(NULL, 'a') + + def test_object_delattrstring(self): + delattrstring = _testcapi.object_delattrstring + obj = TestObject() + obj.a = 1 + setattr(obj, '\U0001f40d', 2) + delattrstring(obj, b'a') + self.assertFalse(hasattr(obj, 'a')) + self.assertRaises(AttributeError, delattrstring, obj, b'b') + delattrstring(obj, '\U0001f40d'.encode()) + self.assertFalse(hasattr(obj, '\U0001f40d')) + + self.assertRaises(AttributeError, delattrstring, 42, b'numerator') + self.assertRaises(RuntimeError, delattrstring, obj, b'evil') + self.assertRaises(UnicodeDecodeError, delattrstring, obj, b'\xff') + # CRASHES delattrstring(obj, NULL) + # CRASHES delattrstring(NULL, b'a') + + + def test_mapping_check(self): + check = _testcapi.mapping_check + self.assertTrue(check({1: 2})) + self.assertTrue(check([1, 2])) + self.assertTrue(check((1, 2))) + self.assertTrue(check('abc')) + self.assertTrue(check(b'abc')) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + self.assertFalse(check(NULL)) + + def test_mapping_size(self): + for size in _testcapi.mapping_size, _testcapi.mapping_length: + self.assertEqual(size({1: 2}), 1) + self.assertEqual(size([1, 2]), 2) + self.assertEqual(size((1, 2)), 2) + self.assertEqual(size('abc'), 3) + self.assertEqual(size(b'abc'), 3) + + self.assertRaises(TypeError, size, 42) + self.assertRaises(TypeError, size, object()) + self.assertRaises(SystemError, size, NULL) + + def test_object_getitem(self): + getitem = _testcapi.object_getitem + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertRaises(KeyError, getitem, dct, 'b') + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = ProxyGetItem(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertRaises(KeyError, getitem, dct2, 'b') + + self.assertEqual(getitem(['a', 'b', 'c'], 1), 'b') + + self.assertRaises(TypeError, getitem, 42, 'a') + self.assertRaises(TypeError, getitem, {}, []) # unhashable + self.assertRaises(SystemError, getitem, {}, NULL) + self.assertRaises(IndexError, getitem, [], 1) + self.assertRaises(TypeError, getitem, [], 'a') + self.assertRaises(SystemError, getitem, NULL, 'a') + + def test_mapping_getitemstring(self): + getitemstring = _testcapi.mapping_getitemstring + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitemstring(dct, b'a'), 1) + self.assertRaises(KeyError, getitemstring, dct, b'b') + self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) + + dct2 = ProxyGetItem(dct) + self.assertEqual(getitemstring(dct2, b'a'), 1) + self.assertRaises(KeyError, getitemstring, dct2, b'b') + + self.assertRaises(TypeError, getitemstring, 42, b'a') + self.assertRaises(UnicodeDecodeError, getitemstring, {}, b'\xff') + self.assertRaises(SystemError, getitemstring, {}, NULL) + self.assertRaises(TypeError, getitemstring, [], b'a') + self.assertRaises(SystemError, getitemstring, NULL, b'a') + + def test_mapping_haskey(self): + haskey = _testcapi.mapping_haskey + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(haskey(dct, 'a')) + self.assertFalse(haskey(dct, 'b')) + self.assertTrue(haskey(dct, '\U0001f40d')) + + dct2 = ProxyGetItem(dct) + self.assertTrue(haskey(dct2, 'a')) + self.assertFalse(haskey(dct2, 'b')) + + self.assertTrue(haskey(['a', 'b', 'c'], 1)) + + self.assertFalse(haskey(42, 'a')) + self.assertFalse(haskey({}, [])) # unhashable + self.assertFalse(haskey({}, NULL)) + self.assertFalse(haskey([], 1)) + self.assertFalse(haskey([], 'a')) + self.assertFalse(haskey(NULL, 'a')) + + def test_mapping_haskeystring(self): + haskeystring = _testcapi.mapping_haskeystring + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(haskeystring(dct, b'a')) + self.assertFalse(haskeystring(dct, b'b')) + self.assertTrue(haskeystring(dct, '\U0001f40d'.encode())) + + dct2 = ProxyGetItem(dct) + self.assertTrue(haskeystring(dct2, b'a')) + self.assertFalse(haskeystring(dct2, b'b')) + + self.assertFalse(haskeystring(42, b'a')) + self.assertFalse(haskeystring({}, b'\xff')) + self.assertFalse(haskeystring({}, NULL)) + self.assertFalse(haskeystring([], b'a')) + self.assertFalse(haskeystring(NULL, b'a')) + + def test_object_setitem(self): + setitem = _testcapi.object_setitem + dct = {} + setitem(dct, 'a', 5) + self.assertEqual(dct, {'a': 5}) + setitem(dct, '\U0001f40d', 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct = {} + dct2 = ProxySetItem(dct) + setitem(dct2, 'a', 5) + self.assertEqual(dct, {'a': 5}) + + lst = ['a', 'b', 'c'] + setitem(lst, 1, 'x') + self.assertEqual(lst, ['a', 'x', 'c']) + + self.assertRaises(TypeError, setitem, 42, 'a', 5) + self.assertRaises(TypeError, setitem, {}, [], 5) # unhashable + self.assertRaises(SystemError, setitem, {}, NULL, 5) + self.assertRaises(SystemError, setitem, {}, 'a', NULL) + self.assertRaises(IndexError, setitem, [], 1, 5) + self.assertRaises(TypeError, setitem, [], 'a', 5) + self.assertRaises(TypeError, setitem, (), 1, 5) + self.assertRaises(SystemError, setitem, NULL, 'a', 5) + + def test_mapping_setitemstring(self): + setitemstring = _testcapi.mapping_setitemstring + dct = {} + setitemstring(dct, b'a', 5) + self.assertEqual(dct, {'a': 5}) + setitemstring(dct, '\U0001f40d'.encode(), 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct = {} + dct2 = ProxySetItem(dct) + setitemstring(dct2, b'a', 5) + self.assertEqual(dct, {'a': 5}) + + self.assertRaises(TypeError, setitemstring, 42, b'a', 5) + self.assertRaises(UnicodeDecodeError, setitemstring, {}, b'\xff', 5) + self.assertRaises(SystemError, setitemstring, {}, NULL, 5) + self.assertRaises(SystemError, setitemstring, {}, b'a', NULL) + self.assertRaises(TypeError, setitemstring, [], b'a', 5) + self.assertRaises(SystemError, setitemstring, NULL, b'a', 5) + + def test_object_delitem(self): + for delitem in _testcapi.object_delitem, _testcapi.mapping_delitem: + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitem(dct, 'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitem, dct, 'b') + delitem(dct, '\U0001f40d') + self.assertEqual(dct, {'c': 2}) + + dct = {'a': 1, 'c': 2} + dct2 = ProxyDelItem(dct) + delitem(dct2, 'a') + self.assertEqual(dct, {'c': 2}) + self.assertRaises(KeyError, delitem, dct2, 'b') + + lst = ['a', 'b', 'c'] + delitem(lst, 1) + self.assertEqual(lst, ['a', 'c']) + + self.assertRaises(TypeError, delitem, 42, 'a') + self.assertRaises(TypeError, delitem, {}, []) # unhashable + self.assertRaises(SystemError, delitem, {}, NULL) + self.assertRaises(IndexError, delitem, [], 1) + self.assertRaises(TypeError, delitem, [], 'a') + self.assertRaises(SystemError, delitem, NULL, 'a') + + def test_mapping_delitemstring(self): + delitemstring = _testcapi.mapping_delitemstring + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitemstring(dct, b'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitemstring, dct, b'b') + delitemstring(dct, '\U0001f40d'.encode()) + self.assertEqual(dct, {'c': 2}) + + dct = {'a': 1, 'c': 2} + dct2 = ProxyDelItem(dct) + delitemstring(dct2, b'a') + self.assertEqual(dct, {'c': 2}) + self.assertRaises(KeyError, delitemstring, dct2, b'b') + + self.assertRaises(TypeError, delitemstring, 42, b'a') + self.assertRaises(UnicodeDecodeError, delitemstring, {}, b'\xff') + self.assertRaises(SystemError, delitemstring, {}, NULL) + self.assertRaises(TypeError, delitemstring, [], b'a') + self.assertRaises(SystemError, delitemstring, NULL, b'a') + + def test_mapping_keys_valuesitems(self): + class Mapping1(dict): + def keys(self): + return list(super().keys()) + def values(self): + return list(super().values()) + def items(self): + return list(super().items()) + class Mapping2(dict): + def keys(self): + return tuple(super().keys()) + def values(self): + return tuple(super().values()) + def items(self): + return tuple(super().items()) + dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} + + for mapping in [{}, OrderedDict(), Mapping1(), Mapping2(), + dict_obj, OrderedDict(dict_obj), + Mapping1(dict_obj), Mapping2(dict_obj)]: + self.assertListEqual(_testcapi.mapping_keys(mapping), + list(mapping.keys())) + self.assertListEqual(_testcapi.mapping_values(mapping), + list(mapping.values())) + self.assertListEqual(_testcapi.mapping_items(mapping), + list(mapping.items())) + + def test_mapping_keys_valuesitems_bad_arg(self): + self.assertRaises(AttributeError, _testcapi.mapping_keys, object()) + self.assertRaises(AttributeError, _testcapi.mapping_values, object()) + self.assertRaises(AttributeError, _testcapi.mapping_items, object()) + self.assertRaises(AttributeError, _testcapi.mapping_keys, []) + self.assertRaises(AttributeError, _testcapi.mapping_values, []) + self.assertRaises(AttributeError, _testcapi.mapping_items, []) + self.assertRaises(SystemError, _testcapi.mapping_keys, NULL) + self.assertRaises(SystemError, _testcapi.mapping_values, NULL) + self.assertRaises(SystemError, _testcapi.mapping_items, NULL) + + class BadMapping: + def keys(self): + return None + def values(self): + return None + def items(self): + return None + bad_mapping = BadMapping() + self.assertRaises(TypeError, _testcapi.mapping_keys, bad_mapping) + self.assertRaises(TypeError, _testcapi.mapping_values, bad_mapping) + self.assertRaises(TypeError, _testcapi.mapping_items, bad_mapping) + + def test_sequence_check(self): + check = _testcapi.sequence_check + self.assertFalse(check({1: 2})) + self.assertTrue(check([1, 2])) + self.assertTrue(check((1, 2))) + self.assertTrue(check('abc')) + self.assertTrue(check(b'abc')) + self.assertFalse(check(42)) + self.assertFalse(check(object())) + # CRASHES check(NULL) + + def test_sequence_size(self): + for size in _testcapi.sequence_size, _testcapi.sequence_length: + self.assertEqual(size([1, 2]), 2) + self.assertEqual(size((1, 2)), 2) + self.assertEqual(size('abc'), 3) + self.assertEqual(size(b'abc'), 3) + + self.assertRaises(TypeError, size, {}) + self.assertRaises(TypeError, size, 42) + self.assertRaises(TypeError, size, object()) + self.assertRaises(SystemError, size, NULL) + + def test_sequence_getitem(self): + getitem = _testcapi.sequence_getitem + lst = ['a', 'b', 'c'] + self.assertEqual(getitem(lst, 1), 'b') + self.assertEqual(getitem(lst, -1), 'c') + self.assertRaises(IndexError, getitem, lst, 3) + + self.assertRaises(TypeError, getitem, 42, 1) + self.assertRaises(TypeError, getitem, {}, 1) + self.assertRaises(SystemError, getitem, NULL, 1) + + def test_sequence_concat(self): + concat = _testcapi.sequence_concat + self.assertEqual(concat(['a', 'b'], [1, 2]), ['a', 'b', 1, 2]) + self.assertEqual(concat(('a', 'b'), (1, 2)), ('a', 'b', 1, 2)) + + self.assertRaises(TypeError, concat, [], ()) + self.assertRaises(TypeError, concat, (), []) + self.assertRaises(TypeError, concat, [], 42) + self.assertRaises(TypeError, concat, 42, []) + self.assertRaises(TypeError, concat, 42, 43) + self.assertRaises(SystemError, concat, [], NULL) + self.assertRaises(SystemError, concat, NULL, []) + + def test_sequence_repeat(self): + repeat = _testcapi.sequence_repeat + self.assertEqual(repeat(['a', 'b'], 2), ['a', 'b', 'a', 'b']) + self.assertEqual(repeat(('a', 'b'), 2), ('a', 'b', 'a', 'b')) + self.assertEqual(repeat(['a', 'b'], 0), []) + self.assertEqual(repeat(['a', 'b'], -1), []) + + self.assertRaises(TypeError, repeat, set(), 2) + self.assertRaises(TypeError, repeat, 42, 2) + self.assertRaises(SystemError, repeat, NULL, 2) + + def test_sequence_inplaceconcat(self): + inplaceconcat = _testcapi.sequence_inplaceconcat + lst = ['a', 'b'] + res = inplaceconcat(lst, [1, 2]) + self.assertEqual(res, ['a', 'b', 1, 2]) + self.assertIs(res, lst) + lst = ['a', 'b'] + res = inplaceconcat(lst, (1, 2)) + self.assertEqual(res, ['a', 'b', 1, 2]) + self.assertIs(res, lst) + self.assertEqual(inplaceconcat(('a', 'b'), (1, 2)), ('a', 'b', 1, 2)) + + self.assertRaises(TypeError, inplaceconcat, (), []) + self.assertRaises(TypeError, inplaceconcat, [], 42) + self.assertRaises(TypeError, inplaceconcat, 42, []) + self.assertRaises(TypeError, inplaceconcat, 42, 43) + self.assertRaises(SystemError, inplaceconcat, [], NULL) + self.assertRaises(SystemError, inplaceconcat, NULL, []) + + def test_sequence_inplacerepeat(self): + inplacerepeat = _testcapi.sequence_inplacerepeat + lst = ['a', 'b'] + res = inplacerepeat(lst, 2) + self.assertEqual(res, ['a', 'b', 'a', 'b']) + self.assertIs(res, lst) + self.assertEqual(inplacerepeat(('a', 'b'), 2), ('a', 'b', 'a', 'b')) + self.assertEqual(inplacerepeat(['a', 'b'], 0), []) + self.assertEqual(inplacerepeat(['a', 'b'], -1), []) + + self.assertRaises(TypeError, inplacerepeat, set(), 2) + self.assertRaises(TypeError, inplacerepeat, 42, 2) + self.assertRaises(SystemError, inplacerepeat, NULL, 2) + + def test_sequence_setitem(self): + setitem = _testcapi.sequence_setitem + lst = ['a', 'b', 'c'] + setitem(lst, 1, 'x') + self.assertEqual(lst, ['a', 'x', 'c']) + setitem(lst, -1, 'y') + self.assertEqual(lst, ['a', 'x', 'y']) + + setitem(lst, 0, NULL) + self.assertEqual(lst, ['x', 'y']) + self.assertRaises(IndexError, setitem, lst, 3, 'x') + + self.assertRaises(TypeError, setitem, 42, 1, 'x') + self.assertRaises(TypeError, setitem, {}, 1, 'x') + self.assertRaises(SystemError, setitem, NULL, 1, 'x') + + def test_sequence_delitem(self): + delitem = _testcapi.sequence_delitem + lst = ['a', 'b', 'c'] + delitem(lst, 1) + self.assertEqual(lst, ['a', 'c']) + delitem(lst, -1) + self.assertEqual(lst, ['a']) + self.assertRaises(IndexError, delitem, lst, 3) + + self.assertRaises(TypeError, delitem, 42, 1) + self.assertRaises(TypeError, delitem, {}, 1) + self.assertRaises(SystemError, delitem, NULL, 1) + + def test_sequence_setslice(self): + setslice = _testcapi.sequence_setslice + + # Correct case: + data = [1, 2, 3, 4, 5] + data_copy = data.copy() + + setslice(data, 1, 3, [8, 9]) + data_copy[1:3] = [8, 9] + self.assertEqual(data, data_copy) + self.assertEqual(data, [1, 8, 9, 4, 5]) + + # Custom class: + class Custom: + def __setitem__(self, index, value): + self.index = index + self.value = value + + c = Custom() + setslice(c, 0, 5, 'abc') + self.assertEqual(c.index, slice(0, 5)) + self.assertEqual(c.value, 'abc') + + # Immutable sequences must raise: + bad_seq1 = (1, 2, 3, 4) + self.assertRaises(TypeError, setslice, bad_seq1, 1, 3, (8, 9)) + self.assertEqual(bad_seq1, (1, 2, 3, 4)) + + bad_seq2 = 'abcd' + self.assertRaises(TypeError, setslice, bad_seq2, 1, 3, 'xy') + self.assertEqual(bad_seq2, 'abcd') + + # Not a sequence: + self.assertRaises(TypeError, setslice, object(), 1, 3, 'xy') + self.assertRaises(SystemError, setslice, NULL, 1, 3, 'xy') + + data_copy = data.copy() + setslice(data_copy, 1, 3, NULL) + self.assertEqual(data_copy, [1, 4, 5]) + + def test_sequence_delslice(self): + delslice = _testcapi.sequence_delslice + + # Correct case: + data = [1, 2, 3, 4, 5] + data_copy = data.copy() + + delslice(data, 1, 3) + del data_copy[1:3] + self.assertEqual(data, data_copy) + self.assertEqual(data, [1, 4, 5]) + + # Custom class: + class Custom: + def __delitem__(self, index): + self.index = index + + c = Custom() + delslice(c, 0, 5) + self.assertEqual(c.index, slice(0, 5)) + + # Immutable sequences must raise: + bad_seq1 = (1, 2, 3, 4) + self.assertRaises(TypeError, delslice, bad_seq1, 1, 3) + self.assertEqual(bad_seq1, (1, 2, 3, 4)) + + bad_seq2 = 'abcd' + self.assertRaises(TypeError, delslice, bad_seq2, 1, 3) + self.assertEqual(bad_seq2, 'abcd') + + # Not a sequence: + self.assertRaises(TypeError, delslice, object(), 1, 3) + self.assertRaises(SystemError, delslice, NULL, 1, 3) + + mapping = {1: 'a', 2: 'b', 3: 'c'} + self.assertRaises(KeyError, delslice, mapping, 1, 3) + self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) + + def test_sequence_count(self): + count = _testcapi.sequence_count + + lst = ['a', 'b', 'a'] + self.assertEqual(count(lst, 'a'), 2) + self.assertEqual(count(lst, 'c'), 0) + self.assertEqual(count(iter(lst), 'a'), 2) + self.assertEqual(count(iter(lst), 'c'), 0) + self.assertEqual(count({'a': 2}, 'a'), 1) + + self.assertRaises(TypeError, count, 42, 'a') + self.assertRaises(SystemError, count, [], NULL) + self.assertRaises(SystemError, count, [1], NULL) + self.assertRaises(SystemError, count, NULL, 'a') + + def test_sequence_contains(self): + contains = _testcapi.sequence_contains + + lst = ['a', 'b', 'a'] + self.assertEqual(contains(lst, 'a'), 1) + self.assertEqual(contains(lst, 'c'), 0) + self.assertEqual(contains(iter(lst), 'a'), 1) + self.assertEqual(contains(iter(lst), 'c'), 0) + self.assertEqual(contains({'a': 2}, 'a'), 1) + + # XXX Only for empty sequences. Should be SystemError? + self.assertEqual(contains([], NULL), 0) + + self.assertRaises(TypeError, contains, 42, 'a') + self.assertRaises(SystemError, contains, [1], NULL) + # CRASHES contains({}, NULL) + # CRASHES contains(set(), NULL) + # CRASHES contains(NULL, 'a') + + def test_sequence_index(self): + index = _testcapi.sequence_index + + lst = ['a', 'b', 'a'] + self.assertEqual(index(lst, 'a'), 0) + self.assertEqual(index(lst, 'b'), 1) + self.assertRaises(ValueError, index, lst, 'c') + self.assertEqual(index(iter(lst), 'a'), 0) + self.assertEqual(index(iter(lst), 'b'), 1) + self.assertRaises(ValueError, index, iter(lst), 'c') + dct = {'a': 2, 'b': 3} + self.assertEqual(index(dct, 'a'), 0) + self.assertEqual(index(dct, 'b'), 1) + self.assertRaises(ValueError, index, dct, 'c') + + self.assertRaises(TypeError, index, 42, 'a') + self.assertRaises(SystemError, index, [], NULL) + self.assertRaises(SystemError, index, [1], NULL) + self.assertRaises(SystemError, index, NULL, 'a') + + def test_sequence_list(self): + xlist = _testcapi.sequence_list + self.assertEqual(xlist(['a', 'b', 'c']), ['a', 'b', 'c']) + self.assertEqual(xlist(('a', 'b', 'c')), ['a', 'b', 'c']) + self.assertEqual(xlist(iter(['a', 'b', 'c'])), ['a', 'b', 'c']) + self.assertEqual(xlist(gen()), ['a', 'b', 'c']) + + self.assertRaises(TypeError, xlist, 42) + self.assertRaises(SystemError, xlist, NULL) + + def test_sequence_tuple(self): + xtuple = _testcapi.sequence_tuple + self.assertEqual(xtuple(['a', 'b', 'c']), ('a', 'b', 'c')) + self.assertEqual(xtuple(('a', 'b', 'c')), ('a', 'b', 'c')) + self.assertEqual(xtuple(iter(['a', 'b', 'c'])), ('a', 'b', 'c')) + self.assertEqual(xtuple(gen()), ('a', 'b', 'c')) + + self.assertRaises(TypeError, xtuple, 42) + self.assertRaises(SystemError, xtuple, NULL) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py new file mode 100644 index 0000000000000..0717708fcce8e --- /dev/null +++ b/Lib/test/test_capi/test_dict.py @@ -0,0 +1,378 @@ +import unittest +import sys +from collections import OrderedDict, UserDict +from types import MappingProxyType +from test import support +from test.support import import_helper +import _testcapi + + +NULL = None + +class DictSubclass(dict): + def __getitem__(self, key): + raise RuntimeError('do not get evil') + def __setitem__(self, key, value): + raise RuntimeError('do not set evil') + def __delitem__(self, key): + raise RuntimeError('do not del evil') + +def gen(): + yield 'a' + yield 'b' + yield 'c' + + +class CAPITest(unittest.TestCase): + + def test_dict_check(self): + check = _testcapi.dict_check + self.assertTrue(check({1: 2})) + self.assertTrue(check(OrderedDict({1: 2}))) + self.assertFalse(check(UserDict({1: 2}))) + self.assertFalse(check([1, 2])) + self.assertFalse(check(object())) + #self.assertFalse(check(NULL)) + + def test_dict_checkexact(self): + check = _testcapi.dict_checkexact + self.assertTrue(check({1: 2})) + self.assertFalse(check(OrderedDict({1: 2}))) + self.assertFalse(check(UserDict({1: 2}))) + self.assertFalse(check([1, 2])) + self.assertFalse(check(object())) + #self.assertFalse(check(NULL)) + + def test_dict_new(self): + dict_new = _testcapi.dict_new + dct = dict_new() + self.assertEqual(dct, {}) + self.assertIs(type(dct), dict) + dct2 = dict_new() + self.assertIsNot(dct2, dct) + + def test_dictproxy_new(self): + dictproxy_new = _testcapi.dictproxy_new + for dct in {1: 2}, OrderedDict({1: 2}), UserDict({1: 2}): + proxy = dictproxy_new(dct) + self.assertIs(type(proxy), MappingProxyType) + self.assertEqual(proxy, dct) + with self.assertRaises(TypeError): + proxy[1] = 3 + self.assertEqual(proxy[1], 2) + dct[1] = 4 + self.assertEqual(proxy[1], 4) + + self.assertRaises(TypeError, dictproxy_new, []) + self.assertRaises(TypeError, dictproxy_new, 42) + # CRASHES dictproxy_new(NULL) + + def test_dict_copy(self): + copy = _testcapi.dict_copy + for dct in {1: 2}, OrderedDict({1: 2}): + dct_copy = copy(dct) + self.assertIs(type(dct_copy), dict) + self.assertEqual(dct_copy, dct) + + self.assertRaises(SystemError, copy, UserDict()) + self.assertRaises(SystemError, copy, []) + self.assertRaises(SystemError, copy, 42) + self.assertRaises(SystemError, copy, NULL) + + def test_dict_clear(self): + clear = _testcapi.dict_clear + dct = {1: 2} + clear(dct) + self.assertEqual(dct, {}) + + # NOTE: It is not safe to call it with OrderedDict. + + # Has no effect for non-dicts. + dct = UserDict({1: 2}) + clear(dct) + self.assertEqual(dct, {1: 2}) + lst = [1, 2] + clear(lst) + self.assertEqual(lst, [1, 2]) + clear(object()) + + # CRASHES? clear(NULL) + + def test_dict_size(self): + size = _testcapi.dict_size + self.assertEqual(size({1: 2}), 1) + self.assertEqual(size(OrderedDict({1: 2})), 1) + + self.assertRaises(SystemError, size, UserDict()) + self.assertRaises(SystemError, size, []) + self.assertRaises(SystemError, size, 42) + self.assertRaises(SystemError, size, object()) + self.assertRaises(SystemError, size, NULL) + + def test_dict_getitem(self): + getitem = _testcapi.dict_getitem + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertIs(getitem(dct, 'b'), KeyError) + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertIs(getitem(dct2, 'b'), KeyError) + + self.assertIs(getitem({}, []), KeyError) # unhashable + self.assertIs(getitem(42, 'a'), KeyError) + self.assertIs(getitem([1], 0), KeyError) + # CRASHES getitem({}, NULL) + # CRASHES getitem(NULL, 'a') + + def test_dict_getitemstring(self): + getitemstring = _testcapi.dict_getitemstring + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitemstring(dct, b'a'), 1) + self.assertIs(getitemstring(dct, b'b'), KeyError) + self.assertEqual(getitemstring(dct, '\U0001f40d'.encode()), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitemstring(dct2, b'a'), 1) + self.assertIs(getitemstring(dct2, b'b'), KeyError) + + self.assertIs(getitemstring({}, b'\xff'), KeyError) + self.assertIs(getitemstring(42, b'a'), KeyError) + self.assertIs(getitemstring([], b'a'), KeyError) + # CRASHES getitemstring({}, NULL) + # CRASHES getitemstring(NULL, b'a') + + def test_dict_getitemwitherror(self): + getitem = _testcapi.dict_getitemwitherror + dct = {'a': 1, '\U0001f40d': 2} + self.assertEqual(getitem(dct, 'a'), 1) + self.assertIs(getitem(dct, 'b'), KeyError) + self.assertEqual(getitem(dct, '\U0001f40d'), 2) + + dct2 = DictSubclass(dct) + self.assertEqual(getitem(dct2, 'a'), 1) + self.assertIs(getitem(dct2, 'b'), KeyError) + + self.assertRaises(SystemError, getitem, 42, 'a') + self.assertRaises(TypeError, getitem, {}, []) # unhashable + self.assertRaises(SystemError, getitem, [], 1) + self.assertRaises(SystemError, getitem, [], 'a') + # CRASHES getitem({}, NULL) + # CRASHES getitem(NULL, 'a') + + def test_dict_contains(self): + contains = _testcapi.dict_contains + dct = {'a': 1, '\U0001f40d': 2} + self.assertTrue(contains(dct, 'a')) + self.assertFalse(contains(dct, 'b')) + self.assertTrue(contains(dct, '\U0001f40d')) + + dct2 = DictSubclass(dct) + self.assertTrue(contains(dct2, 'a')) + self.assertFalse(contains(dct2, 'b')) + + self.assertRaises(TypeError, contains, {}, []) # unhashable + # CRASHES contains({}, NULL) + # CRASHES contains(UserDict(), 'a') + # CRASHES contains(42, 'a') + # CRASHES contains(NULL, 'a') + + def test_dict_setitem(self): + setitem = _testcapi.dict_setitem + dct = {} + setitem(dct, 'a', 5) + self.assertEqual(dct, {'a': 5}) + setitem(dct, '\U0001f40d', 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct2 = DictSubclass() + setitem(dct2, 'a', 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(TypeError, setitem, {}, [], 5) # unhashable + self.assertRaises(SystemError, setitem, UserDict(), 'a', 5) + self.assertRaises(SystemError, setitem, [1], 0, 5) + self.assertRaises(SystemError, setitem, 42, 'a', 5) + # CRASHES setitem({}, NULL, 5) + # CRASHES setitem({}, 'a', NULL) + # CRASHES setitem(NULL, 'a', 5) + + def test_dict_setitemstring(self): + setitemstring = _testcapi.dict_setitemstring + dct = {} + setitemstring(dct, b'a', 5) + self.assertEqual(dct, {'a': 5}) + setitemstring(dct, '\U0001f40d'.encode(), 8) + self.assertEqual(dct, {'a': 5, '\U0001f40d': 8}) + + dct2 = DictSubclass() + setitemstring(dct2, b'a', 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(UnicodeDecodeError, setitemstring, {}, b'\xff', 5) + self.assertRaises(SystemError, setitemstring, UserDict(), b'a', 5) + self.assertRaises(SystemError, setitemstring, 42, b'a', 5) + # CRASHES setitemstring({}, NULL, 5) + # CRASHES setitemstring({}, b'a', NULL) + # CRASHES setitemstring(NULL, b'a', 5) + + def test_dict_delitem(self): + delitem = _testcapi.dict_delitem + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitem(dct, 'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitem, dct, 'b') + delitem(dct, '\U0001f40d') + self.assertEqual(dct, {'c': 2}) + + dct2 = DictSubclass({'a': 1, 'c': 2}) + delitem(dct2, 'a') + self.assertEqual(dct2, {'c': 2}) + self.assertRaises(KeyError, delitem, dct2, 'b') + + self.assertRaises(TypeError, delitem, {}, []) # unhashable + self.assertRaises(SystemError, delitem, UserDict({'a': 1}), 'a') + self.assertRaises(SystemError, delitem, [1], 0) + self.assertRaises(SystemError, delitem, 42, 'a') + # CRASHES delitem({}, NULL) + # CRASHES delitem(NULL, 'a') + + def test_dict_delitemstring(self): + delitemstring = _testcapi.dict_delitemstring + dct = {'a': 1, 'c': 2, '\U0001f40d': 3} + delitemstring(dct, b'a') + self.assertEqual(dct, {'c': 2, '\U0001f40d': 3}) + self.assertRaises(KeyError, delitemstring, dct, b'b') + delitemstring(dct, '\U0001f40d'.encode()) + self.assertEqual(dct, {'c': 2}) + + dct2 = DictSubclass({'a': 1, 'c': 2}) + delitemstring(dct2, b'a') + self.assertEqual(dct2, {'c': 2}) + self.assertRaises(KeyError, delitemstring, dct2, b'b') + + self.assertRaises(UnicodeDecodeError, delitemstring, {}, b'\xff') + self.assertRaises(SystemError, delitemstring, UserDict({'a': 1}), b'a') + self.assertRaises(SystemError, delitemstring, 42, b'a') + # CRASHES delitemstring({}, NULL) + # CRASHES delitemstring(NULL, b'a') + + def test_dict_setdefault(self): + setdefault = _testcapi.dict_setdefault + dct = {} + self.assertEqual(setdefault(dct, 'a', 5), 5) + self.assertEqual(dct, {'a': 5}) + self.assertEqual(setdefault(dct, 'a', 8), 5) + self.assertEqual(dct, {'a': 5}) + + dct2 = DictSubclass() + self.assertEqual(setdefault(dct2, 'a', 5), 5) + self.assertEqual(dct2, {'a': 5}) + self.assertEqual(setdefault(dct2, 'a', 8), 5) + self.assertEqual(dct2, {'a': 5}) + + self.assertRaises(TypeError, setdefault, {}, [], 5) # unhashable + self.assertRaises(SystemError, setdefault, UserDict(), 'a', 5) + self.assertRaises(SystemError, setdefault, [1], 0, 5) + self.assertRaises(SystemError, setdefault, 42, 'a', 5) + # CRASHES setdefault({}, NULL, 5) + # CRASHES setdefault({}, 'a', NULL) + # CRASHES setdefault(NULL, 'a', 5) + + def test_mapping_keys_valuesitems(self): + class BadMapping(dict): + def keys(self): + return None + def values(self): + return None + def items(self): + return None + dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} + for mapping in [dict_obj, DictSubclass(dict_obj), BadMapping(dict_obj)]: + self.assertListEqual(_testcapi.dict_keys(mapping), + list(dict_obj.keys())) + self.assertListEqual(_testcapi.dict_values(mapping), + list(dict_obj.values())) + self.assertListEqual(_testcapi.dict_items(mapping), + list(dict_obj.items())) + + def test_dict_keys_valuesitems_bad_arg(self): + for mapping in UserDict(), [], object(): + self.assertRaises(SystemError, _testcapi.dict_keys, mapping) + self.assertRaises(SystemError, _testcapi.dict_values, mapping) + self.assertRaises(SystemError, _testcapi.dict_items, mapping) + + def test_dict_next(self): + dict_next = _testcapi.dict_next + self.assertIsNone(dict_next({}, 0)) + dct = {'a': 1, 'b': 2, 'c': 3} + pos = 0 + pairs = [] + while True: + res = dict_next(dct, pos) + if res is None: + break + rc, pos, key, value = res + self.assertEqual(rc, 1) + pairs.append((key, value)) + self.assertEqual(pairs, list(dct.items())) + + # CRASHES dict_next(NULL, 0) + + def test_dict_update(self): + update = _testcapi.dict_update + for cls1 in dict, DictSubclass: + for cls2 in dict, DictSubclass, UserDict: + dct = cls1({'a': 1, 'b': 2}) + update(dct, cls2({'b': 3, 'c': 4})) + self.assertEqual(dct, {'a': 1, 'b': 3, 'c': 4}) + + self.assertRaises(AttributeError, update, {}, []) + self.assertRaises(AttributeError, update, {}, 42) + self.assertRaises(SystemError, update, UserDict(), {}) + self.assertRaises(SystemError, update, 42, {}) + self.assertRaises(SystemError, update, {}, NULL) + self.assertRaises(SystemError, update, NULL, {}) + + def test_dict_merge(self): + merge = _testcapi.dict_merge + for cls1 in dict, DictSubclass: + for cls2 in dict, DictSubclass, UserDict: + dct = cls1({'a': 1, 'b': 2}) + merge(dct, cls2({'b': 3, 'c': 4}), 0) + self.assertEqual(dct, {'a': 1, 'b': 2, 'c': 4}) + dct = cls1({'a': 1, 'b': 2}) + merge(dct, cls2({'b': 3, 'c': 4}), 1) + self.assertEqual(dct, {'a': 1, 'b': 3, 'c': 4}) + + self.assertRaises(AttributeError, merge, {}, [], 0) + self.assertRaises(AttributeError, merge, {}, 42, 0) + self.assertRaises(SystemError, merge, UserDict(), {}, 0) + self.assertRaises(SystemError, merge, 42, {}, 0) + self.assertRaises(SystemError, merge, {}, NULL, 0) + self.assertRaises(SystemError, merge, NULL, {}, 0) + + def test_dict_mergefromseq2(self): + mergefromseq2 = _testcapi.dict_mergefromseq2 + for cls1 in dict, DictSubclass: + for cls2 in list, iter: + dct = cls1({'a': 1, 'b': 2}) + mergefromseq2(dct, cls2([('b', 3), ('c', 4)]), 0) + self.assertEqual(dct, {'a': 1, 'b': 2, 'c': 4}) + dct = cls1({'a': 1, 'b': 2}) + mergefromseq2(dct, cls2([('b', 3), ('c', 4)]), 1) + self.assertEqual(dct, {'a': 1, 'b': 3, 'c': 4}) + + self.assertRaises(ValueError, mergefromseq2, {}, [(1,)], 0) + self.assertRaises(ValueError, mergefromseq2, {}, [(1, 2, 3)], 0) + self.assertRaises(TypeError, mergefromseq2, {}, [1], 0) + self.assertRaises(TypeError, mergefromseq2, {}, 42, 0) + # CRASHES mergefromseq2(UserDict(), [], 0) + # CRASHES mergefromseq2(42, [], 0) + # CRASHES mergefromseq2({}, NULL, 0) + # CRASHES mergefromseq2(NULL, {}, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 3d494975c71e2..2a71ac533d9e0 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -301,137 +301,6 @@ def test_getitem_with_error(self): def test_buildvalue_N(self): _testcapi.test_buildvalue_N() - def test_mapping_keys_values_items(self): - class Mapping1(dict): - def keys(self): - return list(super().keys()) - def values(self): - return list(super().values()) - def items(self): - return list(super().items()) - class Mapping2(dict): - def keys(self): - return tuple(super().keys()) - def values(self): - return tuple(super().values()) - def items(self): - return tuple(super().items()) - dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} - - for mapping in [{}, OrderedDict(), Mapping1(), Mapping2(), - dict_obj, OrderedDict(dict_obj), - Mapping1(dict_obj), Mapping2(dict_obj)]: - self.assertListEqual(_testcapi.get_mapping_keys(mapping), - list(mapping.keys())) - self.assertListEqual(_testcapi.get_mapping_values(mapping), - list(mapping.values())) - self.assertListEqual(_testcapi.get_mapping_items(mapping), - list(mapping.items())) - - def test_mapping_keys_values_items_bad_arg(self): - self.assertRaises(AttributeError, _testcapi.get_mapping_keys, None) - self.assertRaises(AttributeError, _testcapi.get_mapping_values, None) - self.assertRaises(AttributeError, _testcapi.get_mapping_items, None) - - class BadMapping: - def keys(self): - return None - def values(self): - return None - def items(self): - return None - bad_mapping = BadMapping() - self.assertRaises(TypeError, _testcapi.get_mapping_keys, bad_mapping) - self.assertRaises(TypeError, _testcapi.get_mapping_values, bad_mapping) - self.assertRaises(TypeError, _testcapi.get_mapping_items, bad_mapping) - - def test_mapping_has_key(self): - dct = {'a': 1} - self.assertTrue(_testcapi.mapping_has_key(dct, 'a')) - self.assertFalse(_testcapi.mapping_has_key(dct, 'b')) - - class SubDict(dict): - pass - - dct2 = SubDict({'a': 1}) - self.assertTrue(_testcapi.mapping_has_key(dct2, 'a')) - self.assertFalse(_testcapi.mapping_has_key(dct2, 'b')) - - def test_sequence_set_slice(self): - # Correct case: - data = [1, 2, 3, 4, 5] - data_copy = data.copy() - - _testcapi.sequence_set_slice(data, 1, 3, [8, 9]) - data_copy[1:3] = [8, 9] - self.assertEqual(data, data_copy) - self.assertEqual(data, [1, 8, 9, 4, 5]) - - # Custom class: - class Custom: - def __setitem__(self, index, value): - self.index = index - self.value = value - - c = Custom() - _testcapi.sequence_set_slice(c, 0, 5, 'abc') - self.assertEqual(c.index, slice(0, 5)) - self.assertEqual(c.value, 'abc') - - # Immutable sequences must raise: - bad_seq1 = (1, 2, 3, 4) - with self.assertRaises(TypeError): - _testcapi.sequence_set_slice(bad_seq1, 1, 3, (8, 9)) - self.assertEqual(bad_seq1, (1, 2, 3, 4)) - - bad_seq2 = 'abcd' - with self.assertRaises(TypeError): - _testcapi.sequence_set_slice(bad_seq2, 1, 3, 'xy') - self.assertEqual(bad_seq2, 'abcd') - - # Not a sequence: - with self.assertRaises(TypeError): - _testcapi.sequence_set_slice(None, 1, 3, 'xy') - - def test_sequence_del_slice(self): - # Correct case: - data = [1, 2, 3, 4, 5] - data_copy = data.copy() - - _testcapi.sequence_del_slice(data, 1, 3) - del data_copy[1:3] - self.assertEqual(data, data_copy) - self.assertEqual(data, [1, 4, 5]) - - # Custom class: - class Custom: - def __delitem__(self, index): - self.index = index - - c = Custom() - _testcapi.sequence_del_slice(c, 0, 5) - self.assertEqual(c.index, slice(0, 5)) - - # Immutable sequences must raise: - bad_seq1 = (1, 2, 3, 4) - with self.assertRaises(TypeError): - _testcapi.sequence_del_slice(bad_seq1, 1, 3) - self.assertEqual(bad_seq1, (1, 2, 3, 4)) - - bad_seq2 = 'abcd' - with self.assertRaises(TypeError): - _testcapi.sequence_del_slice(bad_seq2, 1, 3) - self.assertEqual(bad_seq2, 'abcd') - - # Not a sequence: - with self.assertRaises(TypeError): - _testcapi.sequence_del_slice(None, 1, 3) - - mapping = {1: 'a', 2: 'b', 3: 'c'} - with self.assertRaises(KeyError): - _testcapi.sequence_del_slice(mapping, 1, 3) - self.assertEqual(mapping, {1: 'a', 2: 'b', 3: 'c'}) - @unittest.skipUnless(hasattr(_testcapi, 'negative_refcount'), 'need _testcapi.negative_refcount') def test_negative_refcount(self): diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index d7a48e55b1018..7a159760e98e2 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -455,8 +455,8 @@ def __init__(self): self.attr = 1 a = A() - self.assertEqual(_testcapi.hasattr_string(a, "attr"), True) - self.assertEqual(_testcapi.hasattr_string(a, "noattr"), False) + self.assertEqual(_testcapi.object_hasattrstring(a, b"attr"), 1) + self.assertEqual(_testcapi.object_hasattrstring(a, b"noattr"), 0) self.assertIsNone(sys.exception()) def testDel(self): diff --git a/Misc/NEWS.d/next/Tests/2023-07-24-16-56-59.gh-issue-107178.Gq1usE.rst b/Misc/NEWS.d/next/Tests/2023-07-24-16-56-59.gh-issue-107178.Gq1usE.rst new file mode 100644 index 0000000000000..dd6becf6b0013 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-07-24-16-56-59.gh-issue-107178.Gq1usE.rst @@ -0,0 +1,2 @@ +Add the C API test for functions in the Mapping Protocol, the Sequence +Protocol and some functions in the Object Protocol. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 0fc7955cd9b7c..3e01e25056dfa 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -168,7 +168,7 @@ @MODULE__XXTESTFUZZ_TRUE at _xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE at _testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE at _testinternalcapi _testinternalcapi.c - at MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c + at MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c _testcapi/gc.c @MODULE__TESTCLINIC_TRUE at _testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/abstract.c b/Modules/_testcapi/abstract.c new file mode 100644 index 0000000000000..9715efb821740 --- /dev/null +++ b/Modules/_testcapi/abstract.c @@ -0,0 +1,535 @@ +#include // ptrdiff_t + +#define PY_SSIZE_T_CLEAN +#include "parts.h" + +#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromLong(_ret); \ + } while (0) + +#define RETURN_SIZE(value) do { \ + Py_ssize_t _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromSsize_t(_ret); \ + } while (0) + + +static PyObject * +object_getattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + return PyObject_GetAttr(obj, attr_name); +} + +static PyObject * +object_getattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + return PyObject_GetAttrString(obj, attr_name); +} + +static PyObject * +object_hasattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; + if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + return PyLong_FromLong(PyObject_HasAttr(obj, attr_name)); +} + +static PyObject * +object_hasattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + return PyLong_FromLong(PyObject_HasAttrString(obj, attr_name)); +} + +static PyObject * +object_setattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name, *value; + if (!PyArg_ParseTuple(args, "OOO", &obj, &attr_name, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + NULLABLE(value); + RETURN_INT(PyObject_SetAttr(obj, attr_name, value)); +} + +static PyObject * +object_setattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj, *value; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &obj, &attr_name, &size, &value)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(value); + RETURN_INT(PyObject_SetAttrString(obj, attr_name, value)); +} + +static PyObject * +object_delattr(PyObject *self, PyObject *args) +{ + PyObject *obj, *attr_name; +if (!PyArg_ParseTuple(args, "OO", &obj, &attr_name)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(attr_name); + RETURN_INT(PyObject_DelAttr(obj, attr_name)); +} + +static PyObject * +object_delattrstring(PyObject *self, PyObject *args) +{ + PyObject *obj; + const char *attr_name; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &obj, &attr_name, &size)) { + return NULL; + } + NULLABLE(obj); + RETURN_INT(PyObject_DelAttrString(obj, attr_name)); +} + + +static PyObject * +mapping_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyMapping_Check(obj)); +} + +static PyObject * +mapping_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyMapping_Size(obj)); +} + +static PyObject * +mapping_length(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyMapping_Length(obj)); +} + +static PyObject * +object_getitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + return PyObject_GetItem(mapping, key); +} + +static PyObject * +mapping_getitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + return PyMapping_GetItemString(mapping, key); +} + +static PyObject * +mapping_haskey(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + return PyLong_FromLong(PyMapping_HasKey(mapping, key)); +} + +static PyObject * +mapping_haskeystring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + return PyLong_FromLong(PyMapping_HasKeyString(mapping, key)); +} + +static PyObject * +object_setitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(value); + RETURN_INT(PyObject_SetItem(mapping, key, value)); +} + +static PyObject * +mapping_setitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping, *value; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(value); + RETURN_INT(PyMapping_SetItemString(mapping, key, value)); +} + +static PyObject * +object_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyObject_DelItem(mapping, key)); +} + +static PyObject * +mapping_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyMapping_DelItem(mapping, key)); +} + +static PyObject * +mapping_delitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyMapping_DelItemString(mapping, key)); +} + +static PyObject * +mapping_keys(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Keys(obj); +} + +static PyObject * +mapping_values(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Values(obj); +} + +static PyObject * +mapping_items(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyMapping_Items(obj); +} + + +static PyObject * +sequence_check(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PySequence_Check(obj)); +} + +static PyObject * +sequence_size(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySequence_Size(obj)); +} + +static PyObject * +sequence_length(PyObject* self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PySequence_Length(obj)); +} + +static PyObject * +sequence_concat(PyObject *self, PyObject *args) +{ + PyObject *seq1, *seq2; + if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + return NULL; + } + NULLABLE(seq1); + NULLABLE(seq2); + + return PySequence_Concat(seq1, seq2); +} + +static PyObject * +sequence_repeat(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t count; + if (!PyArg_ParseTuple(args, "On", &seq, &count)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_Repeat(seq, count); +} + +static PyObject * +sequence_inplaceconcat(PyObject *self, PyObject *args) +{ + PyObject *seq1, *seq2; + if (!PyArg_ParseTuple(args, "OO", &seq1, &seq2)) { + return NULL; + } + NULLABLE(seq1); + NULLABLE(seq2); + + return PySequence_InPlaceConcat(seq1, seq2); +} + +static PyObject * +sequence_inplacerepeat(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t count; + if (!PyArg_ParseTuple(args, "On", &seq, &count)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_InPlaceRepeat(seq, count); +} + +static PyObject * +sequence_getitem(PyObject *self, PyObject *args) +{ + PyObject *seq; + Py_ssize_t i; + if (!PyArg_ParseTuple(args, "On", &seq, &i)) { + return NULL; + } + NULLABLE(seq); + + return PySequence_GetItem(seq, i); +} + +static PyObject * +sequence_setitem(PyObject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *seq, *val; + if (!PyArg_ParseTuple(args, "OnO", &seq, &i, &val)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(val); + + RETURN_INT(PySequence_SetItem(seq, i, val)); +} + + +static PyObject * +sequence_delitem(PyObject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *seq; + if (!PyArg_ParseTuple(args, "On", &seq, &i)) { + return NULL; + } + NULLABLE(seq); + + RETURN_INT(PySequence_DelItem(seq, i)); +} + +static PyObject * +sequence_setslice(PyObject* self, PyObject *args) +{ + PyObject *sequence, *obj; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "OnnO", &sequence, &i1, &i2, &obj)) { + return NULL; + } + NULLABLE(sequence); + NULLABLE(obj); + + RETURN_INT(PySequence_SetSlice(sequence, i1, i2, obj)); +} + +static PyObject * +sequence_delslice(PyObject *self, PyObject *args) +{ + PyObject *sequence; + Py_ssize_t i1, i2; + if (!PyArg_ParseTuple(args, "Onn", &sequence, &i1, &i2)) { + return NULL; + } + NULLABLE(sequence); + + RETURN_INT(PySequence_DelSlice(sequence, i1, i2)); +} + +static PyObject * +sequence_count(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_SIZE(PySequence_Count(seq, value)); +} + +static PyObject * +sequence_contains(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_INT(PySequence_Contains(seq, value)); +} + +static PyObject * +sequence_index(PyObject *self, PyObject *args) +{ + PyObject *seq, *value; + if (!PyArg_ParseTuple(args, "OO", &seq, &value)) { + return NULL; + } + NULLABLE(seq); + NULLABLE(value); + + RETURN_SIZE(PySequence_Index(seq, value)); +} + +static PyObject * +sequence_list(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySequence_List(obj); +} + +static PyObject * +sequence_tuple(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PySequence_Tuple(obj); +} + + +static PyMethodDef test_methods[] = { + {"object_getattr", object_getattr, METH_VARARGS}, + {"object_getattrstring", object_getattrstring, METH_VARARGS}, + {"object_hasattr", object_hasattr, METH_VARARGS}, + {"object_hasattrstring", object_hasattrstring, METH_VARARGS}, + {"object_setattr", object_setattr, METH_VARARGS}, + {"object_setattrstring", object_setattrstring, METH_VARARGS}, + {"object_delattr", object_delattr, METH_VARARGS}, + {"object_delattrstring", object_delattrstring, METH_VARARGS}, + + {"mapping_check", mapping_check, METH_O}, + {"mapping_size", mapping_size, METH_O}, + {"mapping_length", mapping_length, METH_O}, + {"object_getitem", object_getitem, METH_VARARGS}, + {"mapping_getitemstring", mapping_getitemstring, METH_VARARGS}, + {"mapping_haskey", mapping_haskey, METH_VARARGS}, + {"mapping_haskeystring", mapping_haskeystring, METH_VARARGS}, + {"object_setitem", object_setitem, METH_VARARGS}, + {"mapping_setitemstring", mapping_setitemstring, METH_VARARGS}, + {"object_delitem", object_delitem, METH_VARARGS}, + {"mapping_delitem", mapping_delitem, METH_VARARGS}, + {"mapping_delitemstring", mapping_delitemstring, METH_VARARGS}, + {"mapping_keys", mapping_keys, METH_O}, + {"mapping_values", mapping_values, METH_O}, + {"mapping_items", mapping_items, METH_O}, + + {"sequence_check", sequence_check, METH_O}, + {"sequence_size", sequence_size, METH_O}, + {"sequence_length", sequence_length, METH_O}, + {"sequence_concat", sequence_concat, METH_VARARGS}, + {"sequence_repeat", sequence_repeat, METH_VARARGS}, + {"sequence_inplaceconcat", sequence_inplaceconcat, METH_VARARGS}, + {"sequence_inplacerepeat", sequence_inplacerepeat, METH_VARARGS}, + {"sequence_getitem", sequence_getitem, METH_VARARGS}, + {"sequence_setitem", sequence_setitem, METH_VARARGS}, + {"sequence_delitem", sequence_delitem, METH_VARARGS}, + {"sequence_setslice", sequence_setslice, METH_VARARGS}, + {"sequence_delslice", sequence_delslice, METH_VARARGS}, + {"sequence_count", sequence_count, METH_VARARGS}, + {"sequence_contains", sequence_contains, METH_VARARGS}, + {"sequence_index", sequence_index, METH_VARARGS}, + {"sequence_list", sequence_list, METH_O}, + {"sequence_tuple", sequence_tuple, METH_O}, + + {NULL}, +}; + +int +_PyTestCapi_Init_Abstract(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c new file mode 100644 index 0000000000000..c0f26e799b5f1 --- /dev/null +++ b/Modules/_testcapi/dict.c @@ -0,0 +1,323 @@ +#include // ptrdiff_t + +#define PY_SSIZE_T_CLEAN +#include "parts.h" + +#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); + +#define RETURN_INT(value) do { \ + int _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromLong(_ret); \ + } while (0) + +#define RETURN_SIZE(value) do { \ + Py_ssize_t _ret = (value); \ + if (_ret == -1) { \ + return NULL; \ + } \ + return PyLong_FromSsize_t(_ret); \ + } while (0) + + +static PyObject * +dict_check(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyDict_Check(obj)); +} + +static PyObject * +dict_checkexact(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyDict_CheckExact(obj)); +} + +static PyObject * +dict_new(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyDict_New(); +} + +static PyObject * +dictproxy_new(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDictProxy_New(obj); +} + +static PyObject * +dict_clear(PyObject *self, PyObject *obj) +{ + PyDict_Clear(obj); + Py_RETURN_NONE; +} + +static PyObject * +dict_copy(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Copy(obj); +} + +static PyObject * +dict_contains(PyObject *self, PyObject *args) +{ + PyObject *obj, *key; + if (!PyArg_ParseTuple(args, "OO", &obj, &key)) { + return NULL; + } + NULLABLE(obj); + NULLABLE(key); + RETURN_INT(PyDict_Contains(obj, key)); +} + +static PyObject * +dict_size(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + RETURN_SIZE(PyDict_Size(obj)); +} + +static PyObject * +dict_getitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + PyObject *value = PyDict_GetItem(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + +static PyObject * +dict_getitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + PyObject *value = PyDict_GetItemString(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + +static PyObject * +dict_getitemwitherror(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + PyObject *value = PyDict_GetItemWithError(mapping, key); + if (value == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + return Py_NewRef(PyExc_KeyError); + } + return Py_NewRef(value); +} + +static PyObject * +dict_setitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(value); + RETURN_INT(PyDict_SetItem(mapping, key, value)); +} + +static PyObject * +dict_setitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping, *value; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#O", &mapping, &key, &size, &value)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(value); + RETURN_INT(PyDict_SetItemString(mapping, key, value)); +} + +static PyObject * +dict_setdefault(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *defaultobj; + if (!PyArg_ParseTuple(args, "OOO", &mapping, &key, &defaultobj)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + NULLABLE(defaultobj); + return PyDict_SetDefault(mapping, key, defaultobj); +} + +static PyObject * +dict_delitem(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key; + if (!PyArg_ParseTuple(args, "OO", &mapping, &key)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(key); + RETURN_INT(PyDict_DelItem(mapping, key)); +} + +static PyObject * +dict_delitemstring(PyObject *self, PyObject *args) +{ + PyObject *mapping; + const char *key; + Py_ssize_t size; + if (!PyArg_ParseTuple(args, "Oz#", &mapping, &key, &size)) { + return NULL; + } + NULLABLE(mapping); + RETURN_INT(PyDict_DelItemString(mapping, key)); +} + +static PyObject * +dict_keys(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Keys(obj); +} + +static PyObject * +dict_values(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Values(obj); +} + +static PyObject * +dict_items(PyObject *self, PyObject *obj) +{ + NULLABLE(obj); + return PyDict_Items(obj); +} + +static PyObject * +dict_next(PyObject *self, PyObject *args) +{ + PyObject *mapping, *key, *value; + Py_ssize_t pos; + if (!PyArg_ParseTuple(args, "On", &mapping, &pos)) { + return NULL; + } + NULLABLE(mapping); + int rc = PyDict_Next(mapping, &pos, &key, &value); + if (rc != 0) { + return Py_BuildValue("inOO", rc, pos, key, value); + } + if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +dict_merge(PyObject *self, PyObject *args) +{ + PyObject *mapping, *mapping2; + int override; + if (!PyArg_ParseTuple(args, "OOi", &mapping, &mapping2, &override)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(mapping2); + RETURN_INT(PyDict_Merge(mapping, mapping2, override)); +} + +static PyObject * +dict_update(PyObject *self, PyObject *args) +{ + PyObject *mapping, *mapping2; + if (!PyArg_ParseTuple(args, "OO", &mapping, &mapping2)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(mapping2); + RETURN_INT(PyDict_Update(mapping, mapping2)); +} + +static PyObject * +dict_mergefromseq2(PyObject *self, PyObject *args) +{ + PyObject *mapping, *seq; + int override; + if (!PyArg_ParseTuple(args, "OOi", &mapping, &seq, &override)) { + return NULL; + } + NULLABLE(mapping); + NULLABLE(seq); + RETURN_INT(PyDict_MergeFromSeq2(mapping, seq, override)); +} + + +static PyMethodDef test_methods[] = { + {"dict_check", dict_check, METH_O}, + {"dict_checkexact", dict_checkexact, METH_O}, + {"dict_new", dict_new, METH_NOARGS}, + {"dictproxy_new", dictproxy_new, METH_O}, + {"dict_clear", dict_clear, METH_O}, + {"dict_copy", dict_copy, METH_O}, + {"dict_size", dict_size, METH_O}, + {"dict_getitem", dict_getitem, METH_VARARGS}, + {"dict_getitemwitherror", dict_getitemwitherror, METH_VARARGS}, + {"dict_getitemstring", dict_getitemstring, METH_VARARGS}, + {"dict_contains", dict_contains, METH_VARARGS}, + {"dict_setitem", dict_setitem, METH_VARARGS}, + {"dict_setitemstring", dict_setitemstring, METH_VARARGS}, + {"dict_delitem", dict_delitem, METH_VARARGS}, + {"dict_delitemstring", dict_delitemstring, METH_VARARGS}, + {"dict_setdefault", dict_setdefault, METH_VARARGS}, + {"dict_keys", dict_keys, METH_O}, + {"dict_values", dict_values, METH_O}, + {"dict_items", dict_items, METH_O}, + {"dict_next", dict_next, METH_VARARGS}, + {"dict_merge", dict_merge, METH_VARARGS}, + {"dict_update", dict_update, METH_VARARGS}, + {"dict_mergefromseq2", dict_mergefromseq2, METH_VARARGS}, + + {NULL}, +}; + +int +_PyTestCapi_Init_Dict(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index d1991ac6b464f..f37be9b67a3d7 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -26,6 +26,7 @@ int _PyTestCapi_Init_Vectorcall(PyObject *module); int _PyTestCapi_Init_Heaptype(PyObject *module); +int _PyTestCapi_Init_Abstract(PyObject *module); int _PyTestCapi_Init_Unicode(PyObject *module); int _PyTestCapi_Init_GetArgs(PyObject *module); int _PyTestCapi_Init_PyTime(PyObject *module); @@ -35,6 +36,7 @@ int _PyTestCapi_Init_Mem(PyObject *module); int _PyTestCapi_Init_Watchers(PyObject *module); int _PyTestCapi_Init_Long(PyObject *module); int _PyTestCapi_Init_Float(PyObject *module); +int _PyTestCapi_Init_Dict(PyObject *module); int _PyTestCapi_Init_Structmember(PyObject *module); int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 098d2cfa6d8ad..ee6f02dd8ab48 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1988,7 +1988,7 @@ return_result_with_error(PyObject *self, PyObject *args) Py_RETURN_NONE; } -static PyObject* +static PyObject * getitem_with_error(PyObject *self, PyObject *args) { PyObject *map, *key; @@ -2065,90 +2065,6 @@ py_w_stopcode(PyObject *self, PyObject *args) #endif -static PyObject * -get_mapping_keys(PyObject* self, PyObject *obj) -{ - return PyMapping_Keys(obj); -} - -static PyObject * -get_mapping_values(PyObject* self, PyObject *obj) -{ - return PyMapping_Values(obj); -} - -static PyObject * -get_mapping_items(PyObject* self, PyObject *obj) -{ - return PyMapping_Items(obj); -} - -static PyObject * -test_mapping_has_key_string(PyObject *self, PyObject *Py_UNUSED(args)) -{ - PyObject *context = PyDict_New(); - PyObject *val = PyLong_FromLong(1); - - // Since this uses `const char*` it is easier to test this in C: - PyDict_SetItemString(context, "a", val); - if (!PyMapping_HasKeyString(context, "a")) { - PyErr_SetString(PyExc_RuntimeError, - "Existing mapping key does not exist"); - return NULL; - } - if (PyMapping_HasKeyString(context, "b")) { - PyErr_SetString(PyExc_RuntimeError, - "Missing mapping key exists"); - return NULL; - } - - Py_DECREF(val); - Py_DECREF(context); - Py_RETURN_NONE; -} - -static PyObject * -mapping_has_key(PyObject* self, PyObject *args) -{ - PyObject *context, *key; - if (!PyArg_ParseTuple(args, "OO", &context, &key)) { - return NULL; - } - return PyLong_FromLong(PyMapping_HasKey(context, key)); -} - -static PyObject * -sequence_set_slice(PyObject* self, PyObject *args) -{ - PyObject *sequence, *obj; - Py_ssize_t i1, i2; - if (!PyArg_ParseTuple(args, "OnnO", &sequence, &i1, &i2, &obj)) { - return NULL; - } - - int res = PySequence_SetSlice(sequence, i1, i2, obj); - if (res == -1) { - return NULL; - } - Py_RETURN_NONE; -} - -static PyObject * -sequence_del_slice(PyObject* self, PyObject *args) -{ - PyObject *sequence; - Py_ssize_t i1, i2; - if (!PyArg_ParseTuple(args, "Onn", &sequence, &i1, &i2)) { - return NULL; - } - - int res = PySequence_DelSlice(sequence, i1, i2); - if (res == -1) { - return NULL; - } - Py_RETURN_NONE; -} - static PyObject * test_pythread_tss_key_state(PyObject *self, PyObject *args) { @@ -2252,72 +2168,6 @@ negative_refcount(PyObject *self, PyObject *Py_UNUSED(args)) #endif -static PyObject * -sequence_getitem(PyObject *self, PyObject *args) -{ - PyObject *seq; - Py_ssize_t i; - if (!PyArg_ParseTuple(args, "On", &seq, &i)) { - return NULL; - } - return PySequence_GetItem(seq, i); -} - - -static PyObject * -sequence_setitem(PyObject *self, PyObject *args) -{ - Py_ssize_t i; - PyObject *seq, *val; - if (!PyArg_ParseTuple(args, "OnO", &seq, &i, &val)) { - return NULL; - } - if (PySequence_SetItem(seq, i, val)) { - return NULL; - } - Py_RETURN_NONE; -} - - -static PyObject * -sequence_delitem(PyObject *self, PyObject *args) -{ - Py_ssize_t i; - PyObject *seq; - if (!PyArg_ParseTuple(args, "On", &seq, &i)) { - return NULL; - } - if (PySequence_DelItem(seq, i)) { - return NULL; - } - Py_RETURN_NONE; -} - -static PyObject * -hasattr_string(PyObject *self, PyObject* args) -{ - PyObject* obj; - PyObject* attr_name; - - if (!PyArg_UnpackTuple(args, "hasattr_string", 2, 2, &obj, &attr_name)) { - return NULL; - } - - if (!PyUnicode_Check(attr_name)) { - PyErr_SetString(PyExc_TypeError, "attribute name must a be string"); - return PyErr_Occurred(); - } - - const char *name_str = PyUnicode_AsUTF8(attr_name); - if (PyObject_HasAttrString(obj, name_str)) { - Py_RETURN_TRUE; - } - else { - Py_RETURN_FALSE; - } -} - - /* Functions for testing C calling conventions (METH_*) are named meth_*, * e.g. "meth_varargs" for METH_VARARGS. * @@ -3459,23 +3309,12 @@ static PyMethodDef TestMethods[] = { #ifdef W_STOPCODE {"W_STOPCODE", py_w_stopcode, METH_VARARGS}, #endif - {"get_mapping_keys", get_mapping_keys, METH_O}, - {"get_mapping_values", get_mapping_values, METH_O}, - {"get_mapping_items", get_mapping_items, METH_O}, - {"test_mapping_has_key_string", test_mapping_has_key_string, METH_NOARGS}, - {"mapping_has_key", mapping_has_key, METH_VARARGS}, - {"sequence_set_slice", sequence_set_slice, METH_VARARGS}, - {"sequence_del_slice", sequence_del_slice, METH_VARARGS}, {"test_pythread_tss_key_state", test_pythread_tss_key_state, METH_VARARGS}, {"hamt", new_hamt, METH_NOARGS}, {"bad_get", _PyCFunction_CAST(bad_get), METH_FASTCALL}, #ifdef Py_REF_DEBUG {"negative_refcount", negative_refcount, METH_NOARGS}, #endif - {"sequence_getitem", sequence_getitem, METH_VARARGS}, - {"sequence_setitem", sequence_setitem, METH_VARARGS}, - {"sequence_delitem", sequence_delitem, METH_VARARGS}, - {"hasattr_string", hasattr_string, METH_VARARGS}, {"meth_varargs", meth_varargs, METH_VARARGS}, {"meth_varargs_keywords", _PyCFunction_CAST(meth_varargs_keywords), METH_VARARGS|METH_KEYWORDS}, {"meth_o", meth_o, METH_O}, @@ -4116,6 +3955,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Heaptype(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Abstract(m) < 0) { + return NULL; + } if (_PyTestCapi_Init_Unicode(m) < 0) { return NULL; } @@ -4143,6 +3985,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Float(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Dict(m) < 0) { + return NULL; + } if (_PyTestCapi_Init_Structmember(m) < 0) { return NULL; } diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 3db9426d1a25f..1843b58de9d15 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -99,7 +99,9 @@ + + @@ -131,4 +133,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 8df4874659fa1..61322332af7d1 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -27,9 +27,15 @@ Source Files + + Source Files + Source Files + + Source Files + Source Files @@ -75,4 +81,4 @@ Resource Files - \ No newline at end of file + From webhook-mailer at python.org Wed Aug 16 08:29:50 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 12:29:50 -0000 Subject: [Python-checkins] [3.12] gh-107715: Escape class name in regular expression (GH-107716) (#107726) Message-ID: https://github.com/python/cpython/commit/ea6865242c184c8afc816e0211652f0d04d9b8f8 commit: ea6865242c184c8afc816e0211652f0d04d9b8f8 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-16T14:29:42+02:00 summary: [3.12] gh-107715: Escape class name in regular expression (GH-107716) (#107726) * gh-107715: Escape class name in regular expression (GH-107716) This patch escapes the class name before embedding it in the regular expression for `pat` in `doctest.DocTestFinder._find_lineno`. While class names do not ordinarily contain special characters, it is possible to encounter these when a class is created dynamically. Escaping the name will correctly return `None` in this scenario, rather than potentially matching a different class or raising `re.error` depending on the symbols used. (cherry picked from commit 85793278793708ad6b7132a54ac9fb4b2c5bcac1) Co-authored-by: Gertjan van Zwieten * Update 2023-08-07-14-12-07.gh-issue-107715.238r2f.rst --------- Co-authored-by: Gertjan van Zwieten Co-authored-by: Serhiy Storchaka files: A Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst M Lib/doctest.py diff --git a/Lib/doctest.py b/Lib/doctest.py index 2776d74bf9b58..a63df46a112e6 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1110,7 +1110,7 @@ def _find_lineno(self, obj, source_lines): if source_lines is None: return None pat = re.compile(r'^\s*class\s*%s\b' % - getattr(obj, '__name__', '-')) + re.escape(getattr(obj, '__name__', '-'))) for i, line in enumerate(source_lines): if pat.match(line): lineno = i diff --git a/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst b/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst new file mode 100644 index 0000000000000..4bf08c071df2f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst @@ -0,0 +1 @@ +Fix :meth:`doctest.DocTestFinder.find` in presence of class names with special characters. Patch by Gertjan van Zwieten. From webhook-mailer at python.org Wed Aug 16 08:30:34 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 12:30:34 -0000 Subject: [Python-checkins] [3.12] gh-100814: Fix exception for invalid callable value of Tkinter image option (GH-107692) (#107722) Message-ID: https://github.com/python/cpython/commit/6fd572f3b3dcc00cd80ba8e9a5a7c3a9cefc60cb commit: 6fd572f3b3dcc00cd80ba8e9a5a7c3a9cefc60cb branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-16T14:30:31+02:00 summary: [3.12] gh-100814: Fix exception for invalid callable value of Tkinter image option (GH-107692) (#107722) gh-100814: Fix exception for invalid callable value of Tkinter image option (GH-107692) Passing a callable object as an option value to a Tkinter image now raises the expected TclError instead of an AttributeError. (cherry picked from commit 50e3cc9748eb2103eb7ed6cc5a74d177df3cfb13) Co-authored-by: Serhiy Storchaka files: A Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst M Lib/test/test_tkinter/test_images.py M Lib/tkinter/__init__.py diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index b6f8b79ae689f..94cfe7df0be26 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -144,6 +144,14 @@ def test_configure_foreground(self): self.assertEqual(image['foreground'], '-foreground {} {} #000000 yellow') + def test_bug_100814(self): + # gh-100814: Passing a callable option value causes AttributeError. + with self.assertRaises(tkinter.TclError): + tkinter.BitmapImage('::img::test', master=self.root, spam=print) + image = tkinter.BitmapImage('::img::test', master=self.root) + with self.assertRaises(tkinter.TclError): + image.configure(spam=print) + class PhotoImageTest(AbstractTkTest, unittest.TestCase): @@ -274,6 +282,14 @@ def test_configure_palette(self): image.configure(palette='3/4/2') self.assertEqual(image['palette'], '3/4/2') + def test_bug_100814(self): + # gh-100814: Passing a callable option value causes AttributeError. + with self.assertRaises(tkinter.TclError): + tkinter.PhotoImage('::img::test', master=self.root, spam=print) + image = tkinter.PhotoImage('::img::test', master=self.root) + with self.assertRaises(tkinter.TclError): + image.configure(spam=print) + def test_blank(self): image = self.create() image.blank() diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index c675c511e0453..c59f8d11e8a9d 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -4069,8 +4069,6 @@ def __init__(self, imgtype, name=None, cnf={}, master=None, **kw): elif kw: cnf = kw options = () for k, v in cnf.items(): - if callable(v): - v = self._register(v) options = options + ('-'+k, v) self.tk.call(('image', 'create', imgtype, name,) + options) self.name = name @@ -4097,8 +4095,6 @@ def configure(self, **kw): for k, v in _cnfmerge(kw).items(): if v is not None: if k[-1] == '_': k = k[:-1] - if callable(v): - v = self._register(v) res = res + ('-'+k, v) self.tk.call((self.name, 'config') + res) diff --git a/Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst b/Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst new file mode 100644 index 0000000000000..86cb7bf79f307 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-06-15-29-00.gh-issue-100814.h195gW.rst @@ -0,0 +1,2 @@ +Passing a callable object as an option value to a Tkinter image now raises +the expected TclError instead of an AttributeError. From webhook-mailer at python.org Wed Aug 16 08:31:02 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 12:31:02 -0000 Subject: [Python-checkins] [3.12] Docs: Fix more Sphinx annotations in ctypes.rst (GH-107708) (#107717) Message-ID: https://github.com/python/cpython/commit/220d7e3120ffa23d01f91b7ad89f488b4dda3ef0 commit: 220d7e3120ffa23d01f91b7ad89f488b4dda3ef0 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-16T14:30:58+02:00 summary: [3.12] Docs: Fix more Sphinx annotations in ctypes.rst (GH-107708) (#107717) Docs: Fix more Sphinx annotations in ctypes.rst (GH-107708) (cherry picked from commit 8c9af6b9a0d6fc9cb237e96588d8dcab727e32b8) Co-authored-by: Erlend E. Aasland files: M Doc/conf.py M Doc/library/ctypes.rst M Doc/tools/.nitignore diff --git a/Doc/conf.py b/Doc/conf.py index fd115f7551f44..8224b248062f4 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -94,6 +94,7 @@ ('c:func', 'sprintf'), ('c:func', 'stat'), ('c:func', 'system'), + ('c:func', 'time'), ('c:func', 'vsnprintf'), # Standard C types ('c:type', 'FILE'), diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 3c794beca32c1..cd5c41e9b59c9 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -41,7 +41,7 @@ You load libraries by accessing them as attributes of these objects. *cdll* loads libraries which export functions using the standard ``cdecl`` calling convention, while *windll* libraries call functions using the ``stdcall`` calling convention. *oledll* also uses the ``stdcall`` calling convention, and -assumes the functions return a Windows :c:type:`HRESULT` error code. The error +assumes the functions return a Windows :c:type:`!HRESULT` error code. The error code is used to automatically raise an :class:`OSError` exception when the function call fails. @@ -477,7 +477,7 @@ Return types By default functions are assumed to return the C :c:expr:`int` type. Other -return types can be specified by setting the :attr:`restype` attribute of the +return types can be specified by setting the :attr:`~_FuncPtr.restype` attribute of the function object. The C prototype of :c:func:`time` is ``time_t time(time_t *)``. Because :c:type:`time_t` @@ -495,7 +495,7 @@ To call the function with a ``NULL`` pointer as first argument, use ``None``:: >>> print(libc.time(None)) # doctest: +SKIP 1150640792 -Here is a more advanced example, it uses the :func:`strchr` function, which expects +Here is a more advanced example, it uses the :func:`!strchr` function, which expects a string pointer and a char, and returns a pointer to a string:: >>> strchr = libc.strchr @@ -528,7 +528,7 @@ single character Python bytes object into a C char: >>> You can also use a callable Python object (a function or a class for example) as -the :attr:`restype` attribute, if the foreign function returns an integer. The +the :attr:`~_FuncPtr.restype` attribute, if the foreign function returns an integer. The callable will be called with the *integer* the C function returns, and the result of this call will be used as the result of your function call. This is useful to check for error return values and automatically raise an exception:: @@ -556,7 +556,8 @@ get the string representation of an error code, and *returns* an exception. :func:`GetLastError` to retrieve it. Please note that a much more powerful error checking mechanism is available -through the :attr:`errcheck` attribute; see the reference manual for details. +through the :attr:`~_FuncPtr.errcheck` attribute; +see the reference manual for details. .. _ctypes-passing-pointers: @@ -594,7 +595,7 @@ Structures and unions Structures and unions must derive from the :class:`Structure` and :class:`Union` base classes which are defined in the :mod:`ctypes` module. Each subclass must -define a :attr:`_fields_` attribute. :attr:`_fields_` must be a list of +define a :attr:`~Structure._fields_` attribute. :attr:`!_fields_` must be a list of *2-tuples*, containing a *field name* and a *field type*. The field type must be a :mod:`ctypes` type like :class:`c_int`, or any other @@ -666,9 +667,9 @@ Structure/union alignment and byte order By default, Structure and Union fields are aligned in the same way the C compiler does it. It is possible to override this behavior by specifying a -:attr:`_pack_` class attribute in the subclass definition. This must be set to a -positive integer and specifies the maximum alignment for the fields. This is -what ``#pragma pack(n)`` also does in MSVC. +:attr:`~Structure._pack_` class attribute in the subclass definition. +This must be set to a positive integer and specifies the maximum alignment for the fields. +This is what ``#pragma pack(n)`` also does in MSVC. :mod:`ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the @@ -684,7 +685,7 @@ Bit fields in structures and unions It is possible to create structures and unions containing bit fields. Bit fields are only possible for integer fields, the bit width is specified as the third -item in the :attr:`_fields_` tuples:: +item in the :attr:`~Structure._fields_` tuples:: >>> class Int(Structure): ... _fields_ = [("first_16", c_int, 16), @@ -876,7 +877,7 @@ pointer types. So, for ``POINTER(c_int)``, ctypes accepts an array of c_int:: >>> In addition, if a function argument is explicitly declared to be a pointer type -(such as ``POINTER(c_int)``) in :attr:`_FuncPtr.argtypes`, an object of the pointed +(such as ``POINTER(c_int)``) in :attr:`~_FuncPtr.argtypes`, an object of the pointed type (``c_int`` in this case) can be passed to the function. ctypes will apply the required :func:`byref` conversion in this case automatically. @@ -952,8 +953,8 @@ work:: >>> because the new ``class cell`` is not available in the class statement itself. -In :mod:`ctypes`, we can define the ``cell`` class and set the :attr:`_fields_` -attribute later, after the class statement:: +In :mod:`ctypes`, we can define the ``cell`` class and set the +:attr:`~Structure._fields_` attribute later, after the class statement:: >>> from ctypes import * >>> class cell(Structure): @@ -1003,8 +1004,8 @@ argument, and the callback functions expected argument types as the remaining arguments. I will present an example here which uses the standard C library's -:c:func:`qsort` function, that is used to sort items with the help of a callback -function. :c:func:`qsort` will be used to sort an array of integers:: +:c:func:`!qsort` function, that is used to sort items with the help of a callback +function. :c:func:`!qsort` will be used to sort an array of integers:: >>> IntArray5 = c_int * 5 >>> ia = IntArray5(5, 1, 7, 33, 99) @@ -1012,7 +1013,7 @@ function. :c:func:`qsort` will be used to sort an array of integers:: >>> qsort.restype = None >>> -:func:`qsort` must be called with a pointer to the data to sort, the number of +:func:`!qsort` must be called with a pointer to the data to sort, the number of items in the data array, the size of one item, and a pointer to the comparison function, the callback. The callback will then be called with two pointers to items, and it must return a negative integer if the first item is smaller than @@ -1104,7 +1105,7 @@ Some shared libraries not only export functions, they also export variables. An example in the Python library itself is the :c:data:`Py_Version`, Python runtime version number encoded in a single constant integer. -:mod:`ctypes` can access values like this with the :meth:`in_dll` class methods of +:mod:`ctypes` can access values like this with the :meth:`~_CData.in_dll` class methods of the type. *pythonapi* is a predefined symbol giving access to the Python C api:: @@ -1294,13 +1295,13 @@ Finding shared libraries When programming in a compiled language, shared libraries are accessed when compiling/linking a program, and when the program is run. -The purpose of the :func:`find_library` function is to locate a library in a way +The purpose of the :func:`~ctypes.util.find_library` function is to locate a library in a way similar to what the compiler or runtime loader does (on platforms with several versions of a shared library the most recent should be loaded), while the ctypes library loaders act like when a program is run, and call the runtime loader directly. -The :mod:`ctypes.util` module provides a function which can help to determine +The :mod:`!ctypes.util` module provides a function which can help to determine the library to load. @@ -1315,7 +1316,7 @@ the library to load. The exact functionality is system dependent. -On Linux, :func:`find_library` tries to run external programs +On Linux, :func:`~ctypes.util.find_library` tries to run external programs (``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the library file. It returns the filename of the library file. @@ -1334,7 +1335,7 @@ Here are some examples:: 'libbz2.so.1.0' >>> -On macOS, :func:`find_library` tries several predefined naming schemes and paths +On macOS, :func:`~ctypes.util.find_library` tries several predefined naming schemes and paths to locate the library, and returns a full pathname if successful:: >>> from ctypes.util import find_library @@ -1348,13 +1349,13 @@ to locate the library, and returns a full pathname if successful:: '/System/Library/Frameworks/AGL.framework/AGL' >>> -On Windows, :func:`find_library` searches along the system search path, and +On Windows, :func:`~ctypes.util.find_library` searches along the system search path, and returns the full pathname, but since there is no predefined naming scheme a call like ``find_library("c")`` will fail and return ``None``. If wrapping a shared library with :mod:`ctypes`, it *may* be better to determine the shared library name at development time, and hardcode that into the wrapper -module instead of using :func:`find_library` to locate the library at runtime. +module instead of using :func:`~ctypes.util.find_library` to locate the library at runtime. .. _ctypes-loading-shared-libraries: @@ -1439,9 +1440,9 @@ function exported by these libraries, and reacquired afterwards. All these classes can be instantiated by calling them with at least one argument, the pathname of the shared library. If you have an existing handle to an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platforms :c:func:`!dlopen` or :c:func:`LoadLibrary` -function is used to load the library into the process, and to get a handle to -it. +parameter, otherwise the underlying platforms :c:func:`!dlopen` or +:c:func:`!LoadLibrary` function is used to load the library into +the process, and to get a handle to it. The *mode* parameter can be used to specify how the library is loaded. For details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is @@ -1461,7 +1462,7 @@ to a new value and returns the former value. The *use_last_error* parameter, when set to true, enables the same mechanism for the Windows error code which is managed by the :func:`GetLastError` and -:func:`SetLastError` Windows API functions; :func:`ctypes.get_last_error` and +:func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and :func:`ctypes.set_last_error` are used to request and change the ctypes private copy of the windows error code. @@ -1533,7 +1534,7 @@ attribute of the loader instance. Class which loads shared libraries. *dlltype* should be one of the :class:`CDLL`, :class:`PyDLL`, :class:`WinDLL`, or :class:`OleDLL` types. - :meth:`__getattr__` has special behavior: It allows loading a shared library by + :meth:`!__getattr__` has special behavior: It allows loading a shared library by accessing it as attribute of a library loader instance. The result is cached, so repeated attribute accesses return the same library each time. @@ -1578,7 +1579,7 @@ object is available: An instance of :class:`PyDLL` that exposes Python C API functions as attributes. Note that all these functions are assumed to return C :c:expr:`int`, which is of course not always the truth, so you have to assign - the correct :attr:`restype` attribute to use these functions. + the correct :attr:`!restype` attribute to use these functions. .. audit-event:: ctypes.dlopen name ctypes.LibraryLoader @@ -1630,7 +1631,7 @@ They are instances of a private class: the callable will be called with this integer, allowing further processing or error checking. Using this is deprecated, for more flexible post processing or error checking use a ctypes data type as - :attr:`restype` and assign a callable to the :attr:`errcheck` attribute. + :attr:`!restype` and assign a callable to the :attr:`errcheck` attribute. .. attribute:: argtypes @@ -1662,7 +1663,7 @@ They are instances of a private class: :module: *result* is what the foreign function returns, as specified by the - :attr:`restype` attribute. + :attr:`!restype` attribute. *func* is the foreign function object itself, this allows reusing the same callable object to check or post process the results of several @@ -1772,7 +1773,7 @@ different ways, depending on the type and number of the parameters in the call: COM methods use a special calling convention: They require a pointer to the COM interface as first argument, in addition to those parameters that - are specified in the :attr:`~_FuncPtr.argtypes` tuple. + are specified in the :attr:`!argtypes` tuple. The optional *paramflags* parameter creates foreign function wrappers with much more functionality than the features described above. @@ -1847,7 +1848,7 @@ value if there is a single one, or a tuple containing the output parameter values when there are more than one, so the GetWindowRect function now returns a RECT instance, when called. -Output parameters can be combined with the :attr:`errcheck` protocol to do +Output parameters can be combined with the :attr:`~_FuncPtr.errcheck` protocol to do further output processing and error checking. The win32 ``GetWindowRect`` api function returns a ``BOOL`` to signal success or failure, so this function could do the error checking, and raises an exception when the api call failed:: @@ -1860,7 +1861,7 @@ do the error checking, and raises an exception when the api call failed:: >>> GetWindowRect.errcheck = errcheck >>> -If the :attr:`errcheck` function returns the argument tuple it receives +If the :attr:`~_FuncPtr.errcheck` function returns the argument tuple it receives unchanged, :mod:`ctypes` continues the normal processing it does on the output parameters. If you want to return a tuple of window coordinates instead of a ``RECT`` instance, you can retrieve the fields in the function and return them @@ -2010,7 +2011,7 @@ Utility functions .. function:: get_last_error() Windows only: returns the current value of the ctypes-private copy of the system - :data:`LastError` variable in the calling thread. + :data:`!LastError` variable in the calling thread. .. audit-event:: ctypes.get_last_error "" ctypes.get_last_error @@ -2063,7 +2064,7 @@ Utility functions .. function:: set_last_error(value) Windows only: set the current value of the ctypes-private copy of the system - :data:`LastError` variable in the calling thread to *value* and return the + :data:`!LastError` variable in the calling thread to *value* and return the previous value. .. audit-event:: ctypes.set_last_error error ctypes.set_last_error @@ -2225,13 +2226,13 @@ Fundamental data types Fundamental data types, when returned as foreign function call results, or, for example, by retrieving structure field members or array items, are transparently converted to native Python types. In other words, if a foreign function has a -:attr:`restype` of :class:`c_char_p`, you will always receive a Python bytes +:attr:`~_FuncPtr.restype` of :class:`c_char_p`, you will always receive a Python bytes object, *not* a :class:`c_char_p` instance. .. XXX above is false, it actually returns a Unicode string Subclasses of fundamental data types do *not* inherit this behavior. So, if a -foreign functions :attr:`restype` is a subclass of :class:`c_void_p`, you will +foreign functions :attr:`!restype` is a subclass of :class:`c_void_p`, you will receive an instance of this subclass from the function call. Of course, you can get the value of the pointer by accessing the ``value`` attribute. @@ -2430,7 +2431,7 @@ These are the fundamental ctypes data types: .. class:: HRESULT - Windows only: Represents a :c:type:`HRESULT` value, which contains success or + Windows only: Represents a :c:type:`!HRESULT` value, which contains success or error information for a function or method call. @@ -2439,9 +2440,9 @@ These are the fundamental ctypes data types: Represents the C :c:expr:`PyObject *` datatype. Calling this without an argument creates a ``NULL`` :c:expr:`PyObject *` pointer. -The :mod:`ctypes.wintypes` module provides quite some other Windows specific -data types, for example :c:type:`HWND`, :c:type:`WPARAM`, or :c:type:`DWORD`. Some -useful structures like :c:type:`MSG` or :c:type:`RECT` are also defined. +The :mod:`!ctypes.wintypes` module provides quite some other Windows specific +data types, for example :c:type:`!HWND`, :c:type:`!WPARAM`, or :c:type:`!DWORD`. +Some useful structures like :c:type:`!MSG` or :c:type:`!RECT` are also defined. .. _ctypes-structured-data-types: diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index b3ef4a6373195..8e9ac17135ebe 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -69,7 +69,6 @@ Doc/library/configparser.rst Doc/library/contextlib.rst Doc/library/copy.rst Doc/library/csv.rst -Doc/library/ctypes.rst Doc/library/datetime.rst Doc/library/dbm.rst Doc/library/decimal.rst From webhook-mailer at python.org Wed Aug 16 08:32:07 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 12:32:07 -0000 Subject: [Python-checkins] [3.12] gh-104496: Use correct Tcl or Tk version in Tkinter tests (GH-107688) (#107709) Message-ID: https://github.com/python/cpython/commit/91d935b47b47f5fa5c3eb0ee5e45a627f79babef commit: 91d935b47b47f5fa5c3eb0ee5e45a627f79babef branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-16T14:32:04+02:00 summary: [3.12] gh-104496: Use correct Tcl or Tk version in Tkinter tests (GH-107688) (#107709) gh-104496: Use correct Tcl or Tk version in Tkinter tests (GH-107688) In future Tcl and Tk versions can be desynchronized. (cherry picked from commit 3c8e8f3ceeae08fc43d885f5a4c65a3ee4b1a2c8) Co-authored-by: Serhiy Storchaka files: M Lib/test/test_tcl.py M Lib/test/test_tkinter/support.py M Lib/test/test_tkinter/test_images.py M Lib/test/test_tkinter/test_widgets.py M Lib/test/test_tkinter/widget_tests.py M Lib/test/test_ttk/test_style.py M Lib/test/test_ttk/test_widgets.py diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index d07b83acb1b50..ebdb58f91d3d8 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -20,14 +20,6 @@ tcl_version = tuple(map(int, _tkinter.TCL_VERSION.split('.'))) -_tk_patchlevel = None -def get_tk_patchlevel(): - global _tk_patchlevel - if _tk_patchlevel is None: - tcl = Tcl() - _tk_patchlevel = tcl.info_patchlevel() - return _tk_patchlevel - class TkinterTest(unittest.TestCase): @@ -571,7 +563,6 @@ def test_splitlist(self): (1, '2', (3.4,)) if self.wantobjects else ('1', '2', '3.4')), ] - tk_patchlevel = get_tk_patchlevel() if not self.wantobjects: expected = ('12', '\u20ac', '\xe2\x82\xac', '3.4') else: @@ -580,8 +571,8 @@ def test_splitlist(self): (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)), expected), ] - dbg_info = ('want objects? %s, Tcl version: %s, Tk patchlevel: %s' - % (self.wantobjects, tcl_version, tk_patchlevel)) + dbg_info = ('want objects? %s, Tcl version: %s, Tcl patchlevel: %s' + % (self.wantobjects, tcl_version, self.interp.info_patchlevel())) for arg, res in testcases: self.assertEqual(splitlist(arg), res, 'arg=%a, %s' % (arg, dbg_info)) diff --git a/Lib/test/test_tkinter/support.py b/Lib/test/test_tkinter/support.py index 9154ebac5c48f..10e64bf40a4af 100644 --- a/Lib/test/test_tkinter/support.py +++ b/Lib/test/test_tkinter/support.py @@ -79,28 +79,28 @@ def simulate_mouse_click(widget, x, y): import _tkinter tcl_version = tuple(map(int, _tkinter.TCL_VERSION.split('.'))) +tk_version = tuple(map(int, _tkinter.TK_VERSION.split('.'))) -def requires_tcl(*version): - if len(version) <= 2: - return unittest.skipUnless(tcl_version >= version, - 'requires Tcl version >= ' + '.'.join(map(str, version))) +def requires_tk(*version): + if len(version) <= 2 and tk_version >= version: + return lambda test: test def deco(test): @functools.wraps(test) def newtest(self): - if get_tk_patchlevel() < version: - self.skipTest('requires Tcl version >= ' + + root = getattr(self, 'root', None) + if get_tk_patchlevel(root) < version: + self.skipTest('requires Tk version >= ' + '.'.join(map(str, version))) test(self) return newtest return deco _tk_patchlevel = None -def get_tk_patchlevel(): +def get_tk_patchlevel(root): global _tk_patchlevel if _tk_patchlevel is None: - tcl = tkinter.Tcl() - _tk_patchlevel = tcl.info_patchlevel() + _tk_patchlevel = tkinter._parse_version(root.tk.globalgetvar('tk_patchLevel')) return _tk_patchlevel units = { diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index 94cfe7df0be26..9f49d6efc7892 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -2,7 +2,7 @@ import tkinter from test import support from test.support import os_helper -from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest, requires_tcl +from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest, requires_tk support.requires('gui') @@ -221,11 +221,11 @@ def test_create_from_gif_file(self): def test_create_from_gif_data(self): self.check_create_from_data('gif') - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_create_from_png_file(self): self.check_create_from_file('png') - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_create_from_png_data(self): self.check_create_from_data('png') diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index 34e67c0cbc44a..d3f942db7baf9 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -4,7 +4,7 @@ import os from test.support import requires -from test.test_tkinter.support import (requires_tcl, +from test.test_tkinter.support import (requires_tk, get_tk_patchlevel, widget_eq, AbstractDefaultRootTest) from test.test_tkinter.widget_tests import ( @@ -613,7 +613,7 @@ def test_configure_inactiveselectbackground(self): widget = self.create() self.checkColorParam(widget, 'inactiveselectbackground') - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_configure_insertunfocussed(self): widget = self.create() self.checkEnumParam(widget, 'insertunfocussed', @@ -924,7 +924,7 @@ def test_coords(self): for i in range(4): self.assertIsInstance(coords[i], float) - @requires_tcl(8, 6) + @requires_tk(8, 6) def test_moveto(self): widget = self.create() i1 = widget.create_rectangle(1, 1, 20, 20, tags='group') @@ -969,7 +969,7 @@ def test_configure_activestyle(self): self.checkEnumParam(widget, 'activestyle', 'dotbox', 'none', 'underline') - test_configure_justify = requires_tcl(8, 6, 5)(StandardOptionsTests.test_configure_justify) + test_configure_justify = requires_tk(8, 6, 5)(StandardOptionsTests.test_configure_justify) def test_configure_listvariable(self): widget = self.create() @@ -1108,7 +1108,7 @@ def test_configure_digits(self): def test_configure_from(self): widget = self.create() - conv = float if get_tk_patchlevel() >= (8, 6, 10) else float_round + conv = float if get_tk_patchlevel(self.root) >= (8, 6, 10) else float_round self.checkFloatParam(widget, 'from', 100, 14.9, 15.1, conv=conv) def test_configure_label(self): @@ -1235,19 +1235,19 @@ def test_configure_opaqueresize(self): widget = self.create() self.checkBooleanParam(widget, 'opaqueresize') - @requires_tcl(8, 6, 5) + @requires_tk(8, 6, 5) def test_configure_proxybackground(self): widget = self.create() self.checkColorParam(widget, 'proxybackground') - @requires_tcl(8, 6, 5) + @requires_tk(8, 6, 5) def test_configure_proxyborderwidth(self): widget = self.create() self.checkPixelsParam(widget, 'proxyborderwidth', 0, 1.3, 2.9, 6, -2, '10p', conv=False) - @requires_tcl(8, 6, 5) + @requires_tk(8, 6, 5) def test_configure_proxyrelief(self): widget = self.create() self.checkReliefParam(widget, 'proxyrelief') diff --git a/Lib/test/test_tkinter/widget_tests.py b/Lib/test/test_tkinter/widget_tests.py index 85b0511aba3c7..514b42bea764a 100644 --- a/Lib/test/test_tkinter/widget_tests.py +++ b/Lib/test/test_tkinter/widget_tests.py @@ -1,7 +1,7 @@ # Common tests for test_tkinter/test_widgets.py and test_ttk/test_widgets.py import tkinter -from test.test_tkinter.support import (AbstractTkTest, tcl_version, +from test.test_tkinter.support import (AbstractTkTest, tk_version, pixels_conv, tcl_obj_eq) import test.support @@ -22,7 +22,7 @@ def scaling(self): return self._scaling def _str(self, value): - if not self._stringify and self.wantobjects and tcl_version >= (8, 6): + if not self._stringify and self.wantobjects and tk_version >= (8, 6): return value if isinstance(value, tuple): return ' '.join(map(self._str, value)) @@ -156,7 +156,7 @@ def checkReliefParam(self, widget, name): 'flat', 'groove', 'raised', 'ridge', 'solid', 'sunken') errmsg='bad relief "spam": must be '\ 'flat, groove, raised, ridge, solid, or sunken' - if tcl_version < (8, 6): + if tk_version < (8, 6): errmsg = None self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) diff --git a/Lib/test/test_ttk/test_style.py b/Lib/test/test_ttk/test_style.py index 0ec95cf6b5ffc..f9c56ec235745 100644 --- a/Lib/test/test_ttk/test_style.py +++ b/Lib/test/test_ttk/test_style.py @@ -170,7 +170,7 @@ def test_map_custom_copy(self): newname = f'C.{name}' self.assertEqual(style.map(newname), {}) style.map(newname, **default) - if theme == 'alt' and name == '.' and get_tk_patchlevel() < (8, 6, 1): + if theme == 'alt' and name == '.' and get_tk_patchlevel(self.root) < (8, 6, 1): default['embossed'] = [('disabled', '1')] self.assertEqual(style.map(newname), default) for key, value in default.items(): diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py index 79d65b496abdc..fd1a748a498ac 100644 --- a/Lib/test/test_ttk/test_widgets.py +++ b/Lib/test/test_ttk/test_widgets.py @@ -5,7 +5,7 @@ import sys from test.test_ttk_textonly import MockTclObj -from test.test_tkinter.support import (AbstractTkTest, tcl_version, get_tk_patchlevel, +from test.test_tkinter.support import (AbstractTkTest, tk_version, get_tk_patchlevel, simulate_mouse_click, AbstractDefaultRootTest) from test.test_tkinter.widget_tests import (add_standard_options, AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) @@ -19,7 +19,7 @@ def test_configure_class(self): widget = self.create() self.assertEqual(widget['class'], '') errmsg='attempt to change read-only option' - if get_tk_patchlevel() < (8, 6, 0, 'beta', 3): + if get_tk_patchlevel(self.root) < (8, 6, 0, 'beta', 3): errmsg='Attempt to change read-only option' self.checkInvalidParam(widget, 'class', 'Foo', errmsg=errmsg) widget2 = self.create(class_='Foo') @@ -560,7 +560,7 @@ def test_configure_orient(self): widget = self.create() self.assertEqual(str(widget['orient']), 'vertical') errmsg='attempt to change read-only option' - if get_tk_patchlevel() < (8, 6, 0, 'beta', 3): + if get_tk_patchlevel(self.root) < (8, 6, 0, 'beta', 3): errmsg='Attempt to change read-only option' self.checkInvalidParam(widget, 'orient', 'horizontal', errmsg=errmsg) @@ -1526,7 +1526,7 @@ def test_heading(self): def test_heading_callback(self): def simulate_heading_click(x, y): - if tcl_version >= (8, 6): + if tk_version >= (8, 6): self.assertEqual(self.tv.identify_column(x), '#0') self.assertEqual(self.tv.identify_region(x, y), 'heading') simulate_mouse_click(self.tv, x, y) From webhook-mailer at python.org Wed Aug 16 09:22:22 2023 From: webhook-mailer at python.org (JelleZijlstra) Date: Wed, 16 Aug 2023 13:22:22 -0000 Subject: [Python-checkins] gh-108000: Test that `lambda` also has `__type_params__` (#108002) Message-ID: https://github.com/python/cpython/commit/a8d440b3837273926af5ce996162b019290ddad5 commit: a8d440b3837273926af5ce996162b019290ddad5 branch: main author: Nikita Sobolev committer: JelleZijlstra date: 2023-08-16T06:22:18-07:00 summary: gh-108000: Test that `lambda` also has `__type_params__` (#108002) files: M Lib/test/test_funcattrs.py diff --git a/Lib/test/test_funcattrs.py b/Lib/test/test_funcattrs.py index e08d72877d8ae..35b473d5e9a0b 100644 --- a/Lib/test/test_funcattrs.py +++ b/Lib/test/test_funcattrs.py @@ -194,16 +194,19 @@ def test___qualname__(self): def test___type_params__(self): def generic[T](): pass def not_generic(): pass + lambda_ = lambda: ... T, = generic.__type_params__ self.assertIsInstance(T, typing.TypeVar) self.assertEqual(generic.__type_params__, (T,)) - self.assertEqual(not_generic.__type_params__, ()) - with self.assertRaises(TypeError): - del not_generic.__type_params__ - with self.assertRaises(TypeError): - not_generic.__type_params__ = 42 - not_generic.__type_params__ = (T,) - self.assertEqual(not_generic.__type_params__, (T,)) + for func in (not_generic, lambda_): + with self.subTest(func=func): + self.assertEqual(func.__type_params__, ()) + with self.assertRaises(TypeError): + del func.__type_params__ + with self.assertRaises(TypeError): + func.__type_params__ = 42 + func.__type_params__ = (T,) + self.assertEqual(func.__type_params__, (T,)) def test___code__(self): num_one, num_two = 7, 8 From webhook-mailer at python.org Wed Aug 16 09:30:07 2023 From: webhook-mailer at python.org (JelleZijlstra) Date: Wed, 16 Aug 2023 13:30:07 -0000 Subject: [Python-checkins] gh-107909: Test explicit `object` base in PEP695 generic classes (#108001) Message-ID: https://github.com/python/cpython/commit/b61f5995aebb93496e968ca8d307375fa86d9329 commit: b61f5995aebb93496e968ca8d307375fa86d9329 branch: main author: Nikita Sobolev committer: JelleZijlstra date: 2023-08-16T06:30:03-07:00 summary: gh-107909: Test explicit `object` base in PEP695 generic classes (#108001) files: M Lib/test/test_type_params.py diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index bced641a9661f..0045057f181e1 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -148,6 +148,10 @@ def test_disallowed_expressions(self): check_syntax_error(self, "def f[T: [(x := 3) for _ in range(2)]](): pass") check_syntax_error(self, "type T = [(x := 3) for _ in range(2)]") + def test_incorrect_mro_explicit_object(self): + with self.assertRaisesRegex(TypeError, r"\(MRO\) for bases object, Generic"): + class My[X](object): ... + class TypeParamsNonlocalTest(unittest.TestCase): def test_nonlocal_disallowed_01(self): From webhook-mailer at python.org Wed Aug 16 09:47:19 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 16 Aug 2023 13:47:19 -0000 Subject: [Python-checkins] gh-104683: Argument Clinic: Extract parse function name helper (#107964) Message-ID: https://github.com/python/cpython/commit/42429d3b9adb8af1eadcfa155f6e8422a254ec67 commit: 42429d3b9adb8af1eadcfa155f6e8422a254ec67 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-16T13:47:15Z summary: gh-104683: Argument Clinic: Extract parse function name helper (#107964) Co-authored-by: Alex Waygood files: M Lib/test/test_clinic.py M Tools/clinic/clinic.py diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 32aac407a028c..38eabd2f7f21d 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -682,7 +682,7 @@ class C "void *" "" foo2 as .illegal. = foo1 [clinic start generated code]*/ """ - err = "Illegal C basename: '.illegal. = foo1'" + err = "Illegal C basename: '.illegal.'" self.expect_failure(block, err, lineno=7) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 6ff2622d33b38..9f7c47430772f 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1674,18 +1674,8 @@ def render_function( full_name = f.full_name template_dict = {'full_name': full_name} template_dict['name'] = f.displayname - - if f.c_basename: - c_basename = f.c_basename - else: - fields = full_name.split(".") - if fields[-1] == '__new__': - fields.pop() - c_basename = "_".join(fields) - - template_dict['c_basename'] = c_basename - - template_dict['methoddef_name'] = c_basename.upper() + "_METHODDEF" + template_dict['c_basename'] = f.c_basename + template_dict['methoddef_name'] = f.c_basename.upper() + "_METHODDEF" template_dict['docstring'] = self.docstring_for_c_string(f) @@ -2653,7 +2643,7 @@ class Function: name: str module: Module | Clinic cls: Class | None - c_basename: str | None + c_basename: str full_name: str return_converter: CReturnConverter kind: FunctionKind @@ -4577,6 +4567,11 @@ class ParamState(enum.IntEnum): RIGHT_SQUARE_AFTER = 6 +class FunctionNames(NamedTuple): + full_name: str + c_basename: str + + class DSLParser: function: Function | None state: StateKeeper @@ -4840,6 +4835,24 @@ def state_dsl_start(self, line: str) -> None: self.next(self.state_modulename_name, line) + @staticmethod + def parse_function_names(line: str) -> FunctionNames: + left, as_, right = line.partition(' as ') + full_name = left.strip() + c_basename = right.strip() + if as_ and not c_basename: + fail("No C basename provided after 'as' keyword") + if not c_basename: + fields = full_name.split(".") + if fields[-1] == '__new__': + fields.pop() + c_basename = "_".join(fields) + if not is_legal_py_identifier(full_name): + fail(f"Illegal function name: {full_name!r}") + if not is_legal_c_identifier(c_basename): + fail(f"Illegal C basename: {c_basename!r}") + return FunctionNames(full_name=full_name, c_basename=c_basename) + def update_function_kind(self, fullname: str) -> None: fields = fullname.split('.') name = fields.pop() @@ -4877,17 +4890,10 @@ def state_modulename_name(self, line: str) -> None: # are we cloning? before, equals, existing = line.rpartition('=') - c_basename: str | None if equals: - full_name, as_, c_basename = before.partition(' as ') - full_name = full_name.strip() - c_basename = c_basename.strip() - if as_ and not c_basename: - fail("No C basename provided after 'as' keyword") + full_name, c_basename = self.parse_function_names(before) existing = existing.strip() - if (is_legal_py_identifier(full_name) and - (not c_basename or is_legal_c_identifier(c_basename)) and - is_legal_py_identifier(existing)): + if is_legal_py_identifier(existing): # we're cloning! fields = [x.strip() for x in existing.split('.')] function_name = fields.pop() @@ -4933,16 +4939,7 @@ def state_modulename_name(self, line: str) -> None: line, _, returns = line.partition('->') returns = returns.strip() - - full_name, as_, c_basename = line.partition(' as ') - full_name = full_name.strip() - c_basename = c_basename.strip() or None - if as_ and not c_basename: - fail("No C basename provided after 'as' keyword") - if not is_legal_py_identifier(full_name): - fail(f"Illegal function name: {full_name!r}") - if c_basename and not is_legal_c_identifier(c_basename): - fail(f"Illegal C basename: {c_basename!r}") + full_name, c_basename = self.parse_function_names(line) return_converter = None if returns: From webhook-mailer at python.org Wed Aug 16 12:10:47 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 16 Aug 2023 16:10:47 -0000 Subject: [Python-checkins] GH-92584: Drop reference to Distutils in ``site.USER_BASE`` (#108031) Message-ID: https://github.com/python/cpython/commit/f2a9dfdee9de381e4adf29a7f1e2aec56580bfda commit: f2a9dfdee9de381e4adf29a7f1e2aec56580bfda branch: main author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: vstinner date: 2023-08-16T18:10:44+02:00 summary: GH-92584: Drop reference to Distutils in ``site.USER_BASE`` (#108031) Drop reference to Distutils in ``site.USER_BASE`` files: M Doc/library/site.rst diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 44f90a3b9e496..ff9e9107f9d16 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -189,7 +189,7 @@ Module contents :func:`getuserbase` hasn't been called yet. Default value is :file:`~/.local` for UNIX and macOS non-framework builds, :file:`~/Library/Python/{X.Y}` for macOS framework builds, and - :file:`{%APPDATA%}\\Python` for Windows. This value is used by Distutils to + :file:`{%APPDATA%}\\Python` for Windows. This value is used to compute the installation directories for scripts, data files, Python modules, etc. for the :ref:`user installation scheme `. See also :envvar:`PYTHONUSERBASE`. From webhook-mailer at python.org Wed Aug 16 12:24:50 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 16 Aug 2023 16:24:50 -0000 Subject: [Python-checkins] gh-107298: Fix C API Buffer documentation (#108011) Message-ID: https://github.com/python/cpython/commit/c2941cba7a986e6158eebb2a0bf33906dcd78616 commit: c2941cba7a986e6158eebb2a0bf33906dcd78616 branch: main author: Victor Stinner committer: vstinner date: 2023-08-16T18:24:46+02:00 summary: gh-107298: Fix C API Buffer documentation (#108011) files: M Doc/c-api/buffer.rst M Doc/c-api/typeobj.rst M Doc/tools/.nitignore diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index ba391a5279f20..e572815ffd625 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -163,13 +163,6 @@ a buffer, see :c:func:`PyObject_GetBuffer`. and :c:member:`~Py_buffer.suboffsets` MUST be ``NULL``. The maximum number of dimensions is given by :c:macro:`PyBUF_MAX_NDIM`. - .. :c:macro:: PyBUF_MAX_NDIM - - The maximum number of dimensions the memory represents. - Exporters MUST respect this limit, consumers of multi-dimensional - buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions. - Currently set to 64. - .. c:member:: Py_ssize_t *shape An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim` @@ -221,6 +214,17 @@ a buffer, see :c:func:`PyObject_GetBuffer`. freed when the buffer is released. The consumer MUST NOT alter this value. + +Constants: + +.. c:macro:: PyBUF_MAX_NDIM + + The maximum number of dimensions the memory represents. + Exporters MUST respect this limit, consumers of multi-dimensional + buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions. + Currently set to 64. + + .. _buffer-request-types: Buffer request types @@ -444,7 +448,7 @@ Buffer-related functions Send a request to *exporter* to fill in *view* as specified by *flags*. If the exporter cannot provide a buffer of the exact type, it MUST raise - :c:data:`PyExc_BufferError`, set ``view->obj`` to ``NULL`` and + :exc:`BufferError`, set ``view->obj`` to ``NULL`` and return ``-1``. On success, fill in *view*, set ``view->obj`` to a new reference @@ -531,7 +535,7 @@ Buffer-related functions and :c:macro:`PyBUF_WRITABLE` is set in *flags*. On success, set ``view->obj`` to a new reference to *exporter* and - return 0. Otherwise, raise :c:data:`PyExc_BufferError`, set + return 0. Otherwise, raise :exc:`BufferError`, set ``view->obj`` to ``NULL`` and return ``-1``; If this function is used as part of a :ref:`getbufferproc `, diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index faa183e27fcfa..d394ce10504b0 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -2443,7 +2443,7 @@ Buffer Object Structures Except for point (3), an implementation of this function MUST take these steps: - (1) Check if the request can be met. If not, raise :c:data:`PyExc_BufferError`, + (1) Check if the request can be met. If not, raise :exc:`BufferError`, set :c:expr:`view->obj` to ``NULL`` and return ``-1``. (2) Fill in the requested fields. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 21b350c4134fd..0f68cefc92b97 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -2,7 +2,6 @@ # as tested on the CI via check-warnings.py in reusable-docs.yml. # Keep lines sorted lexicographically to help avoid merge conflicts. -Doc/c-api/buffer.rst Doc/c-api/datetime.rst Doc/c-api/descriptor.rst Doc/c-api/exceptions.rst From webhook-mailer at python.org Wed Aug 16 14:19:00 2023 From: webhook-mailer at python.org (hugovk) Date: Wed, 16 Aug 2023 18:19:00 -0000 Subject: [Python-checkins] Remove Sphinx problem matcher to avoid annotating unchanged files (#108005) Message-ID: https://github.com/python/cpython/commit/0d7f5d3ba3641f8c7d32facbb177bf70ee7520d1 commit: 0d7f5d3ba3641f8c7d32facbb177bf70ee7520d1 branch: main author: Hugo van Kemenade committer: hugovk date: 2023-08-16T12:18:56-06:00 summary: Remove Sphinx problem matcher to avoid annotating unchanged files (#108005) files: D .github/problem-matchers/sphinx.json M .github/workflows/reusable-docs.yml diff --git a/.github/problem-matchers/sphinx.json b/.github/problem-matchers/sphinx.json deleted file mode 100644 index 09848608a7b03..0000000000000 --- a/.github/problem-matchers/sphinx.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "problemMatcher": [ - { - "owner": "sphinx-problem-matcher", - "pattern": [ - { - "regexp": "^(.*):(\\d+):\\s+(\\w*):\\s+(.*)$", - "file": 1, - "line": 2, - "severity": 3, - "message": 4 - } - ] - }, - { - "owner": "sphinx-problem-matcher-loose", - "pattern": [ - { - "_comment": "A bit of a looser pattern, doesn't look for line numbers, just looks for file names relying on them to start with / and end with .rst", - "regexp": "(\/.*\\.rst):\\s+(\\w*):\\s+(.*)$", - "file": 1, - "severity": 2, - "message": 3 - } - ] - }, - { - "owner": "sphinx-problem-matcher-loose-no-severity", - "pattern": [ - { - "_comment": "Looks for file names ending with .rst and line numbers but without severity", - "regexp": "^(.*\\.rst):(\\d+):(.*)$", - "file": 1, - "line": 2, - "message": 3 - } - ] - } - ] -} \ No newline at end of file diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 56932c4860573..39e5ad62924ad 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -18,8 +18,6 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout at v3 - - name: Register Sphinx problem matcher - run: echo "::add-matcher::.github/problem-matchers/sphinx.json" - name: 'Set up Python' uses: actions/setup-python at v4 with: @@ -76,8 +74,6 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout at v3 - - name: Register Sphinx problem matcher - run: echo "::add-matcher::.github/problem-matchers/sphinx.json" - uses: actions/cache at v3 with: path: ~/.cache/pip From webhook-mailer at python.org Wed Aug 16 14:39:02 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Wed, 16 Aug 2023 18:39:02 -0000 Subject: [Python-checkins] Improve the feature-proposal issue form (#108033) Message-ID: https://github.com/python/cpython/commit/8891a8821d5b03cd83a126fd6c02649448b18f41 commit: 8891a8821d5b03cd83a126fd6c02649448b18f41 branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-16T19:38:58+01:00 summary: Improve the feature-proposal issue form (#108033) Co-authored-by: Hugo van Kemenade Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M .github/ISSUE_TEMPLATE/feature.yml diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml index a1c48bbff829c..0200e623d2a3b 100644 --- a/.github/ISSUE_TEMPLATE/feature.yml +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -10,28 +10,26 @@ body: You'll need to demonstrate widespread support for your idea among the community. Major feature proposals should generally be discussed on [Discourse](https://discuss.python.org/c/ideas/6) before opening a GitHub issue. Wait until it's clear that most people support your idea before filling in this form. - - type: checkboxes + - type: dropdown attributes: - label: Has this already been discussed elsewhere? + label: Has this already been discussed elsewhere? options: - - label: I have already discussed this feature proposal on Discourse - - label: This is a minor feature, which does not need previous discussion elsewhere + - No response given + - I have already discussed this feature proposal on Discourse + - This is a minor feature, which does not need previous discussion elsewhere + multiple: false + validations: + required: true - type: textarea attributes: label: "Links to previous discussion of this feature:" validations: required: false - - type: input - attributes: - label: "Summary of proposal:" - description: A one-line summary of your proposal. - validations: - required: true - type: textarea attributes: - label: "Pitch:" + label: "Proposal:" description: > - Explain why this feature or enhancement should be implemented and how it would be used. + Explain your proposal, why it should be implemented, and how it would be used. Add examples, if applicable. Put any code blocks inside triple backticks. value: | From webhook-mailer at python.org Wed Aug 16 15:02:15 2023 From: webhook-mailer at python.org (hugovk) Date: Wed, 16 Aug 2023 19:02:15 -0000 Subject: [Python-checkins] [3.11] Remove Sphinx problem matcher to avoid annotating unchanged files (GH-108005) (#108050) Message-ID: https://github.com/python/cpython/commit/05ff5fa11d5e082906042ef7b48163c2f0f2c888 commit: 05ff5fa11d5e082906042ef7b48163c2f0f2c888 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: hugovk date: 2023-08-16T19:02:11Z summary: [3.11] Remove Sphinx problem matcher to avoid annotating unchanged files (GH-108005) (#108050) Co-authored-by: Hugo van Kemenade files: D .github/problem-matchers/sphinx.json M .github/workflows/reusable-docs.yml diff --git a/.github/problem-matchers/sphinx.json b/.github/problem-matchers/sphinx.json deleted file mode 100644 index 09848608a7b03..0000000000000 --- a/.github/problem-matchers/sphinx.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "problemMatcher": [ - { - "owner": "sphinx-problem-matcher", - "pattern": [ - { - "regexp": "^(.*):(\\d+):\\s+(\\w*):\\s+(.*)$", - "file": 1, - "line": 2, - "severity": 3, - "message": 4 - } - ] - }, - { - "owner": "sphinx-problem-matcher-loose", - "pattern": [ - { - "_comment": "A bit of a looser pattern, doesn't look for line numbers, just looks for file names relying on them to start with / and end with .rst", - "regexp": "(\/.*\\.rst):\\s+(\\w*):\\s+(.*)$", - "file": 1, - "severity": 2, - "message": 3 - } - ] - }, - { - "owner": "sphinx-problem-matcher-loose-no-severity", - "pattern": [ - { - "_comment": "Looks for file names ending with .rst and line numbers but without severity", - "regexp": "^(.*\\.rst):(\\d+):(.*)$", - "file": 1, - "line": 2, - "message": 3 - } - ] - } - ] -} \ No newline at end of file diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index eade14bc8d619..9c1ed2788080d 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -18,8 +18,6 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout at v3 - - name: Register Sphinx problem matcher - run: echo "::add-matcher::.github/problem-matchers/sphinx.json" - name: 'Set up Python' uses: actions/setup-python at v4 with: @@ -65,8 +63,6 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout at v3 - - name: Register Sphinx problem matcher - run: echo "::add-matcher::.github/problem-matchers/sphinx.json" - uses: actions/cache at v3 with: path: ~/.cache/pip From webhook-mailer at python.org Wed Aug 16 15:13:36 2023 From: webhook-mailer at python.org (carljm) Date: Wed, 16 Aug 2023 19:13:36 -0000 Subject: [Python-checkins] gh-91051: fix type watcher test to be robust to existing watcher (#107989) Message-ID: https://github.com/python/cpython/commit/fce93c80ae2d792b8ca443b044e28abbf28bb89a commit: fce93c80ae2d792b8ca443b044e28abbf28bb89a branch: main author: Carl Meyer committer: carljm date: 2023-08-16T13:13:32-06:00 summary: gh-91051: fix type watcher test to be robust to existing watcher (#107989) files: M Lib/test/test_capi/test_watchers.py diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 10b76e163bfb2..6b8855ec219d2 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -351,12 +351,10 @@ def test_clear_unassigned_watcher_id(self): self.clear_watcher(1) def test_no_more_ids_available(self): - contexts = [self.watcher() for i in range(self.TYPE_MAX_WATCHERS)] - with ExitStack() as stack: - for ctx in contexts: - stack.enter_context(ctx) - with self.assertRaisesRegex(RuntimeError, r"no more type watcher IDs"): - self.add_watcher() + with self.assertRaisesRegex(RuntimeError, r"no more type watcher IDs"): + with ExitStack() as stack: + for _ in range(self.TYPE_MAX_WATCHERS + 1): + stack.enter_context(self.watcher()) class TestCodeObjectWatchers(unittest.TestCase): From webhook-mailer at python.org Wed Aug 16 15:17:31 2023 From: webhook-mailer at python.org (ethanfurman) Date: Wed, 16 Aug 2023 19:17:31 -0000 Subject: [Python-checkins] gh-105522: [Enum] Correctly handle possible exceptions during testing (GH-105523) Message-ID: https://github.com/python/cpython/commit/199438b7cc2ef669b8d005d38797477a18b610cb commit: 199438b7cc2ef669b8d005d38797477a18b610cb branch: main author: Nikita Sobolev committer: ethanfurman date: 2023-08-16T12:17:28-07:00 summary: gh-105522: [Enum] Correctly handle possible exceptions during testing (GH-105523) files: M Lib/test/test_enum.py diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index a286411f7bcf5..6e128439fa356 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -38,6 +38,25 @@ def load_tests(loader, tests, ignore): )) return tests +def reraise_if_not_enum(*enum_types_or_exceptions): + from functools import wraps + + def decorator(func): + @wraps(func) + def inner(*args, **kwargs): + excs = [ + e + for e in enum_types_or_exceptions + if isinstance(e, Exception) + ] + if len(excs) == 1: + raise excs[0] + elif excs: + raise ExceptionGroup('Enum Exceptions', excs) + return func(*args, **kwargs) + return inner + return decorator + MODULE = __name__ SHORT_MODULE = MODULE.split('.')[-1] @@ -75,30 +94,42 @@ class FlagStooges(Flag): except Exception as exc: FlagStooges = exc -class FlagStoogesWithZero(Flag): - NOFLAG = 0 - LARRY = 1 - CURLY = 2 - MOE = 4 - BIG = 389 - -class IntFlagStooges(IntFlag): - LARRY = 1 - CURLY = 2 - MOE = 4 - BIG = 389 - -class IntFlagStoogesWithZero(IntFlag): - NOFLAG = 0 - LARRY = 1 - CURLY = 2 - MOE = 4 - BIG = 389 +try: + class FlagStoogesWithZero(Flag): + NOFLAG = 0 + LARRY = 1 + CURLY = 2 + MOE = 4 + BIG = 389 +except Exception as exc: + FlagStoogesWithZero = exc + +try: + class IntFlagStooges(IntFlag): + LARRY = 1 + CURLY = 2 + MOE = 4 + BIG = 389 +except Exception as exc: + IntFlagStooges = exc + +try: + class IntFlagStoogesWithZero(IntFlag): + NOFLAG = 0 + LARRY = 1 + CURLY = 2 + MOE = 4 + BIG = 389 +except Exception as exc: + IntFlagStoogesWithZero = exc # for pickle test and subclass tests -class Name(StrEnum): - BDFL = 'Guido van Rossum' - FLUFL = 'Barry Warsaw' +try: + class Name(StrEnum): + BDFL = 'Guido van Rossum' + FLUFL = 'Barry Warsaw' +except Exception as exc: + Name = exc try: Question = Enum('Question', 'who what when where why', module=__name__) @@ -204,26 +235,35 @@ def __get__(self, instance, ownerclass): # for global repr tests - at enum.global_enum -class HeadlightsK(IntFlag, boundary=enum.KEEP): - OFF_K = 0 - LOW_BEAM_K = auto() - HIGH_BEAM_K = auto() - FOG_K = auto() +try: + @enum.global_enum + class HeadlightsK(IntFlag, boundary=enum.KEEP): + OFF_K = 0 + LOW_BEAM_K = auto() + HIGH_BEAM_K = auto() + FOG_K = auto() +except Exception as exc: + HeadlightsK = exc - at enum.global_enum -class HeadlightsC(IntFlag, boundary=enum.CONFORM): - OFF_C = 0 - LOW_BEAM_C = auto() - HIGH_BEAM_C = auto() - FOG_C = auto() +try: + @enum.global_enum + class HeadlightsC(IntFlag, boundary=enum.CONFORM): + OFF_C = 0 + LOW_BEAM_C = auto() + HIGH_BEAM_C = auto() + FOG_C = auto() +except Exception as exc: + HeadlightsC = exc - at enum.global_enum -class NoName(Flag): - ONE = 1 - TWO = 2 +try: + @enum.global_enum + class NoName(Flag): + ONE = 1 + TWO = 2 +except Exception as exc: + NoName = exc # tests @@ -1124,9 +1164,8 @@ def red(self): green = 2 blue = 3 + @reraise_if_not_enum(Theory) def test_enum_function_with_qualname(self): - if isinstance(Theory, Exception): - raise Theory self.assertEqual(Theory.__qualname__, 'spanish_inquisition') def test_enum_of_types(self): @@ -1355,6 +1394,7 @@ class MyUnBrokenEnum(UnBrokenInt, Enum): test_pickle_dump_load(self.assertIs, MyUnBrokenEnum.I) test_pickle_dump_load(self.assertIs, MyUnBrokenEnum) + @reraise_if_not_enum(FloatStooges) def test_floatenum_fromhex(self): h = float.hex(FloatStooges.MOE.value) self.assertIs(FloatStooges.fromhex(h), FloatStooges.MOE) @@ -1475,6 +1515,7 @@ class ThreePart(Enum): self.assertIs(ThreePart((3, 3.0, 'three')), ThreePart.THREE) self.assertIs(ThreePart(3, 3.0, 'three'), ThreePart.THREE) + @reraise_if_not_enum(IntStooges) def test_intenum_from_bytes(self): self.assertIs(IntStooges.from_bytes(b'\x00\x03', 'big'), IntStooges.MOE) with self.assertRaises(ValueError): @@ -1503,33 +1544,28 @@ def repr(self): class Huh(MyStr, MyInt, Enum): One = 1 + @reraise_if_not_enum(Stooges) def test_pickle_enum(self): - if isinstance(Stooges, Exception): - raise Stooges test_pickle_dump_load(self.assertIs, Stooges.CURLY) test_pickle_dump_load(self.assertIs, Stooges) + @reraise_if_not_enum(IntStooges) def test_pickle_int(self): - if isinstance(IntStooges, Exception): - raise IntStooges test_pickle_dump_load(self.assertIs, IntStooges.CURLY) test_pickle_dump_load(self.assertIs, IntStooges) + @reraise_if_not_enum(FloatStooges) def test_pickle_float(self): - if isinstance(FloatStooges, Exception): - raise FloatStooges test_pickle_dump_load(self.assertIs, FloatStooges.CURLY) test_pickle_dump_load(self.assertIs, FloatStooges) + @reraise_if_not_enum(Answer) def test_pickle_enum_function(self): - if isinstance(Answer, Exception): - raise Answer test_pickle_dump_load(self.assertIs, Answer.him) test_pickle_dump_load(self.assertIs, Answer) + @reraise_if_not_enum(Question) def test_pickle_enum_function_with_module(self): - if isinstance(Question, Exception): - raise Question test_pickle_dump_load(self.assertIs, Question.who) test_pickle_dump_load(self.assertIs, Question) @@ -1592,9 +1628,8 @@ class Season(Enum): [Season.SUMMER, Season.WINTER, Season.AUTUMN, Season.SPRING], ) + @reraise_if_not_enum(Name) def test_subclassing(self): - if isinstance(Name, Exception): - raise Name self.assertEqual(Name.BDFL, 'Guido van Rossum') self.assertTrue(Name.BDFL, Name('Guido van Rossum')) self.assertIs(Name.BDFL, getattr(Name, 'BDFL')) @@ -3330,9 +3365,13 @@ def test_programatic_function_from_dict(self): self.assertIn(e, Perm) self.assertIs(type(e), Perm) + @reraise_if_not_enum( + FlagStooges, + FlagStoogesWithZero, + IntFlagStooges, + IntFlagStoogesWithZero, + ) def test_pickle(self): - if isinstance(FlagStooges, Exception): - raise FlagStooges test_pickle_dump_load(self.assertIs, FlagStooges.CURLY) test_pickle_dump_load(self.assertEqual, FlagStooges.CURLY|FlagStooges.MOE) @@ -3637,6 +3676,7 @@ def test_type(self): self.assertTrue(isinstance(Open.WO | Open.RW, Open)) self.assertEqual(Open.WO | Open.RW, 3) + @reraise_if_not_enum(HeadlightsK) def test_global_repr_keep(self): self.assertEqual( repr(HeadlightsK(0)), @@ -3651,6 +3691,7 @@ def test_global_repr_keep(self): '%(m)s.HeadlightsK(8)' % {'m': SHORT_MODULE}, ) + @reraise_if_not_enum(HeadlightsC) def test_global_repr_conform1(self): self.assertEqual( repr(HeadlightsC(0)), @@ -3665,6 +3706,7 @@ def test_global_repr_conform1(self): '%(m)s.OFF_C' % {'m': SHORT_MODULE}, ) + @reraise_if_not_enum(NoName) def test_global_enum_str(self): self.assertEqual(str(NoName.ONE & NoName.TWO), 'NoName(0)') self.assertEqual(str(NoName(0)), 'NoName(0)') From webhook-mailer at python.org Wed Aug 16 16:23:16 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 16 Aug 2023 20:23:16 -0000 Subject: [Python-checkins] GH-92584: Remove reference to Distutils in ``cx_Freeze``'s description (#108047) Message-ID: https://github.com/python/cpython/commit/57fcf96e4f21b8955b3ae4b4d70e4b756949712f commit: 57fcf96e4f21b8955b3ae4b4d70e4b756949712f branch: main author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: vstinner date: 2023-08-16T22:23:12+02:00 summary: GH-92584: Remove reference to Distutils in ``cx_Freeze``'s description (#108047) Remove reference to Distutils in ``cx_Freeze``'s description files: M Doc/using/windows.rst diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index d24450e2f963f..ca79c9d3a9d3a 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -1246,8 +1246,8 @@ shipped with PyWin32. It is an embeddable IDE with a built-in debugger. cx_Freeze --------- -`cx_Freeze `_ is a ``distutils`` -extension which wraps Python scripts into executable Windows programs +`cx_Freeze `_ +wraps Python scripts into executable Windows programs (:file:`{*}.exe` files). When you have done this, you can distribute your application without requiring your users to install Python. From webhook-mailer at python.org Wed Aug 16 16:26:25 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 16 Aug 2023 20:26:25 -0000 Subject: [Python-checkins] gh-107211: Fix select extension build on Solaris (#108012) Message-ID: https://github.com/python/cpython/commit/fb8fe377c4ddaea24ea6aa0a8f5d036986373d39 commit: fb8fe377c4ddaea24ea6aa0a8f5d036986373d39 branch: main author: Victor Stinner committer: vstinner date: 2023-08-16T22:26:22+02:00 summary: gh-107211: Fix select extension build on Solaris (#108012) Export the internal _Py_open() and _Py_write() functions for Solaris: the select shared extension uses them. files: M Include/internal/pycore_fileutils.h diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index cb5ac76772de5..0ed139f79b142 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -109,7 +109,8 @@ PyAPI_FUNC(int) _Py_stat( PyObject *path, struct stat *status); -extern int _Py_open( +// Export for 'select' shared extension (Solaris newDevPollObject() uses it) +PyAPI_FUNC(int) _Py_open( const char *pathname, int flags); @@ -126,7 +127,8 @@ extern Py_ssize_t _Py_read( void *buf, size_t count); -extern Py_ssize_t _Py_write( +// Export for 'select' shared extension (Solaris devpoll_flush() uses it) +PyAPI_FUNC(Py_ssize_t) _Py_write( int fd, const void *buf, size_t count); From webhook-mailer at python.org Wed Aug 16 16:32:20 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 16 Aug 2023 20:32:20 -0000 Subject: [Python-checkins] [3.11] gh-107298: Fix C API Buffer documentation (GH-108011) (#108041) Message-ID: https://github.com/python/cpython/commit/4e5dac1fe17cd7bbc8dad19e3e33c249a8a203c2 commit: 4e5dac1fe17cd7bbc8dad19e3e33c249a8a203c2 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: vstinner date: 2023-08-16T22:32:17+02:00 summary: [3.11] gh-107298: Fix C API Buffer documentation (GH-108011) (#108041) gh-107298: Fix C API Buffer documentation (GH-108011) (cherry picked from commit c2941cba7a986e6158eebb2a0bf33906dcd78616) Co-authored-by: Victor Stinner files: M Doc/c-api/buffer.rst M Doc/c-api/typeobj.rst M Doc/tools/.nitignore diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index ba391a5279f20..e572815ffd625 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -163,13 +163,6 @@ a buffer, see :c:func:`PyObject_GetBuffer`. and :c:member:`~Py_buffer.suboffsets` MUST be ``NULL``. The maximum number of dimensions is given by :c:macro:`PyBUF_MAX_NDIM`. - .. :c:macro:: PyBUF_MAX_NDIM - - The maximum number of dimensions the memory represents. - Exporters MUST respect this limit, consumers of multi-dimensional - buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions. - Currently set to 64. - .. c:member:: Py_ssize_t *shape An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim` @@ -221,6 +214,17 @@ a buffer, see :c:func:`PyObject_GetBuffer`. freed when the buffer is released. The consumer MUST NOT alter this value. + +Constants: + +.. c:macro:: PyBUF_MAX_NDIM + + The maximum number of dimensions the memory represents. + Exporters MUST respect this limit, consumers of multi-dimensional + buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions. + Currently set to 64. + + .. _buffer-request-types: Buffer request types @@ -444,7 +448,7 @@ Buffer-related functions Send a request to *exporter* to fill in *view* as specified by *flags*. If the exporter cannot provide a buffer of the exact type, it MUST raise - :c:data:`PyExc_BufferError`, set ``view->obj`` to ``NULL`` and + :exc:`BufferError`, set ``view->obj`` to ``NULL`` and return ``-1``. On success, fill in *view*, set ``view->obj`` to a new reference @@ -531,7 +535,7 @@ Buffer-related functions and :c:macro:`PyBUF_WRITABLE` is set in *flags*. On success, set ``view->obj`` to a new reference to *exporter* and - return 0. Otherwise, raise :c:data:`PyExc_BufferError`, set + return 0. Otherwise, raise :exc:`BufferError`, set ``view->obj`` to ``NULL`` and return ``-1``; If this function is used as part of a :ref:`getbufferproc `, diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 40cbcba63d69a..c2ed4e4391c89 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -2357,7 +2357,7 @@ Buffer Object Structures Except for point (3), an implementation of this function MUST take these steps: - (1) Check if the request can be met. If not, raise :c:data:`PyExc_BufferError`, + (1) Check if the request can be met. If not, raise :exc:`BufferError`, set :c:expr:`view->obj` to ``NULL`` and return ``-1``. (2) Fill in the requested fields. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index df7dcd607489c..1859d53e15284 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -2,7 +2,6 @@ # as tested on the CI via check-warnings.py in reusable-docs.yml. # Keep lines sorted lexicographically to help avoid merge conflicts. -Doc/c-api/buffer.rst Doc/c-api/datetime.rst Doc/c-api/descriptor.rst Doc/c-api/exceptions.rst From webhook-mailer at python.org Wed Aug 16 16:35:41 2023 From: webhook-mailer at python.org (hauntsaninja) Date: Wed, 16 Aug 2023 20:35:41 -0000 Subject: [Python-checkins] gh-56166: Deprecate passing confusing positional arguments in re functions (#107778) Message-ID: https://github.com/python/cpython/commit/882cb79afa2cb11b180ef699fd5cf038e72f6c85 commit: 882cb79afa2cb11b180ef699fd5cf038e72f6c85 branch: main author: Serhiy Storchaka committer: hauntsaninja <12621235+hauntsaninja at users.noreply.github.com> date: 2023-08-16T13:35:35-07:00 summary: gh-56166: Deprecate passing confusing positional arguments in re functions (#107778) Deprecate passing optional arguments maxsplit, count and flags in module-level functions re.split(), re.sub() and re.subn() as positional. They should only be passed by keyword. files: A Misc/NEWS.d/next/Library/2023-08-08-16-09-59.gh-issue-56166.WUMhYG.rst M Doc/library/re.rst M Doc/whatsnew/3.13.rst M Lib/re/__init__.py M Lib/test/test_re.py diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 3f03f0341d816..ab201e2483f8e 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -898,7 +898,7 @@ Functions ['Words', 'words', 'words', ''] >>> re.split(r'(\W+)', 'Words, words, words.') ['Words', ', ', 'words', ', ', 'words', '.', ''] - >>> re.split(r'\W+', 'Words, words, words.', 1) + >>> re.split(r'\W+', 'Words, words, words.', maxsplit=1) ['Words', 'words, words.'] >>> re.split('[a-f]+', '0a3B9', flags=re.IGNORECASE) ['0', '3', '9'] @@ -929,6 +929,11 @@ Functions .. versionchanged:: 3.7 Added support of splitting on a pattern that could match an empty string. + .. deprecated:: 3.13 + Passing *maxsplit* and *flags* as positional arguments is deprecated. + In future Python versions they will be + :ref:`keyword-only parameters `. + .. function:: findall(pattern, string, flags=0) @@ -1027,8 +1032,6 @@ Functions .. versionchanged:: 3.7 Unknown escapes in *repl* consisting of ``'\'`` and an ASCII letter now are errors. - - .. versionchanged:: 3.7 Empty matches for the pattern are replaced when adjacent to a previous non-empty match. @@ -1037,18 +1040,17 @@ Functions In :class:`bytes` replacement strings, group *name* can only contain bytes in the ASCII range (``b'\x00'``-``b'\x7f'``). + .. deprecated:: 3.13 + Passing *count* and *flags* as positional arguments is deprecated. + In future Python versions they will be + :ref:`keyword-only parameters `. + .. function:: subn(pattern, repl, string, count=0, flags=0) Perform the same operation as :func:`sub`, but return a tuple ``(new_string, number_of_subs_made)``. - .. versionchanged:: 3.1 - Added the optional flags argument. - - .. versionchanged:: 3.5 - Unmatched groups are replaced with an empty string. - .. function:: escape(pattern) @@ -1656,7 +1658,7 @@ because the address has spaces, our splitting pattern, in it: .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> [re.split(":? ", entry, 3) for entry in entries] + >>> [re.split(":? ", entry, maxsplit=3) for entry in entries] [['Ross', 'McFluff', '834.345.1254', '155 Elm Street'], ['Ronald', 'Heathmore', '892.345.3428', '436 Finley Avenue'], ['Frank', 'Burger', '925.541.7625', '662 South Dogwood Way'], @@ -1669,7 +1671,7 @@ house number from the street name: .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> [re.split(":? ", entry, 4) for entry in entries] + >>> [re.split(":? ", entry, maxsplit=4) for entry in entries] [['Ross', 'McFluff', '834.345.1254', '155', 'Elm Street'], ['Ronald', 'Heathmore', '892.345.3428', '436', 'Finley Avenue'], ['Frank', 'Burger', '925.541.7625', '662', 'South Dogwood Way'], diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 84ffd84b9f0bb..519dee5eb7d6c 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -832,6 +832,13 @@ Porting to Python 3.13 Deprecated ---------- +* Passing optional arguments *maxsplit*, *count* and *flags* in module-level + functions :func:`re.split`, :func:`re.sub` and :func:`re.subn` as positional + arguments is now deprecated. + In future Python versions these parameters will be + :ref:`keyword-only `. + (Contributed by Serhiy Storchaka in :gh:`56166`.) + * Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types: use directly the :c:type:`wchar_t` type instead. Since Python 3.3, ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` are just aliases to :c:type:`wchar_t`. diff --git a/Lib/re/__init__.py b/Lib/re/__init__.py index d6fccd5bc97cc..428d1b0d5fbd8 100644 --- a/Lib/re/__init__.py +++ b/Lib/re/__init__.py @@ -175,16 +175,39 @@ def search(pattern, string, flags=0): a Match object, or None if no match was found.""" return _compile(pattern, flags).search(string) -def sub(pattern, repl, string, count=0, flags=0): +class _ZeroSentinel(int): + pass +_zero_sentinel = _ZeroSentinel() + +def sub(pattern, repl, string, *args, count=_zero_sentinel, flags=_zero_sentinel): """Return the string obtained by replacing the leftmost non-overlapping occurrences of the pattern in string by the replacement repl. repl can be either a string or a callable; if a string, backslash escapes in it are processed. If it is a callable, it's passed the Match object and must return a replacement string to be used.""" + if args: + if count is not _zero_sentinel: + raise TypeError("sub() got multiple values for argument 'count'") + count, *args = args + if args: + if flags is not _zero_sentinel: + raise TypeError("sub() got multiple values for argument 'flags'") + flags, *args = args + if args: + raise TypeError("sub() takes from 3 to 5 positional arguments " + "but %d were given" % (5 + len(args))) + + import warnings + warnings.warn( + "'count' is passed as positional argument", + DeprecationWarning, stacklevel=2 + ) + return _compile(pattern, flags).sub(repl, string, count) +sub.__text_signature__ = '(pattern, repl, string, count=0, flags=0)' -def subn(pattern, repl, string, count=0, flags=0): +def subn(pattern, repl, string, *args, count=_zero_sentinel, flags=_zero_sentinel): """Return a 2-tuple containing (new_string, number). new_string is the string obtained by replacing the leftmost non-overlapping occurrences of the pattern in the source @@ -193,9 +216,28 @@ def subn(pattern, repl, string, count=0, flags=0): callable; if a string, backslash escapes in it are processed. If it is a callable, it's passed the Match object and must return a replacement string to be used.""" + if args: + if count is not _zero_sentinel: + raise TypeError("subn() got multiple values for argument 'count'") + count, *args = args + if args: + if flags is not _zero_sentinel: + raise TypeError("subn() got multiple values for argument 'flags'") + flags, *args = args + if args: + raise TypeError("subn() takes from 3 to 5 positional arguments " + "but %d were given" % (5 + len(args))) + + import warnings + warnings.warn( + "'count' is passed as positional argument", + DeprecationWarning, stacklevel=2 + ) + return _compile(pattern, flags).subn(repl, string, count) +subn.__text_signature__ = '(pattern, repl, string, count=0, flags=0)' -def split(pattern, string, maxsplit=0, flags=0): +def split(pattern, string, *args, maxsplit=_zero_sentinel, flags=_zero_sentinel): """Split the source string by the occurrences of the pattern, returning a list containing the resulting substrings. If capturing parentheses are used in pattern, then the text of all @@ -203,7 +245,26 @@ def split(pattern, string, maxsplit=0, flags=0): list. If maxsplit is nonzero, at most maxsplit splits occur, and the remainder of the string is returned as the final element of the list.""" + if args: + if maxsplit is not _zero_sentinel: + raise TypeError("split() got multiple values for argument 'maxsplit'") + maxsplit, *args = args + if args: + if flags is not _zero_sentinel: + raise TypeError("split() got multiple values for argument 'flags'") + flags, *args = args + if args: + raise TypeError("split() takes from 2 to 4 positional arguments " + "but %d were given" % (4 + len(args))) + + import warnings + warnings.warn( + "'maxsplit' is passed as positional argument", + DeprecationWarning, stacklevel=2 + ) + return _compile(pattern, flags).split(string, maxsplit) +split.__text_signature__ = '(pattern, string, maxsplit=0, flags=0)' def findall(pattern, string, flags=0): """Return a list of all non-overlapping matches in the string. diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 042f97f57ecf1..45bce1925f9e8 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -127,8 +127,10 @@ def test_basic_re_sub(self): self.assertEqual(re.sub("(?i)b+", "x", "bbbb BBBB"), 'x x') self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y'), '9.3 -3 24x100y') - self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', 3), - '9.3 -3 23x99y') + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', 3), + '9.3 -3 23x99y') + self.assertEqual(w.filename, __file__) self.assertEqual(re.sub(r'\d+', self.bump_num, '08.2 -2 23x99y', count=3), '9.3 -3 23x99y') @@ -235,9 +237,42 @@ def test_sub_template_numeric_escape(self): def test_qualified_re_sub(self): self.assertEqual(re.sub('a', 'b', 'aaaaa'), 'bbbbb') - self.assertEqual(re.sub('a', 'b', 'aaaaa', 1), 'baaaa') + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.sub('a', 'b', 'aaaaa', 1), 'baaaa') + self.assertEqual(w.filename, __file__) self.assertEqual(re.sub('a', 'b', 'aaaaa', count=1), 'baaaa') + with self.assertRaisesRegex(TypeError, + r"sub\(\) got multiple values for argument 'count'"): + re.sub('a', 'b', 'aaaaa', 1, count=1) + with self.assertRaisesRegex(TypeError, + r"sub\(\) got multiple values for argument 'flags'"): + re.sub('a', 'b', 'aaaaa', 1, 0, flags=0) + with self.assertRaisesRegex(TypeError, + r"sub\(\) takes from 3 to 5 positional arguments but 6 " + r"were given"): + re.sub('a', 'b', 'aaaaa', 1, 0, 0) + + def test_misuse_flags(self): + with self.assertWarns(DeprecationWarning) as w: + result = re.sub('a', 'b', 'aaaaa', re.I) + self.assertEqual(result, re.sub('a', 'b', 'aaaaa', count=int(re.I))) + self.assertEqual(str(w.warning), + "'count' is passed as positional argument") + self.assertEqual(w.filename, __file__) + with self.assertWarns(DeprecationWarning) as w: + result = re.subn("b*", "x", "xyz", re.I) + self.assertEqual(result, re.subn("b*", "x", "xyz", count=int(re.I))) + self.assertEqual(str(w.warning), + "'count' is passed as positional argument") + self.assertEqual(w.filename, __file__) + with self.assertWarns(DeprecationWarning) as w: + result = re.split(":", ":a:b::c", re.I) + self.assertEqual(result, re.split(":", ":a:b::c", maxsplit=int(re.I))) + self.assertEqual(str(w.warning), + "'maxsplit' is passed as positional argument") + self.assertEqual(w.filename, __file__) + def test_bug_114660(self): self.assertEqual(re.sub(r'(\S)\s+(\S)', r'\1 \2', 'hello there'), 'hello there') @@ -344,9 +379,22 @@ def test_re_subn(self): self.assertEqual(re.subn("b+", "x", "bbbb BBBB"), ('x BBBB', 1)) self.assertEqual(re.subn("b+", "x", "xyz"), ('xyz', 0)) self.assertEqual(re.subn("b*", "x", "xyz"), ('xxxyxzx', 4)) - self.assertEqual(re.subn("b*", "x", "xyz", 2), ('xxxyz', 2)) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.subn("b*", "x", "xyz", 2), ('xxxyz', 2)) + self.assertEqual(w.filename, __file__) self.assertEqual(re.subn("b*", "x", "xyz", count=2), ('xxxyz', 2)) + with self.assertRaisesRegex(TypeError, + r"subn\(\) got multiple values for argument 'count'"): + re.subn('a', 'b', 'aaaaa', 1, count=1) + with self.assertRaisesRegex(TypeError, + r"subn\(\) got multiple values for argument 'flags'"): + re.subn('a', 'b', 'aaaaa', 1, 0, flags=0) + with self.assertRaisesRegex(TypeError, + r"subn\(\) takes from 3 to 5 positional arguments but 6 " + r"were given"): + re.subn('a', 'b', 'aaaaa', 1, 0, 0) + def test_re_split(self): for string in ":a:b::c", S(":a:b::c"): self.assertTypedEqual(re.split(":", string), @@ -401,7 +449,9 @@ def test_re_split(self): self.assertTypedEqual(re.split(sep, ':a:b::c'), expected) def test_qualified_re_split(self): - self.assertEqual(re.split(":", ":a:b::c", 2), ['', 'a', 'b::c']) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(re.split(":", ":a:b::c", 2), ['', 'a', 'b::c']) + self.assertEqual(w.filename, __file__) self.assertEqual(re.split(":", ":a:b::c", maxsplit=2), ['', 'a', 'b::c']) self.assertEqual(re.split(':', 'a:b:c:d', maxsplit=2), ['a', 'b', 'c:d']) self.assertEqual(re.split("(:)", ":a:b::c", maxsplit=2), @@ -411,6 +461,17 @@ def test_qualified_re_split(self): self.assertEqual(re.split("(:*)", ":a:b::c", maxsplit=2), ['', ':', '', '', 'a:b::c']) + with self.assertRaisesRegex(TypeError, + r"split\(\) got multiple values for argument 'maxsplit'"): + re.split(":", ":a:b::c", 2, maxsplit=2) + with self.assertRaisesRegex(TypeError, + r"split\(\) got multiple values for argument 'flags'"): + re.split(":", ":a:b::c", 2, 0, flags=0) + with self.assertRaisesRegex(TypeError, + r"split\(\) takes from 2 to 4 positional arguments but 5 " + r"were given"): + re.split(":", ":a:b::c", 2, 0, 0) + def test_re_findall(self): self.assertEqual(re.findall(":+", "abc"), []) for string in "a:b::c:::d", S("a:b::c:::d"): diff --git a/Misc/NEWS.d/next/Library/2023-08-08-16-09-59.gh-issue-56166.WUMhYG.rst b/Misc/NEWS.d/next/Library/2023-08-08-16-09-59.gh-issue-56166.WUMhYG.rst new file mode 100644 index 0000000000000..34d776ae8fc3f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-08-16-09-59.gh-issue-56166.WUMhYG.rst @@ -0,0 +1,3 @@ +Deprecate passing optional arguments *maxsplit*, *count* and *flags* in +module-level functions :func:`re.split`, :func:`re.sub` and :func:`re.subn` as positional. +They should only be passed by keyword. From webhook-mailer at python.org Wed Aug 16 16:42:27 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 16 Aug 2023 20:42:27 -0000 Subject: [Python-checkins] GH-92584: Redirect macOS package installation to the PPUG (#108044) Message-ID: https://github.com/python/cpython/commit/902864256cb261428ae9682ca0ffddd597e1f894 commit: 902864256cb261428ae9682ca0ffddd597e1f894 branch: main author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: vstinner date: 2023-08-16T20:42:23Z summary: GH-92584: Redirect macOS package installation to the PPUG (#108044) files: M Doc/using/mac.rst diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 6517827286216..eb1413af2cbc3 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -125,13 +125,9 @@ http://www.hashcollision.org/hkn/python/idle_intro/index.html. Installing Additional Python Packages ===================================== -There are several methods to install additional Python packages: +This section has moved to the `Python Packaging User Guide`_. -* Packages can be installed via the standard Python distutils mode (``python - setup.py install``). - -* Many packages can also be installed via the :program:`setuptools` extension - or :program:`pip` wrapper, see https://pip.pypa.io/. +.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/installing-packages/ GUI Programming on the Mac From webhook-mailer at python.org Wed Aug 16 16:43:33 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 16 Aug 2023 20:43:33 -0000 Subject: [Python-checkins] GH-92584: Remove references to Distutils in ``PYTHONUSERBASE`` (#108040) Message-ID: https://github.com/python/cpython/commit/636ca313b2f7ce09a311889995778dccae8ebe40 commit: 636ca313b2f7ce09a311889995778dccae8ebe40 branch: main author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: vstinner date: 2023-08-16T22:43:30+02:00 summary: GH-92584: Remove references to Distutils in ``PYTHONUSERBASE`` (#108040) Remove references to Distutils in ``PYTHONUSERBASE`` files: M Doc/using/cmdline.rst M Misc/python.man diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 4bf67eb439ec6..23c89400f152b 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -811,8 +811,8 @@ conflict. Defines the :data:`user base directory `, which is used to compute the path of the :data:`user site-packages directory ` - and :ref:`Distutils installation paths ` for - ``python setup.py install --user``. + and :ref:`installation paths ` for + ``python -m pip install --user``. .. seealso:: diff --git a/Misc/python.man b/Misc/python.man index bf7cf767d164a..9f89c94adf502 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -592,8 +592,8 @@ works on Mac OS X. .IP PYTHONUSERBASE Defines the user base directory, which is used to compute the path of the user .IR site-packages -directory and Distutils installation paths for -.IR "python setup\.py install \-\-user" . +directory and installation paths for +.IR "python \-m pip install \-\-user" . .IP PYTHONPROFILEIMPORTTIME If this environment variable is set to a non-empty string, Python will show how long each import takes. This is exactly equivalent to setting From webhook-mailer at python.org Wed Aug 16 16:50:53 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 16 Aug 2023 20:50:53 -0000 Subject: [Python-checkins] [3.11] GH-92584: Redirect macOS package installation to the PPUG (GH-108044) (#108059) Message-ID: https://github.com/python/cpython/commit/ba2d6c9d1a796433aa44dfd69d0f2879e4b16639 commit: ba2d6c9d1a796433aa44dfd69d0f2879e4b16639 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: vstinner date: 2023-08-16T20:50:49Z summary: [3.11] GH-92584: Redirect macOS package installation to the PPUG (GH-108044) (#108059) GH-92584: Redirect macOS package installation to the PPUG (GH-108044) (cherry picked from commit 902864256cb261428ae9682ca0ffddd597e1f894) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/using/mac.rst diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 69cd5c92d884d..84031a3d6a17b 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -125,13 +125,9 @@ http://www.hashcollision.org/hkn/python/idle_intro/index.html. Installing Additional Python Packages ===================================== -There are several methods to install additional Python packages: +This section has moved to the `Python Packaging User Guide`_. -* Packages can be installed via the standard Python distutils mode (``python - setup.py install``). - -* Many packages can also be installed via the :program:`setuptools` extension - or :program:`pip` wrapper, see https://pip.pypa.io/. +.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/installing-packages/ GUI Programming on the Mac From webhook-mailer at python.org Wed Aug 16 17:01:31 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 16 Aug 2023 21:01:31 -0000 Subject: [Python-checkins] [3.11] GH-92584: Remove reference to Distutils in ``cx_Freeze``'s description (GH-108047) (#108061) Message-ID: https://github.com/python/cpython/commit/e3a11e12ab912f0614a90ade7acd34dda7e7f15e commit: e3a11e12ab912f0614a90ade7acd34dda7e7f15e branch: 3.11 author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: vstinner date: 2023-08-16T23:01:27+02:00 summary: [3.11] GH-92584: Remove reference to Distutils in ``cx_Freeze``'s description (GH-108047) (#108061) Remove reference to Distutils in ``cx_Freeze``'s description. (cherry picked from commit 57fcf96e4f21b8955b3ae4b4d70e4b756949712f) files: M Doc/using/windows.rst diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 2a07e42f8443c..7dee82a2223a2 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -1233,11 +1233,10 @@ shipped with PyWin32. It is an embeddable IDE with a built-in debugger. cx_Freeze --------- -`cx_Freeze `_ is a :mod:`distutils` -extension (see :ref:`extending-distutils`) which wraps Python scripts into -executable Windows programs (:file:`{*}.exe` files). When you have done this, -you can distribute your application without requiring your users to install -Python. +`cx_Freeze `_ +wraps Python scripts into executable Windows programs +(:file:`{*}.exe` files). When you have done this, you can distribute your +application without requiring your users to install Python. Compiling Python on Windows From webhook-mailer at python.org Wed Aug 16 17:07:00 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 16 Aug 2023 21:07:00 -0000 Subject: [Python-checkins] GH-92584: Remove Installing Python Modules (Distutils version) (#108020) Message-ID: https://github.com/python/cpython/commit/fbb7cbc0e92168077fd56de942901511e99ca60a commit: fbb7cbc0e92168077fd56de942901511e99ca60a branch: main author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: vstinner date: 2023-08-16T21:06:56Z summary: GH-92584: Remove Installing Python Modules (Distutils version) (#108020) files: D Doc/install/index.rst M Doc/contents.rst M Doc/extending/building.rst M Doc/library/site.rst M Doc/using/cmdline.rst diff --git a/Doc/contents.rst b/Doc/contents.rst index 464f93bdf85f9..649a1344a0b26 100644 --- a/Doc/contents.rst +++ b/Doc/contents.rst @@ -21,10 +21,3 @@ bugs.rst copyright.rst license.rst - -.. to include legacy packaging docs in build - -.. toctree:: - :hidden: - - install/index.rst diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst index 880bb33ee5671..ddde567f6f3ef 100644 --- a/Doc/extending/building.rst +++ b/Doc/extending/building.rst @@ -45,6 +45,7 @@ See the *"Multiple modules in one library"* section in :pep:`489` for details. .. highlight:: c +.. _install-index: .. _setuptools-index: Building C and C++ Extensions with setuptools diff --git a/Doc/install/index.rst b/Doc/install/index.rst deleted file mode 100644 index ffb4a202fe89f..0000000000000 --- a/Doc/install/index.rst +++ /dev/null @@ -1,1081 +0,0 @@ -.. highlight:: none - -.. _install-index: - -******************************************** - Installing Python Modules (Legacy version) -******************************************** - -:Author: Greg Ward - -.. TODO: Fill in XXX comments - -.. note:: - - The entire ``distutils`` package has been deprecated and will be - removed in Python 3.12. This documentation is retained as a - reference only, and will be removed with the package. See the - :ref:`What's New ` entry for more information. - -.. seealso:: - - :ref:`installing-index` - The up to date module installation documentation. For regular Python - usage, you almost certainly want that document rather than this one. - -.. note:: - - This document is being retained solely until the ``setuptools`` documentation - at https://setuptools.readthedocs.io/en/latest/setuptools.html - independently covers all of the relevant information currently included here. - -.. note:: - - This guide only covers the basic tools for building and distributing - extensions that are provided as part of this version of Python. Third party - tools offer easier to use and more secure alternatives. Refer to the `quick - recommendations section `__ - in the Python Packaging User Guide for more information. - - -.. _inst-intro: - - -Introduction -============ - -In Python 2.0, the ``distutils`` API was first added to the standard library. -This provided Linux distro maintainers with a standard way of converting -Python projects into Linux distro packages, and system administrators with a -standard way of installing them directly onto target systems. - -In the many years since Python 2.0 was released, tightly coupling the build -system and package installer to the language runtime release cycle has turned -out to be problematic, and it is now recommended that projects use the -``pip`` package installer and the ``setuptools`` build system, rather than -using ``distutils`` directly. - -See :ref:`installing-index` and :ref:`distributing-index` for more details. - -This legacy documentation is being retained only until we're confident that the -``setuptools`` documentation covers everything needed. - -.. _inst-new-standard: - -Distutils based source distributions ------------------------------------- - -If you download a module source distribution, you can tell pretty quickly if it -was packaged and distributed in the standard way, i.e. using the Distutils. -First, the distribution's name and version number will be featured prominently -in the name of the downloaded archive, e.g. :file:`foo-1.0.tar.gz` or -:file:`widget-0.9.7.zip`. Next, the archive will unpack into a similarly named -directory: :file:`foo-1.0` or :file:`widget-0.9.7`. Additionally, the -distribution will contain a setup script :file:`setup.py`, and a file named -:file:`README.txt` or possibly just :file:`README`, which should explain that -building and installing the module distribution is a simple matter of running -one command from a terminal:: - - python setup.py install - -For Windows, this command should be run from a command prompt window -(:menuselection:`Start --> Accessories`):: - - setup.py install - -If all these things are true, then you already know how to build and install the -modules you've just downloaded: Run the command above. Unless you need to -install things in a non-standard way or customize the build process, you don't -really need this manual. Or rather, the above command is everything you need to -get out of this manual. - - -.. _inst-standard-install: - -Standard Build and Install -========================== - -As described in section :ref:`inst-new-standard`, building and installing a module -distribution using the Distutils is usually one simple command to run from a -terminal:: - - python setup.py install - - -.. _inst-platform-variations: - -Platform variations -------------------- - -You should always run the setup command from the distribution root directory, -i.e. the top-level subdirectory that the module source distribution unpacks -into. For example, if you've just downloaded a module source distribution -:file:`foo-1.0.tar.gz` onto a Unix system, the normal thing to do is:: - - gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0 - cd foo-1.0 - python setup.py install - -On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the -archive file to :file:`C:\\Temp`, then it would unpack into -:file:`C:\\Temp\\foo-1.0`; you can use either an archive manipulator with a -graphical user interface (such as WinZip) or a command-line tool (such as -:program:`unzip` or :program:`pkunzip`) to unpack the archive. Then, open a -command prompt window and run:: - - cd c:\Temp\foo-1.0 - python setup.py install - - -.. _inst-splitting-up: - -Splitting the job up --------------------- - -Running ``setup.py install`` builds and installs all modules in one run. If you -prefer to work incrementally---especially useful if you want to customize the -build process, or if things are going wrong---you can use the setup script to do -one thing at a time. This is particularly helpful when the build and install -will be done by different users---for example, you might want to build a module -distribution and hand it off to a system administrator for installation (or do -it yourself, with super-user privileges). - -For example, you can build everything in one step, and then install everything -in a second step, by invoking the setup script twice:: - - python setup.py build - python setup.py install - -If you do this, you will notice that running the :command:`install` command -first runs the :command:`build` command, which---in this case---quickly notices -that it has nothing to do, since everything in the :file:`build` directory is -up-to-date. - -You may not need this ability to break things down often if all you do is -install modules downloaded off the 'net, but it's very handy for more advanced -tasks. If you get into distributing your own Python modules and extensions, -you'll run lots of individual Distutils commands on their own. - - -.. _inst-how-build-works: - -How building works ------------------- - -As implied above, the :command:`build` command is responsible for putting the -files to install into a *build directory*. By default, this is :file:`build` -under the distribution root; if you're excessively concerned with speed, or want -to keep the source tree pristine, you can change the build directory with the -:option:`!--build-base` option. For example:: - - python setup.py build --build-base=/path/to/pybuild/foo-1.0 - -(Or you could do this permanently with a directive in your system or personal -Distutils configuration file; see section :ref:`inst-config-files`.) Normally, this -isn't necessary. - -The default layout for the build tree is as follows:: - - --- build/ --- lib/ - or - --- build/ --- lib./ - temp./ - -where ```` expands to a brief description of the current OS/hardware -platform and Python version. The first form, with just a :file:`lib` directory, -is used for "pure module distributions"---that is, module distributions that -include only pure Python modules. If a module distribution contains any -extensions (modules written in C/C++), then the second form, with two ```` -directories, is used. In that case, the :file:`temp.{plat}` directory holds -temporary files generated by the compile/link process that don't actually get -installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory -contains all Python modules (pure Python and extensions) that will be installed. - -In the future, more directories will be added to handle Python scripts, -documentation, binary executables, and whatever else is needed to handle the job -of installing Python modules and applications. - - -.. _inst-how-install-works: - -How installation works ----------------------- - -After the :command:`build` command runs (whether you run it explicitly, or the -:command:`install` command does it for you), the work of the :command:`install` -command is relatively simple: all it has to do is copy everything under -:file:`build/lib` (or :file:`build/lib.{plat}`) to your chosen installation -directory. - -If you don't choose an installation directory---i.e., if you just run ``setup.py -install``\ ---then the :command:`install` command installs to the standard -location for third-party Python modules. This location varies by platform and -by how you built/installed Python itself. On Unix (and macOS, which is also -Unix-based), it also depends on whether the module distribution being installed -is pure Python or contains extensions ("non-pure"): - -.. tabularcolumns:: |l|l|l|l| - -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Platform | Standard installation location | Default value | Notes | -+=================+=====================================================+==================================================+=======+ -| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ - -Notes: - -(1) - Most Linux distributions include Python as a standard part of the system, so - :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on - Linux. If you build Python yourself on Linux (or any Unix-like system), the - default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`. - -(2) - The default installation directory on Windows was :file:`C:\\Program - Files\\Python` under Python 1.6a1, 1.5.2, and earlier. - -:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python -is installed to, and where it finds its libraries at run-time. They are always -the same under Windows, and very often the same under Unix and macOS. You -can find out what your Python installation uses for :file:`{prefix}` and -:file:`{exec-prefix}` by running Python in interactive mode and typing a few -simple commands. Under Unix, just type ``python`` at the shell prompt. Under -Windows, choose :menuselection:`Start --> Programs --> Python X.Y --> -Python (command line)`. Once the interpreter is started, you type Python code -at the prompt. For example, on my Linux system, I type the three Python -statements shown below, and get the output as shown, to find out my -:file:`{prefix}` and :file:`{exec-prefix}`: - -.. code-block:: pycon - - Python 2.4 (#26, Aug 7 2004, 17:19:02) - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.prefix - '/usr' - >>> sys.exec_prefix - '/usr' - -A few other placeholders are used in this document: :file:`{X.Y}` stands for the -version of Python, for example ``3.2``; :file:`{abiflags}` will be replaced by -the value of :data:`sys.abiflags` or the empty string for platforms which don't -define ABI flags; :file:`{distname}` will be replaced by the name of the module -distribution being installed. Dots and capitalization are important in the -paths; for example, a value that uses ``python3.2`` on UNIX will typically use -``Python32`` on Windows. - -If you don't want to install modules to the standard location, or if you don't -have permission to write there, then you need to read about alternate -installations in section :ref:`inst-alt-install`. If you want to customize your -installation directories more heavily, see section :ref:`inst-custom-install` on -custom installations. - - -.. _inst-alt-install: - -Alternate Installation -====================== - -Often, it is necessary or desirable to install modules to a location other than -the standard location for third-party Python modules. For example, on a Unix -system you might not have permission to write to the standard third-party module -directory. Or you might wish to try out a module before making it a standard -part of your local Python installation. This is especially true when upgrading -a distribution already present: you want to make sure your existing base of -scripts still works with the new version before actually upgrading. - -The Distutils :command:`install` command is designed to make installing module -distributions to an alternate location simple and painless. The basic idea is -that you supply a base directory for the installation, and the -:command:`install` command picks a set of directories (called an *installation -scheme*) under this base directory in which to install files. The details -differ across platforms, so read whichever of the following sections applies to -you. - -Note that the various alternate installation schemes are mutually exclusive: you -can pass ``--user``, or ``--home``, or ``--prefix`` and ``--exec-prefix``, or -``--install-base`` and ``--install-platbase``, but you can't mix from these -groups. - - -.. _inst-alt-install-user: - -Alternate installation: the user scheme ---------------------------------------- - -This scheme is designed to be the most convenient solution for users that don't -have write permission to the global site-packages directory or don't want to -install into it. It is enabled with a simple option:: - - python setup.py install --user - -Files will be installed into subdirectories of :const:`site.USER_BASE` (written -as :file:`{userbase}` hereafter). This scheme installs pure Python modules and -extension modules in the same location (also known as :const:`site.USER_SITE`). -Here are the values for UNIX, including macOS: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{userbase}/lib/python{X.Y}/site-packages` -scripts :file:`{userbase}/bin` -data :file:`{userbase}` -C headers :file:`{userbase}/include/python{X.Y}{abiflags}/{distname}` -=============== =========================================================== - -And here are the values used on Windows: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{userbase}\\Python{XY}\\site-packages` -scripts :file:`{userbase}\\Python{XY}\\Scripts` -data :file:`{userbase}` -C headers :file:`{userbase}\\Python{XY}\\Include\\{distname}` -=============== =========================================================== - -The advantage of using this scheme compared to the other ones described below is -that the user site-packages directory is under normal conditions always included -in :data:`sys.path` (see :mod:`site` for more information), which means that -there is no additional step to perform after running the :file:`setup.py` script -to finalize the installation. - -The :command:`build_ext` command also has a ``--user`` option to add -:file:`{userbase}/include` to the compiler search path for header files and -:file:`{userbase}/lib` to the compiler search path for libraries as well as to -the runtime search path for shared C libraries (rpath). - - -.. _inst-alt-install-home: - -Alternate installation: the home scheme ---------------------------------------- - -The idea behind the "home scheme" is that you build and maintain a personal -stash of Python modules. This scheme's name is derived from the idea of a -"home" directory on Unix, since it's not unusual for a Unix user to make their -home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`. -This scheme can be used by anyone, regardless of the operating system they -are installing for. - -Installing a new module distribution is as simple as :: - - python setup.py install --home= - -where you can supply any directory you like for the :option:`!--home` option. On -Unix, lazy typists can just type a tilde (``~``); the :command:`install` command -will expand this to your home directory:: - - python setup.py install --home=~ - -To make Python find the distributions installed with this scheme, you may have -to :ref:`modify Python's search path ` or edit -:mod:`!sitecustomize` (see :mod:`site`) to call :func:`site.addsitedir` or edit -:data:`sys.path`. - -The :option:`!--home` option defines the installation base directory. Files are -installed to the following directories under the installation base as follows: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{home}/lib/python` -scripts :file:`{home}/bin` -data :file:`{home}` -C headers :file:`{home}/include/python/{distname}` -=============== =========================================================== - -(Mentally replace slashes with backslashes if you're on Windows.) - - -.. _inst-alt-install-prefix-unix: - -Alternate installation: Unix (the prefix scheme) ------------------------------------------------- - -The "prefix scheme" is useful when you wish to use one Python installation to -perform the build/install (i.e., to run the setup script), but install modules -into the third-party module directory of a different Python installation (or -something that looks like a different Python installation). If this sounds a -trifle unusual, it is---that's why the user and home schemes come before. However, -there are at least two known cases where the prefix scheme will be useful. - -First, consider that many Linux distributions put Python in :file:`/usr`, rather -than the more traditional :file:`/usr/local`. This is entirely appropriate, -since in those cases Python is part of "the system" rather than a local add-on. -However, if you are installing Python modules from source, you probably want -them to go in :file:`/usr/local/lib/python2.{X}` rather than -:file:`/usr/lib/python2.{X}`. This can be done with :: - - /usr/bin/python setup.py install --prefix=/usr/local - -Another possibility is a network filesystem where the name used to write to a -remote directory is different from the name used to read it: for example, the -Python interpreter accessed as :file:`/usr/local/bin/python` might search for -modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to -be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could -be done with :: - - /usr/local/bin/python setup.py install --prefix=/mnt/@server/export - -In either case, the :option:`!--prefix` option defines the installation base, and -the :option:`!--exec-prefix` option defines the platform-specific installation -base, which is used for platform-specific files. (Currently, this just means -non-pure module distributions, but could be expanded to C libraries, binary -executables, etc.) If :option:`!--exec-prefix` is not supplied, it defaults to -:option:`!--prefix`. Files are installed as follows: - -================= ========================================================== -Type of file Installation directory -================= ========================================================== -Python modules :file:`{prefix}/lib/python{X.Y}/site-packages` -extension modules :file:`{exec-prefix}/lib/python{X.Y}/site-packages` -scripts :file:`{prefix}/bin` -data :file:`{prefix}` -C headers :file:`{prefix}/include/python{X.Y}{abiflags}/{distname}` -================= ========================================================== - -There is no requirement that :option:`!--prefix` or :option:`!--exec-prefix` -actually point to an alternate Python installation; if the directories listed -above do not already exist, they are created at installation time. - -Incidentally, the real reason the prefix scheme is important is simply that a -standard Unix installation uses the prefix scheme, but with :option:`!--prefix` -and :option:`!--exec-prefix` supplied by Python itself as ``sys.prefix`` and -``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme, -but every time you run ``python setup.py install`` without any other options, -you're using it. - -Note that installing extensions to an alternate Python installation has no -effect on how those extensions are built: in particular, the Python header files -(:file:`Python.h` and friends) installed with the Python interpreter used to run -the setup script will be used in compiling extensions. It is your -responsibility to ensure that the interpreter used to run extensions installed -in this way is compatible with the interpreter used to build them. The best way -to do this is to ensure that the two interpreters are the same version of Python -(possibly different builds, or possibly copies of the same build). (Of course, -if your :option:`!--prefix` and :option:`!--exec-prefix` don't even point to an -alternate Python installation, this is immaterial.) - - -.. _inst-alt-install-prefix-windows: - -Alternate installation: Windows (the prefix scheme) ---------------------------------------------------- - -Windows has no concept of a user's home directory, and since the standard Python -installation under Windows is simpler than under Unix, the :option:`!--prefix` -option has traditionally been used to install additional packages in separate -locations on Windows. :: - - python setup.py install --prefix="\Temp\Python" - -to install modules to the :file:`\\Temp\\Python` directory on the current drive. - -The installation base is defined by the :option:`!--prefix` option; the -:option:`!--exec-prefix` option is not supported under Windows, which means that -pure Python modules and extension modules are installed into the same location. -Files are installed as follows: - -=============== ========================================================== -Type of file Installation directory -=============== ========================================================== -modules :file:`{prefix}\\Lib\\site-packages` -scripts :file:`{prefix}\\Scripts` -data :file:`{prefix}` -C headers :file:`{prefix}\\Include\\{distname}` -=============== ========================================================== - - -.. _inst-custom-install: - -Custom Installation -=================== - -Sometimes, the alternate installation schemes described in section -:ref:`inst-alt-install` just don't do what you want. You might want to tweak just -one or two directories while keeping everything under the same base directory, -or you might want to completely redefine the installation scheme. In either -case, you're creating a *custom installation scheme*. - -To create a custom installation scheme, you start with one of the alternate -schemes and override some of the installation directories used for the various -types of files, using these options: - -====================== ======================= -Type of file Override option -====================== ======================= -Python modules ``--install-purelib`` -extension modules ``--install-platlib`` -all modules ``--install-lib`` -scripts ``--install-scripts`` -data ``--install-data`` -C headers ``--install-headers`` -====================== ======================= - -These override options can be relative, absolute, -or explicitly defined in terms of one of the installation base directories. -(There are two installation base directories, and they are normally the -same---they only differ when you use the Unix "prefix scheme" and supply -different ``--prefix`` and ``--exec-prefix`` options; using ``--install-lib`` -will override values computed or given for ``--install-purelib`` and -``--install-platlib``, and is recommended for schemes that don't make a -difference between Python and extension modules.) - -For example, say you're installing a module distribution to your home directory -under Unix---but you want scripts to go in :file:`~/scripts` rather than -:file:`~/bin`. As you might expect, you can override this directory with the -:option:`!--install-scripts` option; in this case, it makes most sense to supply -a relative path, which will be interpreted relative to the installation base -directory (your home directory, in this case):: - - python setup.py install --home=~ --install-scripts=scripts - -Another Unix example: suppose your Python installation was built and installed -with a prefix of :file:`/usr/local/python`, so under a standard installation -scripts will wind up in :file:`/usr/local/python/bin`. If you want them in -:file:`/usr/local/bin` instead, you would supply this absolute directory for the -:option:`!--install-scripts` option:: - - python setup.py install --install-scripts=/usr/local/bin - -(This performs an installation using the "prefix scheme", where the prefix is -whatever your Python interpreter was installed with--- :file:`/usr/local/python` -in this case.) - -If you maintain Python on Windows, you might want third-party modules to live in -a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}` -itself. This is almost as easy as customizing the script installation -directory---you just have to remember that there are two types of modules -to worry about, Python and extension modules, which can conveniently be both -controlled by one option:: - - python setup.py install --install-lib=Site - -The specified installation directory is relative to :file:`{prefix}`. Of -course, you also have to ensure that this directory is in Python's module -search path, such as by putting a :file:`.pth` file in a site directory (see -:mod:`site`). See section :ref:`inst-search-path` to find out how to modify -Python's search path. - -If you want to define an entire installation scheme, you just have to supply all -of the installation directory options. The recommended way to do this is to -supply relative paths; for example, if you want to maintain all Python -module-related files under :file:`python` in your home directory, and you want a -separate directory for each platform that you use your home directory from, you -might define the following installation scheme:: - - python setup.py install --home=~ \ - --install-purelib=python/lib \ - --install-platlib=python/lib.$PLAT \ - --install-scripts=python/scripts - --install-data=python/data - -or, equivalently, :: - - python setup.py install --home=~/python \ - --install-purelib=lib \ - --install-platlib='lib.$PLAT' \ - --install-scripts=scripts - --install-data=data - -``$PLAT`` is not (necessarily) an environment variable---it will be expanded by -the Distutils as it parses your command line options, just as it does when -parsing your configuration file(s). - -Obviously, specifying the entire installation scheme every time you install a -new module distribution would be very tedious. Thus, you can put these options -into your Distutils config file (see section :ref:`inst-config-files`): - -.. code-block:: ini - - [install] - install-base=$HOME - install-purelib=python/lib - install-platlib=python/lib.$PLAT - install-scripts=python/scripts - install-data=python/data - -or, equivalently, - -.. code-block:: ini - - [install] - install-base=$HOME/python - install-purelib=lib - install-platlib=lib.$PLAT - install-scripts=scripts - install-data=data - -Note that these two are *not* equivalent if you supply a different installation -base directory when you run the setup script. For example, :: - - python setup.py install --install-base=/tmp - -would install pure modules to :file:`/tmp/python/lib` in the first case, and -to :file:`/tmp/lib` in the second case. (For the second case, you probably -want to supply an installation base of :file:`/tmp/python`.) - -You probably noticed the use of ``$HOME`` and ``$PLAT`` in the sample -configuration file input. These are Distutils configuration variables, which -bear a strong resemblance to environment variables. In fact, you can use -environment variables in config files on platforms that have such a notion but -the Distutils additionally define a few extra variables that may not be in your -environment, such as ``$PLAT``. (And of course, on systems that don't have -environment variables, such as Mac OS 9, the configuration variables supplied by -the Distutils are the only ones you can use.) See section :ref:`inst-config-files` -for details. - -.. note:: When a :ref:`virtual environment ` is activated, any options - that change the installation path will be ignored from all distutils configuration - files to prevent inadvertently installing projects outside of the virtual - environment. - -.. XXX need some Windows examples---when would custom installation schemes be - needed on those platforms? - - -.. XXX Move this to Doc/using - -.. _inst-search-path: - -Modifying Python's Search Path ------------------------------- - -When the Python interpreter executes an :keyword:`import` statement, it searches -for both Python code and extension modules along a search path. A default value -for the path is configured into the Python binary when the interpreter is built. -You can determine the path by importing the :mod:`sys` module and printing the -value of ``sys.path``. :: - - $ python - Python 2.2 (#11, Oct 3 2002, 13:31:27) - [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2 - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.path - ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2', - '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload', - '/usr/local/lib/python2.3/site-packages'] - >>> - -The null string in ``sys.path`` represents the current working directory. - -The expected convention for locally installed packages is to put them in the -:file:`{...}/site-packages/` directory, but you may want to install Python -modules into some arbitrary directory. For example, your site may have a -convention of keeping all software related to the web server under :file:`/www`. -Add-on Python modules might then belong in :file:`/www/python`, and in order to -import them, this directory must be added to ``sys.path``. There are several -different ways to add the directory. - -The most convenient way is to add a path configuration file to a directory -that's already on Python's path, usually to the :file:`.../site-packages/` -directory. Path configuration files have an extension of :file:`.pth`, and each -line must contain a single path that will be appended to ``sys.path``. (Because -the new paths are appended to ``sys.path``, modules in the added directories -will not override standard modules. This means you can't use this mechanism for -installing fixed versions of standard modules.) - -Paths can be absolute or relative, in which case they're relative to the -directory containing the :file:`.pth` file. See the documentation of -the :mod:`site` module for more information. - -A slightly less convenient way is to edit the :file:`site.py` file in Python's -standard library, and modify ``sys.path``. :file:`site.py` is automatically -imported when the Python interpreter is executed, unless the :option:`-S` switch -is supplied to suppress this behaviour. So you could simply edit -:file:`site.py` and add two lines to it: - -.. code-block:: python - - import sys - sys.path.append('/www/python/') - -However, if you reinstall the same minor version of Python (perhaps when -upgrading from 2.2 to 2.2.2, for example) :file:`site.py` will be overwritten by -the stock version. You'd have to remember that it was modified and save a copy -before doing the installation. - -There are two environment variables that can modify ``sys.path``. -:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python -installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``, -the search path will be set to ``['', '/www/python/lib/pythonX.Y/', -'/www/python/lib/pythonX.Y/plat-linux2', ...]``. - -The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be -added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is -set to ``/www/python:/opt/py``, the search path will begin with -``['/www/python', '/opt/py']``. (Note that directories must exist in order to -be added to ``sys.path``; the :mod:`site` module removes paths that don't -exist.) - -Finally, ``sys.path`` is just a regular Python list, so any Python application -can modify it by adding or removing entries. - - -.. _inst-config-files: - -Distutils Configuration Files -============================= - -As mentioned above, you can use Distutils configuration files to record personal -or site preferences for any Distutils options. That is, any option to any -command can be stored in one of two or three (depending on your platform) -configuration files, which will be consulted before the command-line is parsed. -This means that configuration files will override default values, and the -command-line will in turn override configuration files. Furthermore, if -multiple configuration files apply, values from "earlier" files are overridden -by "later" files. - - -.. _inst-config-filenames: - -Location and names of config files ----------------------------------- - -The names and locations of the configuration files vary slightly across -platforms. On Unix and macOS, the three configuration files (in the order -they are processed) are: - -+--------------+----------------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+==========================================================+=======+ -| system | :file:`{prefix}/lib/python{ver}/distutils/distutils.cfg` | \(1) | -+--------------+----------------------------------------------------------+-------+ -| personal | :file:`$HOME/.pydistutils.cfg` | \(2) | -+--------------+----------------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+----------------------------------------------------------+-------+ - -And on Windows, the configuration files are: - -+--------------+-------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+=================================================+=======+ -| system | :file:`{prefix}\\Lib\\distutils\\distutils.cfg` | \(4) | -+--------------+-------------------------------------------------+-------+ -| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) | -+--------------+-------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+-------------------------------------------------+-------+ - -On all platforms, the "personal" file can be temporarily disabled by -passing the ``--no-user-cfg`` option. - -Notes: - -(1) - Strictly speaking, the system-wide configuration file lives in the directory - where the Distutils are installed; under Python 1.6 and later on Unix, this is - as shown. For Python 1.5.2, the Distutils will normally be installed to - :file:`{prefix}/lib/python1.5/site-packages/distutils`, so the system - configuration file should be put there under Python 1.5.2. - -(2) - On Unix, if the :envvar:`HOME` environment variable is not defined, the user's - home directory will be determined with the :func:`~pwd.getpwuid` function from the - standard :mod:`pwd` module. This is done by the :func:`os.path.expanduser` - function used by Distutils. - -(3) - I.e., in the current directory (usually the location of the setup script). - -(4) - (See also note (1).) Under Python 1.6 and later, Python's default "installation - prefix" is :file:`C:\\Python`, so the system configuration file is normally - :file:`C:\\Python\\Lib\\distutils\\distutils.cfg`. Under Python 1.5.2, the - default prefix was :file:`C:\\Program Files\\Python`, and the Distutils were not - part of the standard library---so the system configuration file would be - :file:`C:\\Program Files\\Python\\distutils\\distutils.cfg` in a standard Python - 1.5.2 installation under Windows. - -(5) - On Windows, if the :envvar:`HOME` environment variable is not defined, - :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will - be tried. This is done by the :func:`os.path.expanduser` function used - by Distutils. - - -.. _inst-config-syntax: - -Syntax of config files ----------------------- - -The Distutils configuration files all have the same syntax. The config files -are grouped into sections. There is one section for each Distutils command, -plus a ``global`` section for global options that affect every command. Each -section consists of one option per line, specified as ``option=value``. - -For example, the following is a complete config file that just forces all -commands to run quietly by default: - -.. code-block:: ini - - [global] - verbose=0 - -If this is installed as the system config file, it will affect all processing of -any Python module distribution by any user on the current system. If it is -installed as your personal config file (on systems that support them), it will -affect only module distributions processed by you. And if it is used as the -:file:`setup.cfg` for a particular module distribution, it affects only that -distribution. - -You could override the default "build base" directory and make the -:command:`build\*` commands always forcibly rebuild all files with the -following: - -.. code-block:: ini - - [build] - build-base=blib - force=1 - -which corresponds to the command-line arguments :: - - python setup.py build --build-base=blib --force - -except that including the :command:`build` command on the command-line means -that command will be run. Including a particular command in config files has no -such implication; it only means that if the command is run, the options in the -config file will apply. (Or if other commands that derive values from it are -run, they will use the values in the config file.) - -You can find out the complete list of options for any command using the -:option:`!--help` option, e.g.:: - - python setup.py build --help - -and you can find out the complete list of global options by using -:option:`!--help` without a command:: - - python setup.py --help - -See also the "Reference" section of the "Distributing Python Modules" manual. - - -.. _inst-building-ext: - -Building Extensions: Tips and Tricks -==================================== - -Whenever possible, the Distutils try to use the configuration information made -available by the Python interpreter used to run the :file:`setup.py` script. -For example, the same compiler and linker flags used to compile Python will also -be used for compiling extensions. Usually this will work well, but in -complicated situations this might be inappropriate. This section discusses how -to override the usual Distutils behaviour. - - -.. _inst-tweak-flags: - -Tweaking compiler/linker flags ------------------------------- - -Compiling a Python extension written in C or C++ will sometimes require -specifying custom flags for the compiler and linker in order to use a particular -library or produce a special kind of object code. This is especially true if the -extension hasn't been tested on your platform, or if you're trying to -cross-compile Python. - -In the most general case, the extension author might have foreseen that -compiling the extensions would be complicated, and provided a :file:`Setup` file -for you to edit. This will likely only be done if the module distribution -contains many separate extension modules, or if they often require elaborate -sets of compiler flags in order to work. - -A :file:`Setup` file, if present, is parsed in order to get a list of extensions -to build. Each line in a :file:`Setup` describes a single module. Lines have -the following structure:: - - module ... [sourcefile ...] [cpparg ...] [library ...] - - -Let's examine each of the fields in turn. - -* *module* is the name of the extension module to be built, and should be a - valid Python identifier. You can't just change this in order to rename a module - (edits to the source code would also be needed), so this should be left alone. - -* *sourcefile* is anything that's likely to be a source code file, at least - judging by the filename. Filenames ending in :file:`.c` are assumed to be - written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are - assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed - to be in Objective C. - -* *cpparg* is an argument for the C preprocessor, and is anything starting with - :option:`!-I`, :option:`!-D`, :option:`!-U` or :option:`!-C`. - -* *library* is anything ending in :file:`.a` or beginning with :option:`!-l` or - :option:`!-L`. - -If a particular platform requires a special library on your platform, you can -add it by editing the :file:`Setup` file and running ``python setup.py build``. -For example, if the module defined by the line :: - - foo foomodule.c - -must be linked with the math library :file:`libm.a` on your platform, simply add -:option:`!-lm` to the line:: - - foo foomodule.c -lm - -Arbitrary switches intended for the compiler or the linker can be supplied with -the :option:`!-Xcompiler` *arg* and :option:`!-Xlinker` *arg* options:: - - foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm - -The next option after :option:`!-Xcompiler` and :option:`!-Xlinker` will be -appended to the proper command line, so in the above example the compiler will -be passed the :option:`!-o32` option, and the linker will be passed -:option:`!-shared`. If a compiler option requires an argument, you'll have to -supply multiple :option:`!-Xcompiler` options; for example, to pass ``-x c++`` -the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``. - -Compiler flags can also be supplied through setting the :envvar:`CFLAGS` -environment variable. If set, the contents of :envvar:`CFLAGS` will be added to -the compiler flags specified in the :file:`Setup` file. - - -.. _inst-non-ms-compilers: - -Using non-Microsoft compilers on Windows ----------------------------------------- - -.. sectionauthor:: Rene Liebscher - - - -Borland/CodeGear C++ -^^^^^^^^^^^^^^^^^^^^ - -This subsection describes the necessary steps to use Distutils with the Borland -C++ compiler version 5.5. First you have to know that Borland's object file -format (OMF) is different from the format used by the Python version you can -download from the Python or ActiveState web site. (Python is built with -Microsoft Visual C++, which uses COFF as the object file format.) For this -reason you have to convert Python's library :file:`python25.lib` into the -Borland format. You can do this as follows: - -.. Should we mention that users have to create cfg-files for the compiler? -.. see also http://community.borland.com/article/0,1410,21205,00.html - -:: - - coff2omf python25.lib python25_bcpp.lib - -The :file:`coff2omf` program comes with the Borland compiler. The file -:file:`python25.lib` is in the :file:`Libs` directory of your Python -installation. If your extension uses other libraries (zlib, ...) you have to -convert them too. - -The converted files have to reside in the same directories as the normal -libraries. - -How does Distutils manage to use these libraries with their changed names? If -the extension needs a library (eg. :file:`foo`) Distutils checks first if it -finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then -uses this library. In the case it doesn't find such a special library it uses -the default name (:file:`foo.lib`.) [#]_ - -To let Distutils compile your extension with Borland C++ you now have to type:: - - python setup.py build --compiler=bcpp - -If you want to use the Borland C++ compiler as the default, you could specify -this in your personal or system-wide configuration file for Distutils (see -section :ref:`inst-config-files`.) - - -.. seealso:: - - `C++Builder Compiler `_ - Information about the free C++ compiler from Borland, including links to the - download pages. - - `Creating Python Extensions Using Borland's Free Compiler `_ - Document describing how to use Borland's free command-line C++ compiler to build - Python. - - -GNU C / Cygwin / MinGW -^^^^^^^^^^^^^^^^^^^^^^ - -This section describes the necessary steps to use Distutils with the GNU C/C++ -compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter -that was built with Cygwin, everything should work without any of these -following steps. - -Not all extensions can be built with MinGW or Cygwin, but many can. Extensions -most likely to not work are those that use C++ or depend on Microsoft Visual C -extensions. - -To let Distutils compile your extension with Cygwin you have to type:: - - python setup.py build --compiler=cygwin - -and for Cygwin in no-cygwin mode [#]_ or for MinGW type:: - - python setup.py build --compiler=mingw32 - -If you want to use any of these options/compilers as default, you should -consider writing it in your personal or system-wide configuration file for -Distutils (see section :ref:`inst-config-files`.) - -Older Versions of Python and MinGW -"""""""""""""""""""""""""""""""""" -The following instructions only apply if you're using a version of Python -inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with -binutils-2.13.90-20030111-1). - -These compilers require some special libraries. This task is more complex than -for Borland's C++, because there is no program to convert the library. First -you have to create a list of symbols which the Python DLL exports. (You can find -a good program for this task at -https://sourceforge.net/projects/mingw/files/MinGW/Extension/pexports/). - -.. I don't understand what the next line means. --amk -.. (inclusive the references on data structures.) - -:: - - pexports python25.dll >python25.def - -The location of an installed :file:`python25.dll` will depend on the -installation options and the version and language of Windows. In a "just for -me" installation, it will appear in the root of the installation directory. In -a shared installation, it will be located in the system directory. - -Then you can create from these information an import library for gcc. :: - - /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a - -The resulting library has to be placed in the same directory as -:file:`python25.lib`. (Should be the :file:`libs` directory under your Python -installation directory.) - -If your extension uses other libraries (zlib,...) you might have to convert -them too. The converted files have to reside in the same directories as the -normal libraries do. - - -.. seealso:: - - `Building Python modules on MS Windows platform with MinGW `_ - Information about building the required libraries for the MinGW environment. - - -.. rubric:: Footnotes - -.. [#] This also means you could replace all existing COFF-libraries with OMF-libraries - of the same name. - -.. [#] Check https://www.sourceware.org/cygwin/ for more information - -.. [#] Then you have no POSIX emulation available, but you also don't need - :file:`cygwin1.dll`. diff --git a/Doc/library/site.rst b/Doc/library/site.rst index ff9e9107f9d16..ea3b2e996574e 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -191,7 +191,7 @@ Module contents :file:`~/Library/Python/{X.Y}` for macOS framework builds, and :file:`{%APPDATA%}\\Python` for Windows. This value is used to compute the installation directories for scripts, data files, Python modules, - etc. for the :ref:`user installation scheme `. + etc. for the user installation scheme. See also :envvar:`PYTHONUSERBASE`. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 23c89400f152b..921b6a6961c7b 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -811,7 +811,7 @@ conflict. Defines the :data:`user base directory `, which is used to compute the path of the :data:`user site-packages directory ` - and :ref:`installation paths ` for + and installation paths for ``python -m pip install --user``. .. seealso:: From webhook-mailer at python.org Wed Aug 16 17:13:08 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 16 Aug 2023 21:13:08 -0000 Subject: [Python-checkins] GH-92584: Remove references to Distutils in configure.rst (#108043) Message-ID: https://github.com/python/cpython/commit/e88eb3775ecdcb3af6c6d694a935b7fa5f41e5ce commit: e88eb3775ecdcb3af6c6d694a935b7fa5f41e5ce branch: main author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: vstinner date: 2023-08-16T23:13:05+02:00 summary: GH-92584: Remove references to Distutils in configure.rst (#108043) Remove references to Distutils in configure.rst files: M Doc/using/configure.rst diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 924e73dc54da2..441d346a1a38a 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -686,7 +686,6 @@ Main files of the build system * :file:`pyconfig.h` (created by :file:`configure`); * :file:`Modules/Setup`: C extensions built by the Makefile using :file:`Module/makesetup` shell script; -* :file:`setup.py`: C extensions built using the ``setuptools`` package. Main build steps ---------------- @@ -695,8 +694,7 @@ Main build steps * A static ``libpython`` library (``.a``) is created from objects files. * ``python.o`` and the static ``libpython`` library are linked into the final ``python`` program. -* C extensions are built by the Makefile (see :file:`Modules/Setup`) - and ``python setup.py build``. +* C extensions are built by the Makefile (see :file:`Modules/Setup`). Main Makefile targets --------------------- @@ -748,9 +746,6 @@ Example on Linux x86-64:: At the beginning of the files, C extensions are built as built-in modules. Extensions defined after the ``*shared*`` marker are built as dynamic libraries. -The :file:`setup.py` script only builds C extensions as shared libraries using -the :mod:`distutils` module. - The :c:macro:`PyAPI_FUNC()`, :c:macro:`PyAPI_API()` and :c:macro:`PyMODINIT_FUNC()` macros of :file:`Include/pyport.h` are defined differently depending if the ``Py_BUILD_CORE_MODULE`` macro is defined: @@ -784,7 +779,7 @@ Preprocessor flags headers in a nonstandard directory ````. Both :envvar:`CPPFLAGS` and :envvar:`LDFLAGS` need to contain the shell's - value for setup.py to be able to build extension modules using the + value to be able to build extension modules using the directories specified in the environment variables. .. envvar:: BASECPPFLAGS @@ -821,8 +816,8 @@ Compiler flags .. envvar:: CFLAGS_NODIST :envvar:`CFLAGS_NODIST` is used for building the interpreter and stdlib C - extensions. Use it when a compiler flag should *not* be part of the - distutils :envvar:`CFLAGS` once Python is installed (:issue:`21121`). + extensions. Use it when a compiler flag should *not* be part of + :envvar:`CFLAGS` once Python is installed (:gh:`65320`). In particular, :envvar:`CFLAGS` should not contain: @@ -952,7 +947,7 @@ Linker flags :envvar:`LDFLAGS_NODIST` is used in the same manner as :envvar:`CFLAGS_NODIST`. Use it when a linker flag should *not* be part of - the distutils :envvar:`LDFLAGS` once Python is installed (:issue:`35257`). + :envvar:`LDFLAGS` once Python is installed (:gh:`65320`). In particular, :envvar:`LDFLAGS` should not contain: @@ -974,7 +969,7 @@ Linker flags directory ````. Both :envvar:`CPPFLAGS` and :envvar:`LDFLAGS` need to contain the shell's - value for setup.py to be able to build extension modules using the + value to be able to build extension modules using the directories specified in the environment variables. .. envvar:: LIBS From webhook-mailer at python.org Wed Aug 16 18:25:22 2023 From: webhook-mailer at python.org (iritkatriel) Date: Wed, 16 Aug 2023 22:25:22 -0000 Subject: [Python-checkins] gh-105481: generate op IDs from bytecode.c instead of hard coding them in opcode.py (#107971) Message-ID: https://github.com/python/cpython/commit/665a4391e10167dad1c854fb604c86f336fcd331 commit: 665a4391e10167dad1c854fb604c86f336fcd331 branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-16T22:25:18Z summary: gh-105481: generate op IDs from bytecode.c instead of hard coding them in opcode.py (#107971) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-15-13-06-05.gh-issue-107971.lPbx04.rst M Include/internal/pycore_opcode.h M Include/internal/pycore_opcode_metadata.h M Include/opcode_ids.h M Lib/_opcode_metadata.py M Lib/dis.py M Lib/importlib/_bootstrap_external.py M Lib/opcode.py M Lib/test/test_dis.py M Lib/test/test_embed.py M Makefile.pre.in M Objects/codeobject.c M Objects/frameobject.c M PCbuild/regen.targets M Programs/test_frozenmain.h M Python/bytecodes.c M Python/opcode_targets.h M Tools/build/deepfreeze.py M Tools/build/generate_opcode_h.py M Tools/cases_generator/generate_cases.py M Tools/scripts/summarize_stats.py diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index a187da6e24730..b47e796485236 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -14,8 +14,6 @@ extern "C" { extern const uint8_t _PyOpcode_Caches[256]; -extern const uint8_t _PyOpcode_Deopt[256]; - #ifdef NEED_OPCODE_TABLES const uint8_t _PyOpcode_Caches[256] = { @@ -34,546 +32,8 @@ const uint8_t _PyOpcode_Caches[256] = { [JUMP_BACKWARD] = 1, [TO_BOOL] = 3, }; - -const uint8_t _PyOpcode_Deopt[256] = { - [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH, - [BEFORE_WITH] = BEFORE_WITH, - [BINARY_OP] = BINARY_OP, - [BINARY_OP_ADD_FLOAT] = BINARY_OP, - [BINARY_OP_ADD_INT] = BINARY_OP, - [BINARY_OP_ADD_UNICODE] = BINARY_OP, - [BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP, - [BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP, - [BINARY_OP_MULTIPLY_INT] = BINARY_OP, - [BINARY_OP_SUBTRACT_FLOAT] = BINARY_OP, - [BINARY_OP_SUBTRACT_INT] = BINARY_OP, - [BINARY_SLICE] = BINARY_SLICE, - [BINARY_SUBSCR] = BINARY_SUBSCR, - [BINARY_SUBSCR_DICT] = BINARY_SUBSCR, - [BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR, - [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, - [BINARY_SUBSCR_STR_INT] = BINARY_SUBSCR, - [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR, - [BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP, - [BUILD_LIST] = BUILD_LIST, - [BUILD_MAP] = BUILD_MAP, - [BUILD_SET] = BUILD_SET, - [BUILD_SLICE] = BUILD_SLICE, - [BUILD_STRING] = BUILD_STRING, - [BUILD_TUPLE] = BUILD_TUPLE, - [CACHE] = CACHE, - [CALL] = CALL, - [CALL_BOUND_METHOD_EXACT_ARGS] = CALL, - [CALL_BUILTIN_CLASS] = CALL, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, - [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, - [CALL_INTRINSIC_1] = CALL_INTRINSIC_1, - [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, - [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = CALL, - [CALL_NO_KW_BUILTIN_FAST] = CALL, - [CALL_NO_KW_BUILTIN_O] = CALL, - [CALL_NO_KW_ISINSTANCE] = CALL, - [CALL_NO_KW_LEN] = CALL, - [CALL_NO_KW_LIST_APPEND] = CALL, - [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = CALL, - [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = CALL, - [CALL_NO_KW_METHOD_DESCRIPTOR_O] = CALL, - [CALL_NO_KW_STR_1] = CALL, - [CALL_NO_KW_TUPLE_1] = CALL, - [CALL_NO_KW_TYPE_1] = CALL, - [CALL_PY_EXACT_ARGS] = CALL, - [CALL_PY_WITH_DEFAULTS] = CALL, - [CHECK_EG_MATCH] = CHECK_EG_MATCH, - [CHECK_EXC_MATCH] = CHECK_EXC_MATCH, - [CLEANUP_THROW] = CLEANUP_THROW, - [COMPARE_OP] = COMPARE_OP, - [COMPARE_OP_FLOAT] = COMPARE_OP, - [COMPARE_OP_INT] = COMPARE_OP, - [COMPARE_OP_STR] = COMPARE_OP, - [CONTAINS_OP] = CONTAINS_OP, - [CONVERT_VALUE] = CONVERT_VALUE, - [COPY] = COPY, - [COPY_FREE_VARS] = COPY_FREE_VARS, - [DELETE_ATTR] = DELETE_ATTR, - [DELETE_DEREF] = DELETE_DEREF, - [DELETE_FAST] = DELETE_FAST, - [DELETE_GLOBAL] = DELETE_GLOBAL, - [DELETE_NAME] = DELETE_NAME, - [DELETE_SUBSCR] = DELETE_SUBSCR, - [DICT_MERGE] = DICT_MERGE, - [DICT_UPDATE] = DICT_UPDATE, - [END_ASYNC_FOR] = END_ASYNC_FOR, - [END_FOR] = END_FOR, - [END_SEND] = END_SEND, - [ENTER_EXECUTOR] = ENTER_EXECUTOR, - [EXIT_INIT_CHECK] = EXIT_INIT_CHECK, - [EXTENDED_ARG] = EXTENDED_ARG, - [FORMAT_SIMPLE] = FORMAT_SIMPLE, - [FORMAT_WITH_SPEC] = FORMAT_WITH_SPEC, - [FOR_ITER] = FOR_ITER, - [FOR_ITER_GEN] = FOR_ITER, - [FOR_ITER_LIST] = FOR_ITER, - [FOR_ITER_RANGE] = FOR_ITER, - [FOR_ITER_TUPLE] = FOR_ITER, - [GET_AITER] = GET_AITER, - [GET_ANEXT] = GET_ANEXT, - [GET_AWAITABLE] = GET_AWAITABLE, - [GET_ITER] = GET_ITER, - [GET_LEN] = GET_LEN, - [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER, - [IMPORT_FROM] = IMPORT_FROM, - [IMPORT_NAME] = IMPORT_NAME, - [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, - [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, - [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, - [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, - [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, - [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, - [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, - [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, - [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, - [INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, - [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, - [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, - [INSTRUMENTED_RETURN_CONST] = INSTRUMENTED_RETURN_CONST, - [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, - [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, - [INTERPRETER_EXIT] = INTERPRETER_EXIT, - [IS_OP] = IS_OP, - [JUMP_BACKWARD] = JUMP_BACKWARD, - [JUMP_BACKWARD_NO_INTERRUPT] = JUMP_BACKWARD_NO_INTERRUPT, - [JUMP_FORWARD] = JUMP_FORWARD, - [KW_NAMES] = KW_NAMES, - [LIST_APPEND] = LIST_APPEND, - [LIST_EXTEND] = LIST_EXTEND, - [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR, - [LOAD_ATTR] = LOAD_ATTR, - [LOAD_ATTR_CLASS] = LOAD_ATTR, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, - [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, - [LOAD_ATTR_METHOD_LAZY_DICT] = LOAD_ATTR, - [LOAD_ATTR_METHOD_NO_DICT] = LOAD_ATTR, - [LOAD_ATTR_METHOD_WITH_VALUES] = LOAD_ATTR, - [LOAD_ATTR_MODULE] = LOAD_ATTR, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = LOAD_ATTR, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = LOAD_ATTR, - [LOAD_ATTR_PROPERTY] = LOAD_ATTR, - [LOAD_ATTR_SLOT] = LOAD_ATTR, - [LOAD_ATTR_WITH_HINT] = LOAD_ATTR, - [LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS, - [LOAD_CONST] = LOAD_CONST, - [LOAD_DEREF] = LOAD_DEREF, - [LOAD_FAST] = LOAD_FAST, - [LOAD_FAST_AND_CLEAR] = LOAD_FAST_AND_CLEAR, - [LOAD_FAST_CHECK] = LOAD_FAST_CHECK, - [LOAD_FAST_LOAD_FAST] = LOAD_FAST_LOAD_FAST, - [LOAD_FROM_DICT_OR_DEREF] = LOAD_FROM_DICT_OR_DEREF, - [LOAD_FROM_DICT_OR_GLOBALS] = LOAD_FROM_DICT_OR_GLOBALS, - [LOAD_GLOBAL] = LOAD_GLOBAL, - [LOAD_GLOBAL_BUILTIN] = LOAD_GLOBAL, - [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, - [LOAD_LOCALS] = LOAD_LOCALS, - [LOAD_NAME] = LOAD_NAME, - [LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, - [LOAD_SUPER_ATTR_ATTR] = LOAD_SUPER_ATTR, - [LOAD_SUPER_ATTR_METHOD] = LOAD_SUPER_ATTR, - [MAKE_CELL] = MAKE_CELL, - [MAKE_FUNCTION] = MAKE_FUNCTION, - [MAP_ADD] = MAP_ADD, - [MATCH_CLASS] = MATCH_CLASS, - [MATCH_KEYS] = MATCH_KEYS, - [MATCH_MAPPING] = MATCH_MAPPING, - [MATCH_SEQUENCE] = MATCH_SEQUENCE, - [NOP] = NOP, - [POP_EXCEPT] = POP_EXCEPT, - [POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, - [POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, - [POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, - [POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE, - [POP_TOP] = POP_TOP, - [PUSH_EXC_INFO] = PUSH_EXC_INFO, - [PUSH_NULL] = PUSH_NULL, - [RAISE_VARARGS] = RAISE_VARARGS, - [RERAISE] = RERAISE, - [RESERVED] = RESERVED, - [RESUME] = RESUME, - [RETURN_CONST] = RETURN_CONST, - [RETURN_GENERATOR] = RETURN_GENERATOR, - [RETURN_VALUE] = RETURN_VALUE, - [SEND] = SEND, - [SEND_GEN] = SEND, - [SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS, - [SET_ADD] = SET_ADD, - [SET_FUNCTION_ATTRIBUTE] = SET_FUNCTION_ATTRIBUTE, - [SET_UPDATE] = SET_UPDATE, - [STORE_ATTR] = STORE_ATTR, - [STORE_ATTR_INSTANCE_VALUE] = STORE_ATTR, - [STORE_ATTR_SLOT] = STORE_ATTR, - [STORE_ATTR_WITH_HINT] = STORE_ATTR, - [STORE_DEREF] = STORE_DEREF, - [STORE_FAST] = STORE_FAST, - [STORE_FAST_LOAD_FAST] = STORE_FAST_LOAD_FAST, - [STORE_FAST_STORE_FAST] = STORE_FAST_STORE_FAST, - [STORE_GLOBAL] = STORE_GLOBAL, - [STORE_NAME] = STORE_NAME, - [STORE_SLICE] = STORE_SLICE, - [STORE_SUBSCR] = STORE_SUBSCR, - [STORE_SUBSCR_DICT] = STORE_SUBSCR, - [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, - [SWAP] = SWAP, - [TO_BOOL] = TO_BOOL, - [TO_BOOL_ALWAYS_TRUE] = TO_BOOL, - [TO_BOOL_BOOL] = TO_BOOL, - [TO_BOOL_INT] = TO_BOOL, - [TO_BOOL_LIST] = TO_BOOL, - [TO_BOOL_NONE] = TO_BOOL, - [TO_BOOL_STR] = TO_BOOL, - [UNARY_INVERT] = UNARY_INVERT, - [UNARY_NEGATIVE] = UNARY_NEGATIVE, - [UNARY_NOT] = UNARY_NOT, - [UNPACK_EX] = UNPACK_EX, - [UNPACK_SEQUENCE] = UNPACK_SEQUENCE, - [UNPACK_SEQUENCE_LIST] = UNPACK_SEQUENCE, - [UNPACK_SEQUENCE_TUPLE] = UNPACK_SEQUENCE, - [UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE, - [WITH_EXCEPT_START] = WITH_EXCEPT_START, - [YIELD_VALUE] = YIELD_VALUE, -}; -#endif // NEED_OPCODE_TABLES - - -extern const char *const _PyOpcode_OpName[268]; - -#ifdef NEED_OPCODE_TABLES -const char *const _PyOpcode_OpName[268] = { - [CACHE] = "CACHE", - [POP_TOP] = "POP_TOP", - [PUSH_NULL] = "PUSH_NULL", - [INTERPRETER_EXIT] = "INTERPRETER_EXIT", - [END_FOR] = "END_FOR", - [END_SEND] = "END_SEND", - [TO_BOOL] = "TO_BOOL", - [TO_BOOL_ALWAYS_TRUE] = "TO_BOOL_ALWAYS_TRUE", - [TO_BOOL_BOOL] = "TO_BOOL_BOOL", - [NOP] = "NOP", - [TO_BOOL_INT] = "TO_BOOL_INT", - [UNARY_NEGATIVE] = "UNARY_NEGATIVE", - [UNARY_NOT] = "UNARY_NOT", - [TO_BOOL_LIST] = "TO_BOOL_LIST", - [TO_BOOL_NONE] = "TO_BOOL_NONE", - [UNARY_INVERT] = "UNARY_INVERT", - [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", - [RESERVED] = "RESERVED", - [TO_BOOL_STR] = "TO_BOOL_STR", - [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", - [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", - [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", - [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", - [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", - [MAKE_FUNCTION] = "MAKE_FUNCTION", - [BINARY_SUBSCR] = "BINARY_SUBSCR", - [BINARY_SLICE] = "BINARY_SLICE", - [STORE_SLICE] = "STORE_SLICE", - [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", - [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", - [GET_LEN] = "GET_LEN", - [MATCH_MAPPING] = "MATCH_MAPPING", - [MATCH_SEQUENCE] = "MATCH_SEQUENCE", - [MATCH_KEYS] = "MATCH_KEYS", - [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", - [PUSH_EXC_INFO] = "PUSH_EXC_INFO", - [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", - [CHECK_EG_MATCH] = "CHECK_EG_MATCH", - [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", - [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", - [FORMAT_SIMPLE] = "FORMAT_SIMPLE", - [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", - [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", - [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", - [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", - [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", - [SEND_GEN] = "SEND_GEN", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [WITH_EXCEPT_START] = "WITH_EXCEPT_START", - [GET_AITER] = "GET_AITER", - [GET_ANEXT] = "GET_ANEXT", - [BEFORE_ASYNC_WITH] = "BEFORE_ASYNC_WITH", - [BEFORE_WITH] = "BEFORE_WITH", - [END_ASYNC_FOR] = "END_ASYNC_FOR", - [CLEANUP_THROW] = "CLEANUP_THROW", - [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", - [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [STORE_SUBSCR] = "STORE_SUBSCR", - [DELETE_SUBSCR] = "DELETE_SUBSCR", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", - [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [GET_ITER] = "GET_ITER", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", - [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", - [RETURN_GENERATOR] = "RETURN_GENERATOR", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", - [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", - [RETURN_VALUE] = "RETURN_VALUE", - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", - [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", - [LOAD_LOCALS] = "LOAD_LOCALS", - [COMPARE_OP_INT] = "COMPARE_OP_INT", - [POP_EXCEPT] = "POP_EXCEPT", - [STORE_NAME] = "STORE_NAME", - [DELETE_NAME] = "DELETE_NAME", - [UNPACK_SEQUENCE] = "UNPACK_SEQUENCE", - [FOR_ITER] = "FOR_ITER", - [UNPACK_EX] = "UNPACK_EX", - [STORE_ATTR] = "STORE_ATTR", - [DELETE_ATTR] = "DELETE_ATTR", - [STORE_GLOBAL] = "STORE_GLOBAL", - [DELETE_GLOBAL] = "DELETE_GLOBAL", - [SWAP] = "SWAP", - [LOAD_CONST] = "LOAD_CONST", - [LOAD_NAME] = "LOAD_NAME", - [BUILD_TUPLE] = "BUILD_TUPLE", - [BUILD_LIST] = "BUILD_LIST", - [BUILD_SET] = "BUILD_SET", - [BUILD_MAP] = "BUILD_MAP", - [LOAD_ATTR] = "LOAD_ATTR", - [COMPARE_OP] = "COMPARE_OP", - [IMPORT_NAME] = "IMPORT_NAME", - [IMPORT_FROM] = "IMPORT_FROM", - [JUMP_FORWARD] = "JUMP_FORWARD", - [COMPARE_OP_STR] = "COMPARE_OP_STR", - [FOR_ITER_LIST] = "FOR_ITER_LIST", - [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", - [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", - [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", - [LOAD_GLOBAL] = "LOAD_GLOBAL", - [IS_OP] = "IS_OP", - [CONTAINS_OP] = "CONTAINS_OP", - [RERAISE] = "RERAISE", - [COPY] = "COPY", - [RETURN_CONST] = "RETURN_CONST", - [BINARY_OP] = "BINARY_OP", - [SEND] = "SEND", - [LOAD_FAST] = "LOAD_FAST", - [STORE_FAST] = "STORE_FAST", - [DELETE_FAST] = "DELETE_FAST", - [LOAD_FAST_CHECK] = "LOAD_FAST_CHECK", - [POP_JUMP_IF_NOT_NONE] = "POP_JUMP_IF_NOT_NONE", - [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", - [RAISE_VARARGS] = "RAISE_VARARGS", - [GET_AWAITABLE] = "GET_AWAITABLE", - [FOR_ITER_RANGE] = "FOR_ITER_RANGE", - [BUILD_SLICE] = "BUILD_SLICE", - [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", - [MAKE_CELL] = "MAKE_CELL", - [FOR_ITER_GEN] = "FOR_ITER_GEN", - [LOAD_DEREF] = "LOAD_DEREF", - [STORE_DEREF] = "STORE_DEREF", - [DELETE_DEREF] = "DELETE_DEREF", - [JUMP_BACKWARD] = "JUMP_BACKWARD", - [LOAD_SUPER_ATTR] = "LOAD_SUPER_ATTR", - [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [LOAD_FAST_AND_CLEAR] = "LOAD_FAST_AND_CLEAR", - [EXTENDED_ARG] = "EXTENDED_ARG", - [LIST_APPEND] = "LIST_APPEND", - [SET_ADD] = "SET_ADD", - [MAP_ADD] = "MAP_ADD", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", - [COPY_FREE_VARS] = "COPY_FREE_VARS", - [YIELD_VALUE] = "YIELD_VALUE", - [RESUME] = "RESUME", - [MATCH_CLASS] = "MATCH_CLASS", - [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", - [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", - [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", - [BUILD_STRING] = "BUILD_STRING", - [CONVERT_VALUE] = "CONVERT_VALUE", - [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", - [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", - [LIST_EXTEND] = "LIST_EXTEND", - [SET_UPDATE] = "SET_UPDATE", - [DICT_MERGE] = "DICT_MERGE", - [DICT_UPDATE] = "DICT_UPDATE", - [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", - [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", - [LOAD_FAST_LOAD_FAST] = "LOAD_FAST_LOAD_FAST", - [STORE_FAST_LOAD_FAST] = "STORE_FAST_LOAD_FAST", - [STORE_FAST_STORE_FAST] = "STORE_FAST_STORE_FAST", - [CALL] = "CALL", - [KW_NAMES] = "KW_NAMES", - [CALL_INTRINSIC_1] = "CALL_INTRINSIC_1", - [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", - [LOAD_FROM_DICT_OR_GLOBALS] = "LOAD_FROM_DICT_OR_GLOBALS", - [LOAD_FROM_DICT_OR_DEREF] = "LOAD_FROM_DICT_OR_DEREF", - [SET_FUNCTION_ATTRIBUTE] = "SET_FUNCTION_ATTRIBUTE", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", - [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", - [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", - [CALL_NO_KW_LIST_APPEND] = "CALL_NO_KW_LIST_APPEND", - [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", - [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", - [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = "CALL_NO_KW_ALLOC_AND_ENTER_INIT", - [187] = "<187>", - [188] = "<188>", - [189] = "<189>", - [190] = "<190>", - [191] = "<191>", - [192] = "<192>", - [193] = "<193>", - [194] = "<194>", - [195] = "<195>", - [196] = "<196>", - [197] = "<197>", - [198] = "<198>", - [199] = "<199>", - [200] = "<200>", - [201] = "<201>", - [202] = "<202>", - [203] = "<203>", - [204] = "<204>", - [205] = "<205>", - [206] = "<206>", - [207] = "<207>", - [208] = "<208>", - [209] = "<209>", - [210] = "<210>", - [211] = "<211>", - [212] = "<212>", - [213] = "<213>", - [214] = "<214>", - [215] = "<215>", - [216] = "<216>", - [217] = "<217>", - [218] = "<218>", - [219] = "<219>", - [220] = "<220>", - [221] = "<221>", - [222] = "<222>", - [223] = "<223>", - [224] = "<224>", - [225] = "<225>", - [226] = "<226>", - [227] = "<227>", - [228] = "<228>", - [229] = "<229>", - [ENTER_EXECUTOR] = "ENTER_EXECUTOR", - [231] = "<231>", - [232] = "<232>", - [233] = "<233>", - [234] = "<234>", - [235] = "<235>", - [236] = "<236>", - [INSTRUMENTED_LOAD_SUPER_ATTR] = "INSTRUMENTED_LOAD_SUPER_ATTR", - [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", - [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", - [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", - [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", - [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", - [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", - [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", - [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", - [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", - [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", - [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", - [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", - [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", - [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", - [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", - [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", - [255] = "<255>", - [SETUP_FINALLY] = "SETUP_FINALLY", - [SETUP_CLEANUP] = "SETUP_CLEANUP", - [SETUP_WITH] = "SETUP_WITH", - [POP_BLOCK] = "POP_BLOCK", - [JUMP] = "JUMP", - [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", - [LOAD_METHOD] = "LOAD_METHOD", - [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", - [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", - [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", - [STORE_FAST_MAYBE_NULL] = "STORE_FAST_MAYBE_NULL", - [LOAD_CLOSURE] = "LOAD_CLOSURE", -}; #endif // NEED_OPCODE_TABLES -#define EXTRA_CASES \ - case 187: \ - case 188: \ - case 189: \ - case 190: \ - case 191: \ - case 192: \ - case 193: \ - case 194: \ - case 195: \ - case 196: \ - case 197: \ - case 198: \ - case 199: \ - case 200: \ - case 201: \ - case 202: \ - case 203: \ - case 204: \ - case 205: \ - case 206: \ - case 207: \ - case 208: \ - case 209: \ - case 210: \ - case 211: \ - case 212: \ - case 213: \ - case 214: \ - case 215: \ - case 216: \ - case 217: \ - case 218: \ - case 219: \ - case 220: \ - case 221: \ - case 222: \ - case 223: \ - case 224: \ - case 225: \ - case 226: \ - case 227: \ - case 228: \ - case 229: \ - case 231: \ - case 232: \ - case 233: \ - case 234: \ - case 235: \ - case 236: \ - case 255: \ - ; - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index df9b4184e2a50..7dbd78b32a895 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1537,3 +1537,493 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [INSERT] = "INSERT", }; #endif // NEED_OPCODE_METADATA + +extern const char *const _PyOpcode_OpName[268]; +#ifdef NEED_OPCODE_METADATA +const char *const _PyOpcode_OpName[268] = { + [CACHE] = "CACHE", + [RESERVED] = "RESERVED", + [RESUME] = "RESUME", + [BEFORE_ASYNC_WITH] = "BEFORE_ASYNC_WITH", + [BEFORE_WITH] = "BEFORE_WITH", + [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", + [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", + [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", + [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", + [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", + [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", + [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", + [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", + [BINARY_SLICE] = "BINARY_SLICE", + [BINARY_SUBSCR] = "BINARY_SUBSCR", + [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", + [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", + [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", + [BINARY_SUBSCR_STR_INT] = "BINARY_SUBSCR_STR_INT", + [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", + [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", + [CLEANUP_THROW] = "CLEANUP_THROW", + [DELETE_SUBSCR] = "DELETE_SUBSCR", + [END_ASYNC_FOR] = "END_ASYNC_FOR", + [END_FOR] = "END_FOR", + [END_SEND] = "END_SEND", + [EXIT_INIT_CHECK] = "EXIT_INIT_CHECK", + [FORMAT_SIMPLE] = "FORMAT_SIMPLE", + [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", + [GET_AITER] = "GET_AITER", + [GET_ANEXT] = "GET_ANEXT", + [GET_ITER] = "GET_ITER", + [GET_LEN] = "GET_LEN", + [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", + [INTERPRETER_EXIT] = "INTERPRETER_EXIT", + [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", + [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_LOCALS] = "LOAD_LOCALS", + [MAKE_FUNCTION] = "MAKE_FUNCTION", + [MATCH_KEYS] = "MATCH_KEYS", + [MATCH_MAPPING] = "MATCH_MAPPING", + [MATCH_SEQUENCE] = "MATCH_SEQUENCE", + [NOP] = "NOP", + [POP_EXCEPT] = "POP_EXCEPT", + [POP_TOP] = "POP_TOP", + [PUSH_EXC_INFO] = "PUSH_EXC_INFO", + [PUSH_NULL] = "PUSH_NULL", + [RETURN_GENERATOR] = "RETURN_GENERATOR", + [RETURN_VALUE] = "RETURN_VALUE", + [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", + [STORE_SLICE] = "STORE_SLICE", + [STORE_SUBSCR] = "STORE_SUBSCR", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", + [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [TO_BOOL] = "TO_BOOL", + [TO_BOOL_ALWAYS_TRUE] = "TO_BOOL_ALWAYS_TRUE", + [TO_BOOL_BOOL] = "TO_BOOL_BOOL", + [TO_BOOL_INT] = "TO_BOOL_INT", + [TO_BOOL_LIST] = "TO_BOOL_LIST", + [TO_BOOL_NONE] = "TO_BOOL_NONE", + [TO_BOOL_STR] = "TO_BOOL_STR", + [UNARY_INVERT] = "UNARY_INVERT", + [UNARY_NEGATIVE] = "UNARY_NEGATIVE", + [UNARY_NOT] = "UNARY_NOT", + [WITH_EXCEPT_START] = "WITH_EXCEPT_START", + [BINARY_OP] = "BINARY_OP", + [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", + [BUILD_LIST] = "BUILD_LIST", + [BUILD_MAP] = "BUILD_MAP", + [BUILD_SET] = "BUILD_SET", + [BUILD_SLICE] = "BUILD_SLICE", + [BUILD_STRING] = "BUILD_STRING", + [BUILD_TUPLE] = "BUILD_TUPLE", + [CALL] = "CALL", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", + [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", + [CALL_INTRINSIC_1] = "CALL_INTRINSIC_1", + [CALL_INTRINSIC_2] = "CALL_INTRINSIC_2", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = "CALL_NO_KW_ALLOC_AND_ENTER_INIT", + [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", + [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", + [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", + [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", + [CALL_NO_KW_LIST_APPEND] = "CALL_NO_KW_LIST_APPEND", + [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", + [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", + [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", + [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", + [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", + [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", + [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", + [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", + [COMPARE_OP] = "COMPARE_OP", + [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", + [COMPARE_OP_INT] = "COMPARE_OP_INT", + [COMPARE_OP_STR] = "COMPARE_OP_STR", + [CONTAINS_OP] = "CONTAINS_OP", + [CONVERT_VALUE] = "CONVERT_VALUE", + [COPY] = "COPY", + [COPY_FREE_VARS] = "COPY_FREE_VARS", + [DELETE_ATTR] = "DELETE_ATTR", + [DELETE_DEREF] = "DELETE_DEREF", + [DELETE_FAST] = "DELETE_FAST", + [DELETE_GLOBAL] = "DELETE_GLOBAL", + [DELETE_NAME] = "DELETE_NAME", + [DICT_MERGE] = "DICT_MERGE", + [DICT_UPDATE] = "DICT_UPDATE", + [ENTER_EXECUTOR] = "ENTER_EXECUTOR", + [EXTENDED_ARG] = "EXTENDED_ARG", + [FOR_ITER] = "FOR_ITER", + [FOR_ITER_GEN] = "FOR_ITER_GEN", + [FOR_ITER_LIST] = "FOR_ITER_LIST", + [FOR_ITER_RANGE] = "FOR_ITER_RANGE", + [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", + [GET_AWAITABLE] = "GET_AWAITABLE", + [IMPORT_FROM] = "IMPORT_FROM", + [IMPORT_NAME] = "IMPORT_NAME", + [IS_OP] = "IS_OP", + [JUMP_BACKWARD] = "JUMP_BACKWARD", + [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", + [JUMP_FORWARD] = "JUMP_FORWARD", + [KW_NAMES] = "KW_NAMES", + [LIST_APPEND] = "LIST_APPEND", + [LIST_EXTEND] = "LIST_EXTEND", + [LOAD_ATTR] = "LOAD_ATTR", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [LOAD_CONST] = "LOAD_CONST", + [LOAD_DEREF] = "LOAD_DEREF", + [LOAD_FAST] = "LOAD_FAST", + [LOAD_FAST_AND_CLEAR] = "LOAD_FAST_AND_CLEAR", + [LOAD_FAST_CHECK] = "LOAD_FAST_CHECK", + [LOAD_FAST_LOAD_FAST] = "LOAD_FAST_LOAD_FAST", + [LOAD_FROM_DICT_OR_DEREF] = "LOAD_FROM_DICT_OR_DEREF", + [LOAD_FROM_DICT_OR_GLOBALS] = "LOAD_FROM_DICT_OR_GLOBALS", + [LOAD_GLOBAL] = "LOAD_GLOBAL", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [LOAD_NAME] = "LOAD_NAME", + [LOAD_SUPER_ATTR] = "LOAD_SUPER_ATTR", + [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", + [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", + [MAKE_CELL] = "MAKE_CELL", + [MAP_ADD] = "MAP_ADD", + [MATCH_CLASS] = "MATCH_CLASS", + [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", + [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", + [POP_JUMP_IF_NOT_NONE] = "POP_JUMP_IF_NOT_NONE", + [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", + [RAISE_VARARGS] = "RAISE_VARARGS", + [RERAISE] = "RERAISE", + [RETURN_CONST] = "RETURN_CONST", + [SEND] = "SEND", + [SEND_GEN] = "SEND_GEN", + [SET_ADD] = "SET_ADD", + [SET_FUNCTION_ATTRIBUTE] = "SET_FUNCTION_ATTRIBUTE", + [SET_UPDATE] = "SET_UPDATE", + [STORE_ATTR] = "STORE_ATTR", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_DEREF] = "STORE_DEREF", + [STORE_FAST] = "STORE_FAST", + [STORE_FAST_LOAD_FAST] = "STORE_FAST_LOAD_FAST", + [STORE_FAST_STORE_FAST] = "STORE_FAST_STORE_FAST", + [STORE_GLOBAL] = "STORE_GLOBAL", + [STORE_NAME] = "STORE_NAME", + [SWAP] = "SWAP", + [UNPACK_EX] = "UNPACK_EX", + [UNPACK_SEQUENCE] = "UNPACK_SEQUENCE", + [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", + [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", + [YIELD_VALUE] = "YIELD_VALUE", + [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", + [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", + [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", + [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", + [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", + [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", + [INSTRUMENTED_LOAD_SUPER_ATTR] = "INSTRUMENTED_LOAD_SUPER_ATTR", + [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", + [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", + [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", + [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", + [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", + [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", + [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", + [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", + [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", + [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", + [JUMP] = "JUMP", + [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", + [LOAD_CLOSURE] = "LOAD_CLOSURE", + [LOAD_METHOD] = "LOAD_METHOD", + [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", + [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", + [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", + [POP_BLOCK] = "POP_BLOCK", + [SETUP_CLEANUP] = "SETUP_CLEANUP", + [SETUP_FINALLY] = "SETUP_FINALLY", + [SETUP_WITH] = "SETUP_WITH", + [STORE_FAST_MAYBE_NULL] = "STORE_FAST_MAYBE_NULL", +}; +#endif // NEED_OPCODE_METADATA + +extern const uint8_t _PyOpcode_Deopt[256]; +#ifdef NEED_OPCODE_METADATA +const uint8_t _PyOpcode_Deopt[256] = { + [BEFORE_ASYNC_WITH] = BEFORE_ASYNC_WITH, + [BEFORE_WITH] = BEFORE_WITH, + [BINARY_OP] = BINARY_OP, + [BINARY_OP_ADD_FLOAT] = BINARY_OP, + [BINARY_OP_ADD_INT] = BINARY_OP, + [BINARY_OP_ADD_UNICODE] = BINARY_OP, + [BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP, + [BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP, + [BINARY_OP_MULTIPLY_INT] = BINARY_OP, + [BINARY_OP_SUBTRACT_FLOAT] = BINARY_OP, + [BINARY_OP_SUBTRACT_INT] = BINARY_OP, + [BINARY_SLICE] = BINARY_SLICE, + [BINARY_SUBSCR] = BINARY_SUBSCR, + [BINARY_SUBSCR_DICT] = BINARY_SUBSCR, + [BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR, + [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, + [BINARY_SUBSCR_STR_INT] = BINARY_SUBSCR, + [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR, + [BUILD_CONST_KEY_MAP] = BUILD_CONST_KEY_MAP, + [BUILD_LIST] = BUILD_LIST, + [BUILD_MAP] = BUILD_MAP, + [BUILD_SET] = BUILD_SET, + [BUILD_SLICE] = BUILD_SLICE, + [BUILD_STRING] = BUILD_STRING, + [BUILD_TUPLE] = BUILD_TUPLE, + [CACHE] = CACHE, + [CALL] = CALL, + [CALL_BOUND_METHOD_EXACT_ARGS] = CALL, + [CALL_BUILTIN_CLASS] = CALL, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, + [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, + [CALL_INTRINSIC_1] = CALL_INTRINSIC_1, + [CALL_INTRINSIC_2] = CALL_INTRINSIC_2, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, + [CALL_NO_KW_ALLOC_AND_ENTER_INIT] = CALL, + [CALL_NO_KW_BUILTIN_FAST] = CALL, + [CALL_NO_KW_BUILTIN_O] = CALL, + [CALL_NO_KW_ISINSTANCE] = CALL, + [CALL_NO_KW_LEN] = CALL, + [CALL_NO_KW_LIST_APPEND] = CALL, + [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = CALL, + [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = CALL, + [CALL_NO_KW_METHOD_DESCRIPTOR_O] = CALL, + [CALL_NO_KW_STR_1] = CALL, + [CALL_NO_KW_TUPLE_1] = CALL, + [CALL_NO_KW_TYPE_1] = CALL, + [CALL_PY_EXACT_ARGS] = CALL, + [CALL_PY_WITH_DEFAULTS] = CALL, + [CHECK_EG_MATCH] = CHECK_EG_MATCH, + [CHECK_EXC_MATCH] = CHECK_EXC_MATCH, + [CLEANUP_THROW] = CLEANUP_THROW, + [COMPARE_OP] = COMPARE_OP, + [COMPARE_OP_FLOAT] = COMPARE_OP, + [COMPARE_OP_INT] = COMPARE_OP, + [COMPARE_OP_STR] = COMPARE_OP, + [CONTAINS_OP] = CONTAINS_OP, + [CONVERT_VALUE] = CONVERT_VALUE, + [COPY] = COPY, + [COPY_FREE_VARS] = COPY_FREE_VARS, + [DELETE_ATTR] = DELETE_ATTR, + [DELETE_DEREF] = DELETE_DEREF, + [DELETE_FAST] = DELETE_FAST, + [DELETE_GLOBAL] = DELETE_GLOBAL, + [DELETE_NAME] = DELETE_NAME, + [DELETE_SUBSCR] = DELETE_SUBSCR, + [DICT_MERGE] = DICT_MERGE, + [DICT_UPDATE] = DICT_UPDATE, + [END_ASYNC_FOR] = END_ASYNC_FOR, + [END_FOR] = END_FOR, + [END_SEND] = END_SEND, + [ENTER_EXECUTOR] = ENTER_EXECUTOR, + [EXIT_INIT_CHECK] = EXIT_INIT_CHECK, + [EXTENDED_ARG] = EXTENDED_ARG, + [FORMAT_SIMPLE] = FORMAT_SIMPLE, + [FORMAT_WITH_SPEC] = FORMAT_WITH_SPEC, + [FOR_ITER] = FOR_ITER, + [FOR_ITER_GEN] = FOR_ITER, + [FOR_ITER_LIST] = FOR_ITER, + [FOR_ITER_RANGE] = FOR_ITER, + [FOR_ITER_TUPLE] = FOR_ITER, + [GET_AITER] = GET_AITER, + [GET_ANEXT] = GET_ANEXT, + [GET_AWAITABLE] = GET_AWAITABLE, + [GET_ITER] = GET_ITER, + [GET_LEN] = GET_LEN, + [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER, + [IMPORT_FROM] = IMPORT_FROM, + [IMPORT_NAME] = IMPORT_NAME, + [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, + [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, + [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, + [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, + [INSTRUMENTED_LOAD_SUPER_ATTR] = INSTRUMENTED_LOAD_SUPER_ATTR, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_CONST] = INSTRUMENTED_RETURN_CONST, + [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, + [INTERPRETER_EXIT] = INTERPRETER_EXIT, + [IS_OP] = IS_OP, + [JUMP_BACKWARD] = JUMP_BACKWARD, + [JUMP_BACKWARD_NO_INTERRUPT] = JUMP_BACKWARD_NO_INTERRUPT, + [JUMP_FORWARD] = JUMP_FORWARD, + [KW_NAMES] = KW_NAMES, + [LIST_APPEND] = LIST_APPEND, + [LIST_EXTEND] = LIST_EXTEND, + [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR, + [LOAD_ATTR] = LOAD_ATTR, + [LOAD_ATTR_CLASS] = LOAD_ATTR, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, + [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, + [LOAD_ATTR_METHOD_LAZY_DICT] = LOAD_ATTR, + [LOAD_ATTR_METHOD_NO_DICT] = LOAD_ATTR, + [LOAD_ATTR_METHOD_WITH_VALUES] = LOAD_ATTR, + [LOAD_ATTR_MODULE] = LOAD_ATTR, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = LOAD_ATTR, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = LOAD_ATTR, + [LOAD_ATTR_PROPERTY] = LOAD_ATTR, + [LOAD_ATTR_SLOT] = LOAD_ATTR, + [LOAD_ATTR_WITH_HINT] = LOAD_ATTR, + [LOAD_BUILD_CLASS] = LOAD_BUILD_CLASS, + [LOAD_CONST] = LOAD_CONST, + [LOAD_DEREF] = LOAD_DEREF, + [LOAD_FAST] = LOAD_FAST, + [LOAD_FAST_AND_CLEAR] = LOAD_FAST_AND_CLEAR, + [LOAD_FAST_CHECK] = LOAD_FAST_CHECK, + [LOAD_FAST_LOAD_FAST] = LOAD_FAST_LOAD_FAST, + [LOAD_FROM_DICT_OR_DEREF] = LOAD_FROM_DICT_OR_DEREF, + [LOAD_FROM_DICT_OR_GLOBALS] = LOAD_FROM_DICT_OR_GLOBALS, + [LOAD_GLOBAL] = LOAD_GLOBAL, + [LOAD_GLOBAL_BUILTIN] = LOAD_GLOBAL, + [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, + [LOAD_LOCALS] = LOAD_LOCALS, + [LOAD_NAME] = LOAD_NAME, + [LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_ATTR] = LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_METHOD] = LOAD_SUPER_ATTR, + [MAKE_CELL] = MAKE_CELL, + [MAKE_FUNCTION] = MAKE_FUNCTION, + [MAP_ADD] = MAP_ADD, + [MATCH_CLASS] = MATCH_CLASS, + [MATCH_KEYS] = MATCH_KEYS, + [MATCH_MAPPING] = MATCH_MAPPING, + [MATCH_SEQUENCE] = MATCH_SEQUENCE, + [NOP] = NOP, + [POP_EXCEPT] = POP_EXCEPT, + [POP_JUMP_IF_FALSE] = POP_JUMP_IF_FALSE, + [POP_JUMP_IF_NONE] = POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = POP_JUMP_IF_NOT_NONE, + [POP_JUMP_IF_TRUE] = POP_JUMP_IF_TRUE, + [POP_TOP] = POP_TOP, + [PUSH_EXC_INFO] = PUSH_EXC_INFO, + [PUSH_NULL] = PUSH_NULL, + [RAISE_VARARGS] = RAISE_VARARGS, + [RERAISE] = RERAISE, + [RESERVED] = RESERVED, + [RESUME] = RESUME, + [RETURN_CONST] = RETURN_CONST, + [RETURN_GENERATOR] = RETURN_GENERATOR, + [RETURN_VALUE] = RETURN_VALUE, + [SEND] = SEND, + [SEND_GEN] = SEND, + [SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS, + [SET_ADD] = SET_ADD, + [SET_FUNCTION_ATTRIBUTE] = SET_FUNCTION_ATTRIBUTE, + [SET_UPDATE] = SET_UPDATE, + [STORE_ATTR] = STORE_ATTR, + [STORE_ATTR_INSTANCE_VALUE] = STORE_ATTR, + [STORE_ATTR_SLOT] = STORE_ATTR, + [STORE_ATTR_WITH_HINT] = STORE_ATTR, + [STORE_DEREF] = STORE_DEREF, + [STORE_FAST] = STORE_FAST, + [STORE_FAST_LOAD_FAST] = STORE_FAST_LOAD_FAST, + [STORE_FAST_STORE_FAST] = STORE_FAST_STORE_FAST, + [STORE_GLOBAL] = STORE_GLOBAL, + [STORE_NAME] = STORE_NAME, + [STORE_SLICE] = STORE_SLICE, + [STORE_SUBSCR] = STORE_SUBSCR, + [STORE_SUBSCR_DICT] = STORE_SUBSCR, + [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, + [SWAP] = SWAP, + [TO_BOOL] = TO_BOOL, + [TO_BOOL_ALWAYS_TRUE] = TO_BOOL, + [TO_BOOL_BOOL] = TO_BOOL, + [TO_BOOL_INT] = TO_BOOL, + [TO_BOOL_LIST] = TO_BOOL, + [TO_BOOL_NONE] = TO_BOOL, + [TO_BOOL_STR] = TO_BOOL, + [UNARY_INVERT] = UNARY_INVERT, + [UNARY_NEGATIVE] = UNARY_NEGATIVE, + [UNARY_NOT] = UNARY_NOT, + [UNPACK_EX] = UNPACK_EX, + [UNPACK_SEQUENCE] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_LIST] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_TUPLE] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE, + [WITH_EXCEPT_START] = WITH_EXCEPT_START, + [YIELD_VALUE] = YIELD_VALUE, +}; +#endif // NEED_OPCODE_METADATA + +#define EXTRA_CASES \ + case 188: \ + case 189: \ + case 190: \ + case 191: \ + case 192: \ + case 193: \ + case 194: \ + case 195: \ + case 196: \ + case 197: \ + case 198: \ + case 199: \ + case 200: \ + case 201: \ + case 202: \ + case 203: \ + case 204: \ + case 205: \ + case 206: \ + case 207: \ + case 208: \ + case 209: \ + case 210: \ + case 211: \ + case 212: \ + case 213: \ + case 214: \ + case 215: \ + case 216: \ + case 217: \ + case 218: \ + case 219: \ + case 220: \ + case 221: \ + case 222: \ + case 223: \ + case 224: \ + case 225: \ + case 226: \ + case 227: \ + case 228: \ + case 229: \ + case 230: \ + case 231: \ + case 232: \ + case 233: \ + case 234: \ + case 235: \ + case 236: \ + case 255: \ + ; + diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 2d9d24cca4542..cd43716415d1d 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -1,4 +1,7 @@ -// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py +// This file is generated by Tools/cases_generator/generate_cases.py +// from: +// Python/bytecodes.c +// Do not edit! #ifndef Py_OPCODE_IDS_H #define Py_OPCODE_IDS_H @@ -6,228 +9,227 @@ extern "C" { #endif - /* Instruction opcodes for compiled code */ #define CACHE 0 -#define POP_TOP 1 -#define PUSH_NULL 2 -#define INTERPRETER_EXIT 3 -#define END_FOR 4 -#define END_SEND 5 -#define TO_BOOL 6 -#define NOP 9 -#define UNARY_NEGATIVE 11 -#define UNARY_NOT 12 -#define UNARY_INVERT 15 -#define EXIT_INIT_CHECK 16 +#define BEFORE_ASYNC_WITH 1 +#define BEFORE_WITH 2 +#define BINARY_OP_ADD_FLOAT 3 +#define BINARY_OP_ADD_INT 4 +#define BINARY_OP_ADD_UNICODE 5 +#define BINARY_OP_INPLACE_ADD_UNICODE 6 +#define BINARY_OP_MULTIPLY_FLOAT 7 +#define BINARY_OP_MULTIPLY_INT 8 +#define BINARY_OP_SUBTRACT_FLOAT 9 +#define BINARY_OP_SUBTRACT_INT 10 +#define BINARY_SLICE 11 +#define BINARY_SUBSCR 12 +#define BINARY_SUBSCR_DICT 13 +#define BINARY_SUBSCR_GETITEM 14 +#define BINARY_SUBSCR_LIST_INT 15 +#define BINARY_SUBSCR_STR_INT 16 #define RESERVED 17 -#define MAKE_FUNCTION 24 -#define BINARY_SUBSCR 25 -#define BINARY_SLICE 26 -#define STORE_SLICE 27 -#define GET_LEN 30 -#define MATCH_MAPPING 31 -#define MATCH_SEQUENCE 32 -#define MATCH_KEYS 33 -#define PUSH_EXC_INFO 35 -#define CHECK_EXC_MATCH 36 -#define CHECK_EG_MATCH 37 -#define FORMAT_SIMPLE 40 -#define FORMAT_WITH_SPEC 41 -#define WITH_EXCEPT_START 49 -#define GET_AITER 50 -#define GET_ANEXT 51 -#define BEFORE_ASYNC_WITH 52 -#define BEFORE_WITH 53 -#define END_ASYNC_FOR 54 -#define CLEANUP_THROW 55 -#define STORE_SUBSCR 60 -#define DELETE_SUBSCR 61 -#define GET_ITER 68 -#define GET_YIELD_FROM_ITER 69 -#define LOAD_BUILD_CLASS 71 -#define LOAD_ASSERTION_ERROR 74 -#define RETURN_GENERATOR 75 -#define RETURN_VALUE 83 -#define SETUP_ANNOTATIONS 85 -#define LOAD_LOCALS 87 -#define POP_EXCEPT 89 -#define STORE_NAME 90 -#define DELETE_NAME 91 -#define UNPACK_SEQUENCE 92 -#define FOR_ITER 93 -#define UNPACK_EX 94 -#define STORE_ATTR 95 -#define DELETE_ATTR 96 -#define STORE_GLOBAL 97 -#define DELETE_GLOBAL 98 -#define SWAP 99 -#define LOAD_CONST 100 -#define LOAD_NAME 101 -#define BUILD_TUPLE 102 -#define BUILD_LIST 103 -#define BUILD_SET 104 -#define BUILD_MAP 105 -#define LOAD_ATTR 106 -#define COMPARE_OP 107 -#define IMPORT_NAME 108 -#define IMPORT_FROM 109 -#define JUMP_FORWARD 110 -#define POP_JUMP_IF_FALSE 114 -#define POP_JUMP_IF_TRUE 115 -#define LOAD_GLOBAL 116 -#define IS_OP 117 -#define CONTAINS_OP 118 -#define RERAISE 119 -#define COPY 120 -#define RETURN_CONST 121 -#define BINARY_OP 122 -#define SEND 123 -#define LOAD_FAST 124 -#define STORE_FAST 125 -#define DELETE_FAST 126 -#define LOAD_FAST_CHECK 127 -#define POP_JUMP_IF_NOT_NONE 128 -#define POP_JUMP_IF_NONE 129 -#define RAISE_VARARGS 130 -#define GET_AWAITABLE 131 -#define BUILD_SLICE 133 -#define JUMP_BACKWARD_NO_INTERRUPT 134 -#define MAKE_CELL 135 -#define LOAD_DEREF 137 -#define STORE_DEREF 138 -#define DELETE_DEREF 139 -#define JUMP_BACKWARD 140 -#define LOAD_SUPER_ATTR 141 -#define CALL_FUNCTION_EX 142 -#define LOAD_FAST_AND_CLEAR 143 -#define EXTENDED_ARG 144 -#define LIST_APPEND 145 -#define SET_ADD 146 -#define MAP_ADD 147 -#define COPY_FREE_VARS 149 -#define YIELD_VALUE 150 -#define RESUME 151 -#define MATCH_CLASS 152 -#define BUILD_CONST_KEY_MAP 156 -#define BUILD_STRING 157 -#define CONVERT_VALUE 158 -#define LIST_EXTEND 162 -#define SET_UPDATE 163 -#define DICT_MERGE 164 -#define DICT_UPDATE 165 -#define LOAD_FAST_LOAD_FAST 168 -#define STORE_FAST_LOAD_FAST 169 -#define STORE_FAST_STORE_FAST 170 -#define CALL 171 -#define KW_NAMES 172 -#define CALL_INTRINSIC_1 173 -#define CALL_INTRINSIC_2 174 -#define LOAD_FROM_DICT_OR_GLOBALS 175 -#define LOAD_FROM_DICT_OR_DEREF 176 -#define SET_FUNCTION_ATTRIBUTE 177 -#define ENTER_EXECUTOR 230 +#define BINARY_SUBSCR_TUPLE_INT 18 +#define CHECK_EG_MATCH 19 +#define CHECK_EXC_MATCH 20 +#define CLEANUP_THROW 21 +#define DELETE_SUBSCR 22 +#define END_ASYNC_FOR 23 +#define END_FOR 24 +#define END_SEND 25 +#define EXIT_INIT_CHECK 26 +#define FORMAT_SIMPLE 27 +#define FORMAT_WITH_SPEC 28 +#define GET_AITER 29 +#define GET_ANEXT 30 +#define GET_ITER 31 +#define GET_LEN 32 +#define GET_YIELD_FROM_ITER 33 +#define INTERPRETER_EXIT 34 +#define LOAD_ASSERTION_ERROR 35 +#define LOAD_BUILD_CLASS 36 +#define LOAD_LOCALS 37 +#define MAKE_FUNCTION 38 +#define MATCH_KEYS 39 +#define MATCH_MAPPING 40 +#define MATCH_SEQUENCE 41 +#define NOP 42 +#define POP_EXCEPT 43 +#define POP_TOP 44 +#define PUSH_EXC_INFO 45 +#define PUSH_NULL 46 +#define RETURN_GENERATOR 47 +#define RETURN_VALUE 48 +#define SETUP_ANNOTATIONS 49 +#define STORE_ATTR_INSTANCE_VALUE 50 +#define STORE_ATTR_SLOT 51 +#define STORE_SLICE 52 +#define STORE_SUBSCR 53 +#define STORE_SUBSCR_DICT 54 +#define STORE_SUBSCR_LIST_INT 55 +#define TO_BOOL 56 +#define TO_BOOL_ALWAYS_TRUE 57 +#define TO_BOOL_BOOL 58 +#define TO_BOOL_INT 59 +#define TO_BOOL_LIST 60 +#define TO_BOOL_NONE 61 +#define TO_BOOL_STR 62 +#define UNARY_INVERT 63 +#define UNARY_NEGATIVE 64 +#define UNARY_NOT 65 +#define WITH_EXCEPT_START 66 +#define HAVE_ARGUMENT 67 +#define BINARY_OP 67 +#define BUILD_CONST_KEY_MAP 68 +#define BUILD_LIST 69 +#define BUILD_MAP 70 +#define BUILD_SET 71 +#define BUILD_SLICE 72 +#define BUILD_STRING 73 +#define BUILD_TUPLE 74 +#define CALL 75 +#define CALL_BOUND_METHOD_EXACT_ARGS 76 +#define CALL_BUILTIN_CLASS 77 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 78 +#define CALL_FUNCTION_EX 79 +#define CALL_INTRINSIC_1 80 +#define CALL_INTRINSIC_2 81 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 82 +#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 83 +#define CALL_NO_KW_BUILTIN_FAST 84 +#define CALL_NO_KW_BUILTIN_O 85 +#define CALL_NO_KW_ISINSTANCE 86 +#define CALL_NO_KW_LEN 87 +#define CALL_NO_KW_LIST_APPEND 88 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 89 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 90 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 91 +#define CALL_NO_KW_STR_1 92 +#define CALL_NO_KW_TUPLE_1 93 +#define CALL_NO_KW_TYPE_1 94 +#define CALL_PY_EXACT_ARGS 95 +#define CALL_PY_WITH_DEFAULTS 96 +#define COMPARE_OP 97 +#define COMPARE_OP_FLOAT 98 +#define COMPARE_OP_INT 99 +#define COMPARE_OP_STR 100 +#define CONTAINS_OP 101 +#define CONVERT_VALUE 102 +#define COPY 103 +#define COPY_FREE_VARS 104 +#define DELETE_ATTR 105 +#define DELETE_DEREF 106 +#define DELETE_FAST 107 +#define DELETE_GLOBAL 108 +#define DELETE_NAME 109 +#define DICT_MERGE 110 +#define DICT_UPDATE 111 +#define ENTER_EXECUTOR 112 +#define EXTENDED_ARG 113 +#define FOR_ITER 114 +#define FOR_ITER_GEN 115 +#define FOR_ITER_LIST 116 +#define FOR_ITER_RANGE 117 +#define FOR_ITER_TUPLE 118 +#define GET_AWAITABLE 119 +#define IMPORT_FROM 120 +#define IMPORT_NAME 121 +#define IS_OP 122 +#define JUMP_BACKWARD 123 +#define JUMP_BACKWARD_NO_INTERRUPT 124 +#define JUMP_FORWARD 125 +#define KW_NAMES 126 +#define LIST_APPEND 127 +#define LIST_EXTEND 128 +#define LOAD_ATTR 129 +#define LOAD_ATTR_CLASS 130 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 131 +#define LOAD_ATTR_INSTANCE_VALUE 132 +#define LOAD_ATTR_METHOD_LAZY_DICT 133 +#define LOAD_ATTR_METHOD_NO_DICT 134 +#define LOAD_ATTR_METHOD_WITH_VALUES 135 +#define LOAD_ATTR_MODULE 136 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 137 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 138 +#define LOAD_ATTR_PROPERTY 139 +#define LOAD_ATTR_SLOT 140 +#define LOAD_ATTR_WITH_HINT 141 +#define LOAD_CONST 142 +#define LOAD_DEREF 143 +#define LOAD_FAST 144 +#define LOAD_FAST_AND_CLEAR 145 +#define LOAD_FAST_CHECK 146 +#define LOAD_FAST_LOAD_FAST 147 +#define LOAD_FROM_DICT_OR_DEREF 148 +#define LOAD_FROM_DICT_OR_GLOBALS 149 +#define LOAD_GLOBAL 150 +#define LOAD_GLOBAL_BUILTIN 151 +#define LOAD_GLOBAL_MODULE 152 +#define LOAD_NAME 153 +#define LOAD_SUPER_ATTR 154 +#define LOAD_SUPER_ATTR_ATTR 155 +#define LOAD_SUPER_ATTR_METHOD 156 +#define MAKE_CELL 157 +#define MAP_ADD 158 +#define MATCH_CLASS 159 +#define POP_JUMP_IF_FALSE 160 +#define POP_JUMP_IF_NONE 161 +#define POP_JUMP_IF_NOT_NONE 162 +#define POP_JUMP_IF_TRUE 163 +#define RAISE_VARARGS 164 +#define RERAISE 165 +#define RESUME 166 +#define RETURN_CONST 167 +#define SEND 168 +#define SEND_GEN 169 +#define SET_ADD 170 +#define SET_FUNCTION_ATTRIBUTE 171 +#define SET_UPDATE 172 +#define STORE_ATTR 173 +#define STORE_ATTR_WITH_HINT 174 +#define STORE_DEREF 175 +#define STORE_FAST 176 +#define STORE_FAST_LOAD_FAST 177 +#define STORE_FAST_STORE_FAST 178 +#define STORE_GLOBAL 179 +#define STORE_NAME 180 +#define SWAP 181 +#define UNPACK_EX 182 +#define UNPACK_SEQUENCE 183 +#define UNPACK_SEQUENCE_LIST 184 +#define UNPACK_SEQUENCE_TUPLE 185 +#define UNPACK_SEQUENCE_TWO_TUPLE 186 +#define YIELD_VALUE 187 #define MIN_INSTRUMENTED_OPCODE 237 -#define INSTRUMENTED_LOAD_SUPER_ATTR 237 -#define INSTRUMENTED_POP_JUMP_IF_NONE 238 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 239 -#define INSTRUMENTED_RESUME 240 -#define INSTRUMENTED_CALL 241 -#define INSTRUMENTED_RETURN_VALUE 242 -#define INSTRUMENTED_YIELD_VALUE 243 -#define INSTRUMENTED_CALL_FUNCTION_EX 244 -#define INSTRUMENTED_JUMP_FORWARD 245 -#define INSTRUMENTED_JUMP_BACKWARD 246 -#define INSTRUMENTED_RETURN_CONST 247 -#define INSTRUMENTED_FOR_ITER 248 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 249 +#define INSTRUMENTED_RESUME 237 +#define INSTRUMENTED_END_FOR 238 +#define INSTRUMENTED_END_SEND 239 +#define INSTRUMENTED_RETURN_VALUE 240 +#define INSTRUMENTED_RETURN_CONST 241 +#define INSTRUMENTED_YIELD_VALUE 242 +#define INSTRUMENTED_LOAD_SUPER_ATTR 243 +#define INSTRUMENTED_FOR_ITER 244 +#define INSTRUMENTED_CALL 245 +#define INSTRUMENTED_CALL_FUNCTION_EX 246 +#define INSTRUMENTED_INSTRUCTION 247 +#define INSTRUMENTED_JUMP_FORWARD 248 +#define INSTRUMENTED_JUMP_BACKWARD 249 #define INSTRUMENTED_POP_JUMP_IF_TRUE 250 -#define INSTRUMENTED_END_FOR 251 -#define INSTRUMENTED_END_SEND 252 -#define INSTRUMENTED_INSTRUCTION 253 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 251 +#define INSTRUMENTED_POP_JUMP_IF_NONE 252 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 253 #define INSTRUMENTED_LINE 254 -#define SETUP_FINALLY 256 -#define SETUP_CLEANUP 257 -#define SETUP_WITH 258 -#define POP_BLOCK 259 -#define JUMP 260 -#define JUMP_NO_INTERRUPT 261 -#define LOAD_METHOD 262 -#define LOAD_SUPER_METHOD 263 -#define LOAD_ZERO_SUPER_METHOD 264 -#define LOAD_ZERO_SUPER_ATTR 265 -#define STORE_FAST_MAYBE_NULL 266 -#define LOAD_CLOSURE 267 -#define TO_BOOL_ALWAYS_TRUE 7 -#define TO_BOOL_BOOL 8 -#define TO_BOOL_INT 10 -#define TO_BOOL_LIST 13 -#define TO_BOOL_NONE 14 -#define TO_BOOL_STR 18 -#define BINARY_OP_MULTIPLY_INT 19 -#define BINARY_OP_ADD_INT 20 -#define BINARY_OP_SUBTRACT_INT 21 -#define BINARY_OP_MULTIPLY_FLOAT 22 -#define BINARY_OP_ADD_FLOAT 23 -#define BINARY_OP_SUBTRACT_FLOAT 28 -#define BINARY_OP_ADD_UNICODE 29 -#define BINARY_OP_INPLACE_ADD_UNICODE 34 -#define BINARY_SUBSCR_DICT 38 -#define BINARY_SUBSCR_GETITEM 39 -#define BINARY_SUBSCR_LIST_INT 42 -#define BINARY_SUBSCR_STR_INT 43 -#define BINARY_SUBSCR_TUPLE_INT 44 -#define STORE_SUBSCR_DICT 45 -#define STORE_SUBSCR_LIST_INT 46 -#define SEND_GEN 47 -#define UNPACK_SEQUENCE_TWO_TUPLE 48 -#define UNPACK_SEQUENCE_TUPLE 56 -#define UNPACK_SEQUENCE_LIST 57 -#define STORE_ATTR_INSTANCE_VALUE 58 -#define STORE_ATTR_SLOT 59 -#define STORE_ATTR_WITH_HINT 62 -#define LOAD_GLOBAL_MODULE 63 -#define LOAD_GLOBAL_BUILTIN 64 -#define LOAD_SUPER_ATTR_ATTR 65 -#define LOAD_SUPER_ATTR_METHOD 66 -#define LOAD_ATTR_INSTANCE_VALUE 67 -#define LOAD_ATTR_MODULE 70 -#define LOAD_ATTR_WITH_HINT 72 -#define LOAD_ATTR_SLOT 73 -#define LOAD_ATTR_CLASS 76 -#define LOAD_ATTR_PROPERTY 77 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 78 -#define LOAD_ATTR_METHOD_WITH_VALUES 79 -#define LOAD_ATTR_METHOD_NO_DICT 80 -#define LOAD_ATTR_METHOD_LAZY_DICT 81 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 82 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 84 -#define COMPARE_OP_FLOAT 86 -#define COMPARE_OP_INT 88 -#define COMPARE_OP_STR 111 -#define FOR_ITER_LIST 112 -#define FOR_ITER_TUPLE 113 -#define FOR_ITER_RANGE 132 -#define FOR_ITER_GEN 136 -#define CALL_BOUND_METHOD_EXACT_ARGS 148 -#define CALL_PY_EXACT_ARGS 153 -#define CALL_PY_WITH_DEFAULTS 154 -#define CALL_NO_KW_TYPE_1 155 -#define CALL_NO_KW_STR_1 159 -#define CALL_NO_KW_TUPLE_1 160 -#define CALL_BUILTIN_CLASS 161 -#define CALL_NO_KW_BUILTIN_O 166 -#define CALL_NO_KW_BUILTIN_FAST 167 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 178 -#define CALL_NO_KW_LEN 179 -#define CALL_NO_KW_ISINSTANCE 180 -#define CALL_NO_KW_LIST_APPEND 181 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 182 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 183 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 184 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 185 -#define CALL_NO_KW_ALLOC_AND_ENTER_INIT 186 - +#define JUMP 256 +#define JUMP_NO_INTERRUPT 257 +#define LOAD_CLOSURE 258 +#define LOAD_METHOD 259 +#define LOAD_SUPER_METHOD 260 +#define LOAD_ZERO_SUPER_ATTR 261 +#define LOAD_ZERO_SUPER_METHOD 262 +#define POP_BLOCK 263 +#define SETUP_CLEANUP 264 +#define SETUP_FINALLY 265 +#define SETUP_WITH 266 +#define STORE_FAST_MAYBE_NULL 267 #ifdef __cplusplus } diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 17101d1d94757..b02aa771c347e 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -103,4 +103,228 @@ # An irregular case: _specializations["BINARY_OP"].append("BINARY_OP_INPLACE_ADD_UNICODE") -_specialized_instructions = [opcode for family in _specializations.values() for opcode in family] +_specialized_opmap = { + 'BINARY_OP_ADD_FLOAT': 3, + 'BINARY_OP_ADD_INT': 4, + 'BINARY_OP_ADD_UNICODE': 5, + 'BINARY_OP_INPLACE_ADD_UNICODE': 6, + 'BINARY_OP_MULTIPLY_FLOAT': 7, + 'BINARY_OP_MULTIPLY_INT': 8, + 'BINARY_OP_SUBTRACT_FLOAT': 9, + 'BINARY_OP_SUBTRACT_INT': 10, + 'BINARY_SUBSCR_DICT': 13, + 'BINARY_SUBSCR_GETITEM': 14, + 'BINARY_SUBSCR_LIST_INT': 15, + 'BINARY_SUBSCR_STR_INT': 16, + 'BINARY_SUBSCR_TUPLE_INT': 18, + 'STORE_ATTR_INSTANCE_VALUE': 50, + 'STORE_ATTR_SLOT': 51, + 'STORE_SUBSCR_DICT': 54, + 'STORE_SUBSCR_LIST_INT': 55, + 'TO_BOOL_ALWAYS_TRUE': 57, + 'TO_BOOL_BOOL': 58, + 'TO_BOOL_INT': 59, + 'TO_BOOL_LIST': 60, + 'TO_BOOL_NONE': 61, + 'TO_BOOL_STR': 62, + 'CALL_BOUND_METHOD_EXACT_ARGS': 76, + 'CALL_BUILTIN_CLASS': 77, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 78, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 82, + 'CALL_NO_KW_ALLOC_AND_ENTER_INIT': 83, + 'CALL_NO_KW_BUILTIN_FAST': 84, + 'CALL_NO_KW_BUILTIN_O': 85, + 'CALL_NO_KW_ISINSTANCE': 86, + 'CALL_NO_KW_LEN': 87, + 'CALL_NO_KW_LIST_APPEND': 88, + 'CALL_NO_KW_METHOD_DESCRIPTOR_FAST': 89, + 'CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS': 90, + 'CALL_NO_KW_METHOD_DESCRIPTOR_O': 91, + 'CALL_NO_KW_STR_1': 92, + 'CALL_NO_KW_TUPLE_1': 93, + 'CALL_NO_KW_TYPE_1': 94, + 'CALL_PY_EXACT_ARGS': 95, + 'CALL_PY_WITH_DEFAULTS': 96, + 'COMPARE_OP_FLOAT': 98, + 'COMPARE_OP_INT': 99, + 'COMPARE_OP_STR': 100, + 'FOR_ITER_GEN': 115, + 'FOR_ITER_LIST': 116, + 'FOR_ITER_RANGE': 117, + 'FOR_ITER_TUPLE': 118, + 'LOAD_ATTR_CLASS': 130, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 131, + 'LOAD_ATTR_INSTANCE_VALUE': 132, + 'LOAD_ATTR_METHOD_LAZY_DICT': 133, + 'LOAD_ATTR_METHOD_NO_DICT': 134, + 'LOAD_ATTR_METHOD_WITH_VALUES': 135, + 'LOAD_ATTR_MODULE': 136, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 137, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 138, + 'LOAD_ATTR_PROPERTY': 139, + 'LOAD_ATTR_SLOT': 140, + 'LOAD_ATTR_WITH_HINT': 141, + 'LOAD_GLOBAL_BUILTIN': 151, + 'LOAD_GLOBAL_MODULE': 152, + 'LOAD_SUPER_ATTR_ATTR': 155, + 'LOAD_SUPER_ATTR_METHOD': 156, + 'SEND_GEN': 169, + 'STORE_ATTR_WITH_HINT': 174, + 'UNPACK_SEQUENCE_LIST': 184, + 'UNPACK_SEQUENCE_TUPLE': 185, + 'UNPACK_SEQUENCE_TWO_TUPLE': 186, +} + +opmap = { + 'CACHE': 0, + 'BEFORE_ASYNC_WITH': 1, + 'BEFORE_WITH': 2, + 'BINARY_SLICE': 11, + 'BINARY_SUBSCR': 12, + 'RESERVED': 17, + 'CHECK_EG_MATCH': 19, + 'CHECK_EXC_MATCH': 20, + 'CLEANUP_THROW': 21, + 'DELETE_SUBSCR': 22, + 'END_ASYNC_FOR': 23, + 'END_FOR': 24, + 'END_SEND': 25, + 'EXIT_INIT_CHECK': 26, + 'FORMAT_SIMPLE': 27, + 'FORMAT_WITH_SPEC': 28, + 'GET_AITER': 29, + 'GET_ANEXT': 30, + 'GET_ITER': 31, + 'GET_LEN': 32, + 'GET_YIELD_FROM_ITER': 33, + 'INTERPRETER_EXIT': 34, + 'LOAD_ASSERTION_ERROR': 35, + 'LOAD_BUILD_CLASS': 36, + 'LOAD_LOCALS': 37, + 'MAKE_FUNCTION': 38, + 'MATCH_KEYS': 39, + 'MATCH_MAPPING': 40, + 'MATCH_SEQUENCE': 41, + 'NOP': 42, + 'POP_EXCEPT': 43, + 'POP_TOP': 44, + 'PUSH_EXC_INFO': 45, + 'PUSH_NULL': 46, + 'RETURN_GENERATOR': 47, + 'RETURN_VALUE': 48, + 'SETUP_ANNOTATIONS': 49, + 'STORE_SLICE': 52, + 'STORE_SUBSCR': 53, + 'TO_BOOL': 56, + 'UNARY_INVERT': 63, + 'UNARY_NEGATIVE': 64, + 'UNARY_NOT': 65, + 'WITH_EXCEPT_START': 66, + 'BINARY_OP': 67, + 'BUILD_CONST_KEY_MAP': 68, + 'BUILD_LIST': 69, + 'BUILD_MAP': 70, + 'BUILD_SET': 71, + 'BUILD_SLICE': 72, + 'BUILD_STRING': 73, + 'BUILD_TUPLE': 74, + 'CALL': 75, + 'CALL_FUNCTION_EX': 79, + 'CALL_INTRINSIC_1': 80, + 'CALL_INTRINSIC_2': 81, + 'COMPARE_OP': 97, + 'CONTAINS_OP': 101, + 'CONVERT_VALUE': 102, + 'COPY': 103, + 'COPY_FREE_VARS': 104, + 'DELETE_ATTR': 105, + 'DELETE_DEREF': 106, + 'DELETE_FAST': 107, + 'DELETE_GLOBAL': 108, + 'DELETE_NAME': 109, + 'DICT_MERGE': 110, + 'DICT_UPDATE': 111, + 'ENTER_EXECUTOR': 112, + 'EXTENDED_ARG': 113, + 'FOR_ITER': 114, + 'GET_AWAITABLE': 119, + 'IMPORT_FROM': 120, + 'IMPORT_NAME': 121, + 'IS_OP': 122, + 'JUMP_BACKWARD': 123, + 'JUMP_BACKWARD_NO_INTERRUPT': 124, + 'JUMP_FORWARD': 125, + 'KW_NAMES': 126, + 'LIST_APPEND': 127, + 'LIST_EXTEND': 128, + 'LOAD_ATTR': 129, + 'LOAD_CONST': 142, + 'LOAD_DEREF': 143, + 'LOAD_FAST': 144, + 'LOAD_FAST_AND_CLEAR': 145, + 'LOAD_FAST_CHECK': 146, + 'LOAD_FAST_LOAD_FAST': 147, + 'LOAD_FROM_DICT_OR_DEREF': 148, + 'LOAD_FROM_DICT_OR_GLOBALS': 149, + 'LOAD_GLOBAL': 150, + 'LOAD_NAME': 153, + 'LOAD_SUPER_ATTR': 154, + 'MAKE_CELL': 157, + 'MAP_ADD': 158, + 'MATCH_CLASS': 159, + 'POP_JUMP_IF_FALSE': 160, + 'POP_JUMP_IF_NONE': 161, + 'POP_JUMP_IF_NOT_NONE': 162, + 'POP_JUMP_IF_TRUE': 163, + 'RAISE_VARARGS': 164, + 'RERAISE': 165, + 'RESUME': 166, + 'RETURN_CONST': 167, + 'SEND': 168, + 'SET_ADD': 170, + 'SET_FUNCTION_ATTRIBUTE': 171, + 'SET_UPDATE': 172, + 'STORE_ATTR': 173, + 'STORE_DEREF': 175, + 'STORE_FAST': 176, + 'STORE_FAST_LOAD_FAST': 177, + 'STORE_FAST_STORE_FAST': 178, + 'STORE_GLOBAL': 179, + 'STORE_NAME': 180, + 'SWAP': 181, + 'UNPACK_EX': 182, + 'UNPACK_SEQUENCE': 183, + 'YIELD_VALUE': 187, + 'INSTRUMENTED_RESUME': 237, + 'INSTRUMENTED_END_FOR': 238, + 'INSTRUMENTED_END_SEND': 239, + 'INSTRUMENTED_RETURN_VALUE': 240, + 'INSTRUMENTED_RETURN_CONST': 241, + 'INSTRUMENTED_YIELD_VALUE': 242, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 243, + 'INSTRUMENTED_FOR_ITER': 244, + 'INSTRUMENTED_CALL': 245, + 'INSTRUMENTED_CALL_FUNCTION_EX': 246, + 'INSTRUMENTED_INSTRUCTION': 247, + 'INSTRUMENTED_JUMP_FORWARD': 248, + 'INSTRUMENTED_JUMP_BACKWARD': 249, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 250, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 251, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 252, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 253, + 'INSTRUMENTED_LINE': 254, + 'JUMP': 256, + 'JUMP_NO_INTERRUPT': 257, + 'LOAD_CLOSURE': 258, + 'LOAD_METHOD': 259, + 'LOAD_SUPER_METHOD': 260, + 'LOAD_ZERO_SUPER_ATTR': 261, + 'LOAD_ZERO_SUPER_METHOD': 262, + 'POP_BLOCK': 263, + 'SETUP_CLEANUP': 264, + 'SETUP_FINALLY': 265, + 'SETUP_WITH': 266, + 'STORE_FAST_MAYBE_NULL': 267, +} +MIN_INSTRUMENTED_OPCODE = 237 +HAVE_ARGUMENT = 67 diff --git a/Lib/dis.py b/Lib/dis.py index bf1a1e2ff7ac1..4f4e77f32503d 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -14,7 +14,7 @@ _intrinsic_1_descs, _intrinsic_2_descs, _specializations, - _specialized_instructions, + _specialized_opmap, ) __all__ = ["code_info", "dis", "disassemble", "distb", "disco", @@ -49,11 +49,11 @@ _all_opname = list(opname) _all_opmap = dict(opmap) -_empty_slot = [slot for slot, name in enumerate(_all_opname) if name.startswith("<")] -for spec_op, specialized in zip(_empty_slot, _specialized_instructions): +for name, op in _specialized_opmap.items(): # fill opname and opmap - _all_opname[spec_op] = specialized - _all_opmap[specialized] = spec_op + assert op < len(_all_opname) + _all_opname[op] = name + _all_opmap[name] = op deoptmap = { specialized: base for base, family in _specializations.items() for specialized in family diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 5f0d659b1ed53..0717e202ecbcd 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -454,6 +454,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.13a1 3556 (Convert LOAD_CLOSURE to a pseudo-op) # Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit) # Python 3.13a1 3558 (Reorder the stack items for CALL) +# Python 3.13a1 3559 (Generate opcode IDs from bytecodes.c) # Python 3.14 will start with 3600 @@ -470,7 +471,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3558).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3559).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 5a9f8ddd0738d..6b9d9ce811a61 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -6,8 +6,7 @@ # Note that __all__ is further extended below -__all__ = ["cmp_op", "opname", "opmap", "stack_effect", "hascompare", - "HAVE_ARGUMENT", "EXTENDED_ARG"] +__all__ = ["cmp_op", "stack_effect", "hascompare"] import _opcode from _opcode import stack_effect @@ -15,214 +14,17 @@ import sys # The build uses older versions of Python which do not have _opcode_metadata if sys.version_info[:2] >= (3, 13): - from _opcode_metadata import _specializations, _specialized_instructions + from _opcode_metadata import _specializations, _specialized_opmap + from _opcode_metadata import opmap, HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE + EXTENDED_ARG = opmap['EXTENDED_ARG'] -cmp_op = ('<', '<=', '==', '!=', '>', '>=') - -opmap = {} - -def def_op(name, op): - opmap[name] = op - -# Instruction opcodes for compiled code -# Blank lines correspond to available opcodes - -def_op('CACHE', 0) -def_op('POP_TOP', 1) -def_op('PUSH_NULL', 2) -def_op('INTERPRETER_EXIT', 3) -def_op('END_FOR', 4) -def_op('END_SEND', 5) -def_op('TO_BOOL', 6) - -def_op('NOP', 9) - -def_op('UNARY_NEGATIVE', 11) -def_op('UNARY_NOT', 12) - -def_op('UNARY_INVERT', 15) -def_op('EXIT_INIT_CHECK', 16) - -# We reserve 17 as it is the initial value for the specializing counter -# This helps us catch cases where we attempt to execute a cache. -def_op('RESERVED', 17) - -def_op('MAKE_FUNCTION', 24) -def_op('BINARY_SUBSCR', 25) -def_op('BINARY_SLICE', 26) -def_op('STORE_SLICE', 27) - -def_op('GET_LEN', 30) -def_op('MATCH_MAPPING', 31) -def_op('MATCH_SEQUENCE', 32) -def_op('MATCH_KEYS', 33) - -def_op('PUSH_EXC_INFO', 35) -def_op('CHECK_EXC_MATCH', 36) -def_op('CHECK_EG_MATCH', 37) - -def_op('FORMAT_SIMPLE', 40) -def_op('FORMAT_WITH_SPEC', 41) - -def_op('WITH_EXCEPT_START', 49) -def_op('GET_AITER', 50) -def_op('GET_ANEXT', 51) -def_op('BEFORE_ASYNC_WITH', 52) -def_op('BEFORE_WITH', 53) -def_op('END_ASYNC_FOR', 54) -def_op('CLEANUP_THROW', 55) - -def_op('STORE_SUBSCR', 60) -def_op('DELETE_SUBSCR', 61) - -def_op('GET_ITER', 68) -def_op('GET_YIELD_FROM_ITER', 69) - -def_op('LOAD_BUILD_CLASS', 71) - -def_op('LOAD_ASSERTION_ERROR', 74) -def_op('RETURN_GENERATOR', 75) - -def_op('RETURN_VALUE', 83) + opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] + for op, i in opmap.items(): + opname[i] = op -def_op('SETUP_ANNOTATIONS', 85) + __all__.extend(["opname", "opmap", "HAVE_ARGUMENT", "EXTENDED_ARG"]) -def_op('LOAD_LOCALS', 87) - -def_op('POP_EXCEPT', 89) - -HAVE_ARGUMENT = 90 # real opcodes from here have an argument: - -def_op('STORE_NAME', 90) # Index in name list -def_op('DELETE_NAME', 91) # "" -def_op('UNPACK_SEQUENCE', 92) # Number of tuple items -def_op('FOR_ITER', 93) -def_op('UNPACK_EX', 94) -def_op('STORE_ATTR', 95) # Index in name list -def_op('DELETE_ATTR', 96) # "" -def_op('STORE_GLOBAL', 97) # "" -def_op('DELETE_GLOBAL', 98) # "" -def_op('SWAP', 99) -def_op('LOAD_CONST', 100) # Index in const list -def_op('LOAD_NAME', 101) # Index in name list -def_op('BUILD_TUPLE', 102) # Number of tuple items -def_op('BUILD_LIST', 103) # Number of list items -def_op('BUILD_SET', 104) # Number of set items -def_op('BUILD_MAP', 105) # Number of dict entries -def_op('LOAD_ATTR', 106) # Index in name list -def_op('COMPARE_OP', 107) # Comparison operator -def_op('IMPORT_NAME', 108) # Index in name list -def_op('IMPORT_FROM', 109) # Index in name list -def_op('JUMP_FORWARD', 110) # Number of words to skip - -def_op('POP_JUMP_IF_FALSE', 114) -def_op('POP_JUMP_IF_TRUE', 115) -def_op('LOAD_GLOBAL', 116) # Index in name list -def_op('IS_OP', 117) -def_op('CONTAINS_OP', 118) -def_op('RERAISE', 119) -def_op('COPY', 120) -def_op('RETURN_CONST', 121) -def_op('BINARY_OP', 122) -def_op('SEND', 123) # Number of words to skip -def_op('LOAD_FAST', 124) # Local variable number, no null check -def_op('STORE_FAST', 125) # Local variable number -def_op('DELETE_FAST', 126) # Local variable number -def_op('LOAD_FAST_CHECK', 127) # Local variable number -def_op('POP_JUMP_IF_NOT_NONE', 128) -def_op('POP_JUMP_IF_NONE', 129) -def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) -def_op('GET_AWAITABLE', 131) -def_op('BUILD_SLICE', 133) # Number of items -def_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards) -def_op('MAKE_CELL', 135) -def_op('LOAD_DEREF', 137) -def_op('STORE_DEREF', 138) -def_op('DELETE_DEREF', 139) -def_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) -def_op('LOAD_SUPER_ATTR', 141) -def_op('CALL_FUNCTION_EX', 142) # Flags -def_op('LOAD_FAST_AND_CLEAR', 143) # Local variable number -def_op('EXTENDED_ARG', 144) -EXTENDED_ARG = opmap['EXTENDED_ARG'] -def_op('LIST_APPEND', 145) -def_op('SET_ADD', 146) -def_op('MAP_ADD', 147) -def_op('COPY_FREE_VARS', 149) -def_op('YIELD_VALUE', 150) -def_op('RESUME', 151) # This must be kept in sync with deepfreeze.py -def_op('MATCH_CLASS', 152) - -def_op('BUILD_CONST_KEY_MAP', 156) -def_op('BUILD_STRING', 157) -def_op('CONVERT_VALUE', 158) - -def_op('LIST_EXTEND', 162) -def_op('SET_UPDATE', 163) -def_op('DICT_MERGE', 164) -def_op('DICT_UPDATE', 165) - -def_op('LOAD_FAST_LOAD_FAST', 168) -def_op('STORE_FAST_LOAD_FAST', 169) -def_op('STORE_FAST_STORE_FAST', 170) -def_op('CALL', 171) -def_op('KW_NAMES', 172) -def_op('CALL_INTRINSIC_1', 173) -def_op('CALL_INTRINSIC_2', 174) -def_op('LOAD_FROM_DICT_OR_GLOBALS', 175) -def_op('LOAD_FROM_DICT_OR_DEREF', 176) -def_op('SET_FUNCTION_ATTRIBUTE', 177) # Attribute - -# Optimizer hook -def_op('ENTER_EXECUTOR', 230) - -# Instrumented instructions -MIN_INSTRUMENTED_OPCODE = 237 - -def_op('INSTRUMENTED_LOAD_SUPER_ATTR', 237) -def_op('INSTRUMENTED_POP_JUMP_IF_NONE', 238) -def_op('INSTRUMENTED_POP_JUMP_IF_NOT_NONE', 239) -def_op('INSTRUMENTED_RESUME', 240) -def_op('INSTRUMENTED_CALL', 241) -def_op('INSTRUMENTED_RETURN_VALUE', 242) -def_op('INSTRUMENTED_YIELD_VALUE', 243) -def_op('INSTRUMENTED_CALL_FUNCTION_EX', 244) -def_op('INSTRUMENTED_JUMP_FORWARD', 245) -def_op('INSTRUMENTED_JUMP_BACKWARD', 246) -def_op('INSTRUMENTED_RETURN_CONST', 247) -def_op('INSTRUMENTED_FOR_ITER', 248) -def_op('INSTRUMENTED_POP_JUMP_IF_FALSE', 249) -def_op('INSTRUMENTED_POP_JUMP_IF_TRUE', 250) -def_op('INSTRUMENTED_END_FOR', 251) -def_op('INSTRUMENTED_END_SEND', 252) -def_op('INSTRUMENTED_INSTRUCTION', 253) -def_op('INSTRUMENTED_LINE', 254) -# 255 is reserved - - -# Pseudo ops are above 255: - -def_op('SETUP_FINALLY', 256) -def_op('SETUP_CLEANUP', 257) -def_op('SETUP_WITH', 258) -def_op('POP_BLOCK', 259) - -def_op('JUMP', 260) -def_op('JUMP_NO_INTERRUPT', 261) - -def_op('LOAD_METHOD', 262) -def_op('LOAD_SUPER_METHOD', 263) -def_op('LOAD_ZERO_SUPER_METHOD', 264) -def_op('LOAD_ZERO_SUPER_ATTR', 265) - -def_op('STORE_FAST_MAYBE_NULL', 266) -def_op('LOAD_CLOSURE', 267) - -del def_op - -opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] -for op, i in opmap.items(): - opname[i] = op +cmp_op = ('<', '<=', '==', '!=', '>', '>=') # The build uses older versions of Python which do not have _opcode.has_* functions if sys.version_info[:2] >= (3, 13): @@ -243,7 +45,7 @@ def def_op(name, op): _intrinsic_1_descs = _opcode.get_intrinsic1_descs() _intrinsic_2_descs = _opcode.get_intrinsic2_descs() -hascompare = [opmap["COMPARE_OP"]] + hascompare = [opmap["COMPARE_OP"]] _nb_ops = [ ("NB_ADD", "+"), diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index f49c60a01a54e..aa967178c1dc3 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -196,11 +196,11 @@ def bug42562(): # Extended arg followed by NOP code_bug_45757 = bytes([ - 0x90, 0x01, # EXTENDED_ARG 0x01 - 0x09, 0xFF, # NOP 0xFF - 0x90, 0x01, # EXTENDED_ARG 0x01 - 0x64, 0x29, # LOAD_CONST 0x29 - 0x53, 0x00, # RETURN_VALUE 0x00 + opcode.opmap['EXTENDED_ARG'], 0x01, # EXTENDED_ARG 0x01 + opcode.opmap['NOP'], 0xFF, # NOP 0xFF + opcode.opmap['EXTENDED_ARG'], 0x01, # EXTENDED_ARG 0x01 + opcode.opmap['LOAD_CONST'], 0x29, # LOAD_CONST 0x29 + opcode.opmap['RETURN_VALUE'], 0x00, # RETURN_VALUE 0x00 ]) dis_bug_45757 = """\ @@ -931,7 +931,7 @@ def do_disassembly_test(self, func, expected, with_offsets=False): with_offsets) def test_opmap(self): - self.assertEqual(dis.opmap["NOP"], 9) + self.assertEqual(dis.opmap["CACHE"], 0) self.assertIn(dis.opmap["LOAD_CONST"], dis.hasconst) self.assertIn(dis.opmap["STORE_NAME"], dis.hasname) @@ -940,7 +940,6 @@ def test_opname(self): def test_boundaries(self): self.assertEqual(dis.opmap["EXTENDED_ARG"], dis.EXTENDED_ARG) - self.assertEqual(dis.opmap["STORE_NAME"], dis.HAVE_ARGUMENT) def test_widths(self): long_opcodes = set(['JUMP_BACKWARD_NO_INTERRUPT', @@ -1617,197 +1616,197 @@ def _prepare_test_cases(): Instruction = dis.Instruction expected_opinfo_outer = [ - Instruction(opname='MAKE_CELL', opcode=135, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_CELL', opcode=135, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=1, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=102, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_FUNCTION', opcode=24, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=7, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_LIST', opcode=103, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=8, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=157, arg=0, argval='a', argrepr='a', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=157, arg=1, argval='b', argrepr='b', offset=2, start_offset=2, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=4, start_offset=4, starts_line=1, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=5, argval=(3, 4), argrepr='(3, 4)', offset=6, start_offset=6, starts_line=2, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='a', argrepr='a', offset=8, start_offset=8, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=1, argval='b', argrepr='b', offset=10, start_offset=10, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BUILD_TUPLE', opcode=74, arg=2, argval=2, argrepr='', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=1, argval=code_object_f, argrepr=repr(code_object_f), offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_FUNCTION', opcode=38, arg=None, argval=None, argrepr='', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=8, argval=8, argrepr='closure', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=1, argval=1, argrepr='defaults', offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=176, arg=2, argval='f', argrepr='f', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='print', argrepr='print + NULL', offset=24, start_offset=24, starts_line=7, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=0, argval='a', argrepr='a', offset=34, start_offset=34, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=1, argval='b', argrepr='b', offset=36, start_offset=36, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval='', argrepr="''", offset=38, start_offset=38, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=1, argrepr='1', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BUILD_LIST', opcode=69, arg=0, argval=0, argrepr='', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BUILD_MAP', opcode=70, arg=0, argval=0, argrepr='', offset=44, start_offset=44, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=46, start_offset=46, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=7, argval=7, argrepr='', offset=48, start_offset=48, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=2, argval='f', argrepr='f', offset=58, start_offset=58, starts_line=8, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=48, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_f = [ - Instruction(opname='COPY_FREE_VARS', opcode=149, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_CELL', opcode=135, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_CELL', opcode=135, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=2, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BUILD_TUPLE', opcode=102, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='MAKE_FUNCTION', opcode=24, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=177, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=6, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=104, arg=2, argval=2, argrepr='', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=157, arg=0, argval='c', argrepr='c', offset=2, start_offset=2, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_CELL', opcode=157, arg=1, argval='d', argrepr='d', offset=4, start_offset=4, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=6, start_offset=6, starts_line=2, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval=(5, 6), argrepr='(5, 6)', offset=8, start_offset=8, starts_line=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=3, argval='a', argrepr='a', offset=10, start_offset=10, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=4, argval='b', argrepr='b', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='c', argrepr='c', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=1, argval='d', argrepr='d', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BUILD_TUPLE', opcode=74, arg=4, argval=4, argrepr='', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=1, argval=code_object_inner, argrepr=repr(code_object_inner), offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='MAKE_FUNCTION', opcode=38, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=8, argval=8, argrepr='closure', offset=24, start_offset=24, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='SET_FUNCTION_ATTRIBUTE', opcode=171, arg=1, argval=1, argrepr='defaults', offset=26, start_offset=26, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=176, arg=2, argval='inner', argrepr='inner', offset=28, start_offset=28, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=3, argval='a', argrepr='a', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=4, argval='b', argrepr='b', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=0, argval='c', argrepr='c', offset=44, start_offset=44, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=1, argval='d', argrepr='d', offset=46, start_offset=46, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=4, argval=4, argrepr='', offset=48, start_offset=48, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=56, start_offset=56, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=2, argval='inner', argrepr='inner', offset=58, start_offset=58, starts_line=6, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=48, arg=None, argval=None, argrepr='', offset=60, start_offset=60, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_inner = [ - Instruction(opname='COPY_FREE_VARS', opcode=149, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_DEREF', opcode=137, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=168, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY_FREE_VARS', opcode=104, arg=4, argval=4, argrepr='', offset=0, start_offset=0, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='print', argrepr='print + NULL', offset=4, start_offset=4, starts_line=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=2, argval='a', argrepr='a', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=3, argval='b', argrepr='b', offset=16, start_offset=16, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=4, argval='c', argrepr='c', offset=18, start_offset=18, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_DEREF', opcode=143, arg=5, argval='d', argrepr='d', offset=20, start_offset=20, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST_LOAD_FAST', opcode=147, arg=1, argval=('e', 'f'), argrepr='e, f', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=6, argval=6, argrepr='', offset=24, start_offset=24, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=32, start_offset=32, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=167, arg=0, argval=None, argrepr='None', offset=34, start_offset=34, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_jumpy = [ - Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=1, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='FOR_ITER', opcode=93, arg=28, argval=84, argrepr='to 84', offset=24, start_offset=24, starts_line=None, is_jump_target=True, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=2, argval=66, argrepr='to 66', offset=60, start_offset=60, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=21, argval=24, argrepr='to 24', offset=62, start_offset=62, starts_line=6, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=66, start_offset=66, starts_line=7, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=68, start_offset=68, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=148, argval='>', argrepr='bool(>)', offset=70, start_offset=70, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=2, argval=80, argrepr='to 80', offset=74, start_offset=74, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=28, argval=24, argrepr='to 24', offset=76, start_offset=76, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=80, start_offset=80, starts_line=8, is_jump_target=True, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=110, arg=12, argval=108, argrepr='to 108', offset=82, start_offset=82, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='END_FOR', opcode=4, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=3, is_jump_target=True, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=86, start_offset=86, starts_line=10, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=96, start_offset=96, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=98, start_offset=98, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=106, start_offset=106, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=127, arg=0, argval='i', argrepr='i', offset=108, start_offset=108, starts_line=11, is_jump_target=True, positions=None), - Instruction(opname='TO_BOOL', opcode=6, arg=None, argval=None, argrepr='', offset=110, start_offset=110, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=37, argval=194, argrepr='to 194', offset=118, start_offset=118, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=120, start_offset=120, starts_line=12, is_jump_target=True, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=130, start_offset=130, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=132, start_offset=132, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=140, start_offset=140, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=142, start_offset=142, starts_line=13, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=144, start_offset=144, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=122, arg=23, argval=23, argrepr='-=', offset=146, start_offset=146, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=152, start_offset=152, starts_line=14, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=154, start_offset=154, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=148, argval='>', argrepr='bool(>)', offset=156, start_offset=156, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=2, argval=166, argrepr='to 166', offset=160, start_offset=160, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=29, argval=108, argrepr='to 108', offset=162, start_offset=162, starts_line=15, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=166, start_offset=166, starts_line=16, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=168, start_offset=168, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=18, argval='<', argrepr='bool(<)', offset=170, start_offset=170, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=178, argrepr='to 178', offset=174, start_offset=174, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=110, arg=19, argval=216, argrepr='to 216', offset=176, start_offset=176, starts_line=17, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=178, start_offset=178, starts_line=11, is_jump_target=True, positions=None), - Instruction(opname='TO_BOOL', opcode=6, arg=None, argval=None, argrepr='', offset=180, start_offset=180, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=2, argval=194, argrepr='to 194', offset=188, start_offset=188, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=37, argval=120, argrepr='to 120', offset=190, start_offset=190, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=194, start_offset=194, starts_line=19, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=204, start_offset=204, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=206, start_offset=206, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=214, start_offset=214, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='NOP', opcode=9, arg=None, argval=None, argrepr='', offset=216, start_offset=216, starts_line=20, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=218, start_offset=218, starts_line=21, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=220, start_offset=220, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=122, arg=11, argval=11, argrepr='/', offset=222, start_offset=222, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=226, start_offset=226, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=228, start_offset=228, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='BEFORE_WITH', opcode=53, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=232, start_offset=232, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=234, start_offset=234, starts_line=26, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=244, start_offset=244, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=246, start_offset=246, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=254, start_offset=254, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=256, start_offset=256, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=258, start_offset=258, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=260, start_offset=260, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=2, argval=2, argrepr='', offset=262, start_offset=262, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=270, start_offset=270, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=272, start_offset=272, starts_line=28, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=282, start_offset=282, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=284, start_offset=284, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=292, start_offset=292, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=294, start_offset=294, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=296, start_offset=296, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='WITH_EXCEPT_START', opcode=49, arg=None, argval=None, argrepr='', offset=298, start_offset=298, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='TO_BOOL', opcode=6, arg=None, argval=None, argrepr='', offset=300, start_offset=300, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=312, argrepr='to 312', offset=308, start_offset=308, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=2, argval=2, argrepr='', offset=310, start_offset=310, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=None, is_jump_target=True, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=316, start_offset=316, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=318, start_offset=318, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=26, argval=272, argrepr='to 272', offset=320, start_offset=320, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=324, start_offset=324, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=328, start_offset=328, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=332, start_offset=332, starts_line=22, is_jump_target=False, positions=None), - Instruction(opname='CHECK_EXC_MATCH', opcode=36, arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=15, argval=376, argrepr='to 376', offset=344, start_offset=344, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=348, start_offset=348, starts_line=23, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=358, start_offset=358, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=360, start_offset=360, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=370, start_offset=370, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=52, argval=272, argrepr='to 272', offset=372, start_offset=372, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=376, start_offset=376, starts_line=22, is_jump_target=True, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=378, start_offset=378, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=382, start_offset=382, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='print + NULL', offset=386, start_offset=386, starts_line=28, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=396, start_offset=396, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=398, start_offset=398, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=406, start_offset=406, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=408, start_offset=408, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=410, start_offset=410, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=412, start_offset=412, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=1, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=1, argval='range', argrepr='range + NULL', offset=2, start_offset=2, starts_line=3, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=1, argval=10, argrepr='10', offset=12, start_offset=12, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=14, start_offset=14, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='GET_ITER', opcode=31, arg=None, argval=None, argrepr='', offset=22, start_offset=22, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='FOR_ITER', opcode=114, arg=28, argval=84, argrepr='to 84', offset=24, start_offset=24, starts_line=None, is_jump_target=True, positions=None), + Instruction(opname='STORE_FAST', opcode=176, arg=0, argval='i', argrepr='i', offset=28, start_offset=28, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=30, start_offset=30, starts_line=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=40, start_offset=40, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=42, start_offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=50, start_offset=50, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=52, start_offset=52, starts_line=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval=4, argrepr='4', offset=54, start_offset=54, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=97, arg=18, argval='<', argrepr='bool(<)', offset=56, start_offset=56, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=66, argrepr='to 66', offset=60, start_offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=21, argval=24, argrepr='to 24', offset=62, start_offset=62, starts_line=6, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=66, start_offset=66, starts_line=7, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=6, argrepr='6', offset=68, start_offset=68, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=97, arg=148, argval='>', argrepr='bool(>)', offset=70, start_offset=70, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=163, arg=2, argval=80, argrepr='to 80', offset=74, start_offset=74, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=28, argval=24, argrepr='to 24', offset=76, start_offset=76, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=80, start_offset=80, starts_line=8, is_jump_target=True, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=125, arg=12, argval=108, argrepr='to 108', offset=82, start_offset=82, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='END_FOR', opcode=24, arg=None, argval=None, argrepr='', offset=84, start_offset=84, starts_line=3, is_jump_target=True, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=86, start_offset=86, starts_line=10, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=96, start_offset=96, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=98, start_offset=98, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=106, start_offset=106, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=146, arg=0, argval='i', argrepr='i', offset=108, start_offset=108, starts_line=11, is_jump_target=True, positions=None), + Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=110, start_offset=110, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=37, argval=194, argrepr='to 194', offset=118, start_offset=118, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=120, start_offset=120, starts_line=12, is_jump_target=True, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=130, start_offset=130, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=132, start_offset=132, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=140, start_offset=140, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=142, start_offset=142, starts_line=13, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=5, argval=1, argrepr='1', offset=144, start_offset=144, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=67, arg=23, argval=23, argrepr='-=', offset=146, start_offset=146, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=176, arg=0, argval='i', argrepr='i', offset=150, start_offset=150, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=152, start_offset=152, starts_line=14, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=3, argval=6, argrepr='6', offset=154, start_offset=154, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=97, arg=148, argval='>', argrepr='bool(>)', offset=156, start_offset=156, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=166, argrepr='to 166', offset=160, start_offset=160, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=29, argval=108, argrepr='to 108', offset=162, start_offset=162, starts_line=15, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=166, start_offset=166, starts_line=16, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=2, argval=4, argrepr='4', offset=168, start_offset=168, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=97, arg=18, argval='<', argrepr='bool(<)', offset=170, start_offset=170, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=1, argval=178, argrepr='to 178', offset=174, start_offset=174, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=125, arg=19, argval=216, argrepr='to 216', offset=176, start_offset=176, starts_line=17, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=178, start_offset=178, starts_line=11, is_jump_target=True, positions=None), + Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=180, start_offset=180, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=2, argval=194, argrepr='to 194', offset=188, start_offset=188, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=37, argval=120, argrepr='to 120', offset=190, start_offset=190, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=194, start_offset=194, starts_line=19, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=204, start_offset=204, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=206, start_offset=206, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=214, start_offset=214, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='NOP', opcode=42, arg=None, argval=None, argrepr='', offset=216, start_offset=216, starts_line=20, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=5, argval=1, argrepr='1', offset=218, start_offset=218, starts_line=21, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=7, argval=0, argrepr='0', offset=220, start_offset=220, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=67, arg=11, argval=11, argrepr='/', offset=222, start_offset=222, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=226, start_offset=226, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=144, arg=0, argval='i', argrepr='i', offset=228, start_offset=228, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='BEFORE_WITH', opcode=2, arg=None, argval=None, argrepr='', offset=230, start_offset=230, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=176, arg=1, argval='dodgy', argrepr='dodgy', offset=232, start_offset=232, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=234, start_offset=234, starts_line=26, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=244, start_offset=244, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=246, start_offset=246, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=254, start_offset=254, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=0, argval=None, argrepr='None', offset=256, start_offset=256, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=0, argval=None, argrepr='None', offset=258, start_offset=258, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=0, argval=None, argrepr='None', offset=260, start_offset=260, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=2, argval=2, argrepr='', offset=262, start_offset=262, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=270, start_offset=270, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=272, start_offset=272, starts_line=28, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=282, start_offset=282, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=284, start_offset=284, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=292, start_offset=292, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=167, arg=0, argval=None, argrepr='None', offset=294, start_offset=294, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=45, arg=None, argval=None, argrepr='', offset=296, start_offset=296, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='WITH_EXCEPT_START', opcode=66, arg=None, argval=None, argrepr='', offset=298, start_offset=298, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='TO_BOOL', opcode=56, arg=None, argval=None, argrepr='', offset=300, start_offset=300, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=163, arg=1, argval=312, argrepr='to 312', offset=308, start_offset=308, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=2, argval=2, argrepr='', offset=310, start_offset=310, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=312, start_offset=312, starts_line=None, is_jump_target=True, positions=None), + Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=314, start_offset=314, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=316, start_offset=316, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=318, start_offset=318, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=26, argval=272, argrepr='to 272', offset=320, start_offset=320, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=103, arg=3, argval=3, argrepr='', offset=324, start_offset=324, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=326, start_offset=326, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=1, argval=1, argrepr='', offset=328, start_offset=328, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=45, arg=None, argval=None, argrepr='', offset=330, start_offset=330, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=332, start_offset=332, starts_line=22, is_jump_target=False, positions=None), + Instruction(opname='CHECK_EXC_MATCH', opcode=20, arg=None, argval=None, argrepr='', offset=342, start_offset=342, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=160, arg=15, argval=376, argrepr='to 376', offset=344, start_offset=344, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=346, start_offset=346, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=348, start_offset=348, starts_line=23, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=358, start_offset=358, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=360, start_offset=360, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=368, start_offset=368, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=370, start_offset=370, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=123, arg=52, argval=272, argrepr='to 272', offset=372, start_offset=372, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=0, argval=0, argrepr='', offset=376, start_offset=376, starts_line=22, is_jump_target=True, positions=None), + Instruction(opname='COPY', opcode=103, arg=3, argval=3, argrepr='', offset=378, start_offset=378, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=380, start_offset=380, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=1, argval=1, argrepr='', offset=382, start_offset=382, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=45, arg=None, argval=None, argrepr='', offset=384, start_offset=384, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=150, arg=3, argval='print', argrepr='print + NULL', offset=386, start_offset=386, starts_line=28, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=142, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=396, start_offset=396, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=75, arg=1, argval=1, argrepr='', offset=398, start_offset=398, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=44, arg=None, argval=None, argrepr='', offset=406, start_offset=406, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=0, argval=0, argrepr='', offset=408, start_offset=408, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=103, arg=3, argval=3, argrepr='', offset=410, start_offset=410, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=43, arg=None, argval=None, argrepr='', offset=412, start_offset=412, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=165, arg=1, argval=1, argrepr='', offset=414, start_offset=414, starts_line=None, is_jump_target=False, positions=None), ] # One last piece of inspect fodder to check the default line number handling def simple(): pass expected_opinfo_simple = [ - Instruction(opname='RESUME', opcode=151, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=simple.__code__.co_firstlineno, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=None, is_jump_target=False), + Instruction(opname='RESUME', opcode=166, arg=0, argval=0, argrepr='', offset=0, start_offset=0, starts_line=simple.__code__.co_firstlineno, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=167, arg=0, argval=None, argrepr='None', offset=2, start_offset=2, starts_line=None, is_jump_target=False), ] @@ -1943,7 +1942,7 @@ def test_baseopname_and_baseopcode(self): self.assertEqual(code, baseopcode) # Specialized instructions - for name in opcode._specialized_instructions: + for name in opcode._specialized_opmap: instruction = Instruction(opname=name, opcode=dis._all_opmap[name], arg=None, argval=None, argrepr='', offset=0, start_offset=0, starts_line=1, is_jump_target=False, positions=None) baseopname = instruction.baseopname diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 582392ecddcb9..50c9f61017e02 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -361,7 +361,7 @@ def is_specialized(f): for instruction in dis.get_instructions(f, adaptive=True): opname = instruction.opname if ( - opname in opcode._specialized_instructions + opname in opcode._specialized_opmap # Exclude superinstructions: and "__" not in opname ): diff --git a/Makefile.pre.in b/Makefile.pre.in index 32d928316f221..bcec0782f6e95 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1431,14 +1431,9 @@ regen-opcode: # using Tools/build/generate_opcode_h.py $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \ $(srcdir)/Lib/opcode.py \ - $(srcdir)/Lib/_opcode_metadata.py \ - $(srcdir)/Include/opcode_ids.h.new \ $(srcdir)/Include/opcode.h.new \ - $(srcdir)/Python/opcode_targets.h.new \ $(srcdir)/Include/internal/pycore_opcode.h.new - $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new - $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new .PHONY: regen-token @@ -1550,12 +1545,16 @@ regen-cases: $(srcdir)/Tools/cases_generator/generate_cases.py \ $(CASESFLAG) \ -o $(srcdir)/Python/generated_cases.c.h.new \ + -n $(srcdir)/Include/opcode_ids.h.new \ + -t $(srcdir)/Python/opcode_targets.h.new \ -m $(srcdir)/Include/internal/pycore_opcode_metadata.h.new \ -e $(srcdir)/Python/executor_cases.c.h.new \ -p $(srcdir)/Lib/_opcode_metadata.py.new \ -a $(srcdir)/Python/abstract_interp_cases.c.h.new \ $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/generated_cases.c.h $(srcdir)/Python/generated_cases.c.h.new + $(UPDATE_FILE) $(srcdir)/Include/opcode_ids.h $(srcdir)/Include/opcode_ids.h.new + $(UPDATE_FILE) $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/opcode_targets.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode_metadata.h $(srcdir)/Include/internal/pycore_opcode_metadata.h.new $(UPDATE_FILE) $(srcdir)/Python/executor_cases.c.h $(srcdir)/Python/executor_cases.c.h.new $(UPDATE_FILE) $(srcdir)/Python/abstract_interp_cases.c.h $(srcdir)/Python/abstract_interp_cases.c.h.new diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-15-13-06-05.gh-issue-107971.lPbx04.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-15-13-06-05.gh-issue-107971.lPbx04.rst new file mode 100644 index 0000000000000..dc10f672d8871 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-15-13-06-05.gh-issue-107971.lPbx04.rst @@ -0,0 +1,2 @@ +Opcode IDs are generated from bytecodes.c instead of being hard coded in +opcode.py. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 6987a2382d81c..2c9c8cec77ff9 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -6,7 +6,8 @@ #include "pycore_code.h" // _PyCodeConstructor #include "pycore_frame.h" // FRAME_SPECIALS_SIZE #include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs -#include "pycore_opcode.h" // _PyOpcode_Deopt +#include "pycore_opcode.h" // _PyOpcode_Caches +#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_tuple.h" // _PyTuple_ITEMS() diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 17571535048e2..80e118e8a8aa9 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -7,6 +7,8 @@ #include "pycore_moduleobject.h" // _PyModule_GetDict() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode.h" // _PyOpcode_Caches +#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt + #include "frameobject.h" // PyFrameObject #include "pycore_frame.h" diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index c1189c883b667..ed02f9163549b 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -14,7 +14,7 @@ -C <_OpcodeSources Include="$(PySourcePath)Tools\build\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" /> - <_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Include\internal\pycore_opcode.h;$(PySourcePath)Python\opcode_targets.h" /> + <_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Include\internal\pycore_opcode.h" /> <_TokenSources Include="$(PySourcePath)Grammar\Tokens" /> <_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc"> rst @@ -59,7 +59,7 @@ Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)" DependsOnTargets="FindPythonForBuild"> - diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 0dca507e28bc1..3fd6cdade69f9 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,17 +1,17 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, - 0,0,0,0,0,243,164,0,0,0,151,0,100,0,100,1, - 108,0,90,0,100,0,100,1,108,1,90,1,101,2,2,0, - 100,2,171,1,0,0,0,0,0,0,1,0,101,2,2,0, - 100,3,101,0,106,6,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,171,2,0,0,0,0,0,0, - 1,0,101,1,106,8,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,2,0,171,0,0,0,0,0, - 0,0,100,4,25,0,0,0,90,5,100,5,68,0,93,20, - 0,0,90,6,101,2,2,0,100,6,101,6,40,0,100,7, - 101,5,101,6,25,0,0,0,40,0,157,4,171,1,0,0, - 0,0,0,0,1,0,140,22,0,0,4,0,121,1,41,8, + 0,0,0,0,0,243,164,0,0,0,166,0,142,0,142,1, + 121,0,180,0,142,0,142,1,121,1,180,1,153,2,46,0, + 142,2,75,1,0,0,0,0,0,0,44,0,153,2,46,0, + 142,3,153,0,129,6,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,75,2,0,0,0,0,0,0, + 44,0,153,1,129,8,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,46,0,75,0,0,0,0,0, + 0,0,142,4,12,0,0,0,180,5,142,5,31,0,114,20, + 0,0,180,6,153,2,46,0,142,6,153,6,27,0,142,7, + 153,5,153,6,12,0,0,0,27,0,73,4,75,1,0,0, + 0,0,0,0,44,0,123,22,0,0,24,0,167,1,41,8, 233,0,0,0,0,78,122,18,70,114,111,122,101,110,32,72, 101,108,108,111,32,87,111,114,108,100,122,8,115,121,115,46, 97,114,103,118,218,6,99,111,110,102,105,103,41,5,218,12, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e9a5cf59e7d68..2a5ad2c942fb3 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -17,7 +17,6 @@ #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_opcode.h" // EXTRA_CASES #include "pycore_opcode_metadata.h" // uop names #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 210c37b37225b..305eb0bfe2a7c 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -1,191 +1,192 @@ static void *opcode_targets[256] = { &&TARGET_CACHE, - &&TARGET_POP_TOP, - &&TARGET_PUSH_NULL, - &&TARGET_INTERPRETER_EXIT, - &&TARGET_END_FOR, - &&TARGET_END_SEND, - &&TARGET_TO_BOOL, - &&TARGET_TO_BOOL_ALWAYS_TRUE, - &&TARGET_TO_BOOL_BOOL, - &&TARGET_NOP, - &&TARGET_TO_BOOL_INT, - &&TARGET_UNARY_NEGATIVE, - &&TARGET_UNARY_NOT, - &&TARGET_TO_BOOL_LIST, - &&TARGET_TO_BOOL_NONE, - &&TARGET_UNARY_INVERT, - &&TARGET_EXIT_INIT_CHECK, - &&TARGET_RESERVED, - &&TARGET_TO_BOOL_STR, - &&TARGET_BINARY_OP_MULTIPLY_INT, - &&TARGET_BINARY_OP_ADD_INT, - &&TARGET_BINARY_OP_SUBTRACT_INT, - &&TARGET_BINARY_OP_MULTIPLY_FLOAT, + &&TARGET_BEFORE_ASYNC_WITH, + &&TARGET_BEFORE_WITH, &&TARGET_BINARY_OP_ADD_FLOAT, - &&TARGET_MAKE_FUNCTION, - &&TARGET_BINARY_SUBSCR, - &&TARGET_BINARY_SLICE, - &&TARGET_STORE_SLICE, - &&TARGET_BINARY_OP_SUBTRACT_FLOAT, + &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, - &&TARGET_GET_LEN, - &&TARGET_MATCH_MAPPING, - &&TARGET_MATCH_SEQUENCE, - &&TARGET_MATCH_KEYS, &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, - &&TARGET_PUSH_EXC_INFO, - &&TARGET_CHECK_EXC_MATCH, - &&TARGET_CHECK_EG_MATCH, + &&TARGET_BINARY_OP_MULTIPLY_FLOAT, + &&TARGET_BINARY_OP_MULTIPLY_INT, + &&TARGET_BINARY_OP_SUBTRACT_FLOAT, + &&TARGET_BINARY_OP_SUBTRACT_INT, + &&TARGET_BINARY_SLICE, + &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_BINARY_SUBSCR_GETITEM, - &&TARGET_FORMAT_SIMPLE, - &&TARGET_FORMAT_WITH_SPEC, &&TARGET_BINARY_SUBSCR_LIST_INT, &&TARGET_BINARY_SUBSCR_STR_INT, + &&TARGET_RESERVED, &&TARGET_BINARY_SUBSCR_TUPLE_INT, - &&TARGET_STORE_SUBSCR_DICT, - &&TARGET_STORE_SUBSCR_LIST_INT, - &&TARGET_SEND_GEN, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, - &&TARGET_WITH_EXCEPT_START, - &&TARGET_GET_AITER, - &&TARGET_GET_ANEXT, - &&TARGET_BEFORE_ASYNC_WITH, - &&TARGET_BEFORE_WITH, - &&TARGET_END_ASYNC_FOR, + &&TARGET_CHECK_EG_MATCH, + &&TARGET_CHECK_EXC_MATCH, &&TARGET_CLEANUP_THROW, - &&TARGET_UNPACK_SEQUENCE_TUPLE, - &&TARGET_UNPACK_SEQUENCE_LIST, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_ATTR_SLOT, - &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, - &&TARGET_STORE_ATTR_WITH_HINT, - &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_LOAD_GLOBAL_BUILTIN, - &&TARGET_LOAD_SUPER_ATTR_ATTR, - &&TARGET_LOAD_SUPER_ATTR_METHOD, - &&TARGET_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_END_ASYNC_FOR, + &&TARGET_END_FOR, + &&TARGET_END_SEND, + &&TARGET_EXIT_INIT_CHECK, + &&TARGET_FORMAT_SIMPLE, + &&TARGET_FORMAT_WITH_SPEC, + &&TARGET_GET_AITER, + &&TARGET_GET_ANEXT, &&TARGET_GET_ITER, + &&TARGET_GET_LEN, &&TARGET_GET_YIELD_FROM_ITER, - &&TARGET_LOAD_ATTR_MODULE, - &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_LOAD_ATTR_WITH_HINT, - &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_INTERPRETER_EXIT, &&TARGET_LOAD_ASSERTION_ERROR, + &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_LOCALS, + &&TARGET_MAKE_FUNCTION, + &&TARGET_MATCH_KEYS, + &&TARGET_MATCH_MAPPING, + &&TARGET_MATCH_SEQUENCE, + &&TARGET_NOP, + &&TARGET_POP_EXCEPT, + &&TARGET_POP_TOP, + &&TARGET_PUSH_EXC_INFO, + &&TARGET_PUSH_NULL, &&TARGET_RETURN_GENERATOR, - &&TARGET_LOAD_ATTR_CLASS, - &&TARGET_LOAD_ATTR_PROPERTY, - &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, - &&TARGET_LOAD_ATTR_METHOD_NO_DICT, - &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, - &&TARGET_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, &&TARGET_RETURN_VALUE, - &&TARGET_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_COMPARE_OP_FLOAT, - &&TARGET_LOAD_LOCALS, - &&TARGET_COMPARE_OP_INT, - &&TARGET_POP_EXCEPT, - &&TARGET_STORE_NAME, - &&TARGET_DELETE_NAME, - &&TARGET_UNPACK_SEQUENCE, - &&TARGET_FOR_ITER, - &&TARGET_UNPACK_EX, - &&TARGET_STORE_ATTR, - &&TARGET_DELETE_ATTR, - &&TARGET_STORE_GLOBAL, - &&TARGET_DELETE_GLOBAL, - &&TARGET_SWAP, - &&TARGET_LOAD_CONST, - &&TARGET_LOAD_NAME, - &&TARGET_BUILD_TUPLE, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_STORE_ATTR_SLOT, + &&TARGET_STORE_SLICE, + &&TARGET_STORE_SUBSCR, + &&TARGET_STORE_SUBSCR_DICT, + &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_TO_BOOL, + &&TARGET_TO_BOOL_ALWAYS_TRUE, + &&TARGET_TO_BOOL_BOOL, + &&TARGET_TO_BOOL_INT, + &&TARGET_TO_BOOL_LIST, + &&TARGET_TO_BOOL_NONE, + &&TARGET_TO_BOOL_STR, + &&TARGET_UNARY_INVERT, + &&TARGET_UNARY_NEGATIVE, + &&TARGET_UNARY_NOT, + &&TARGET_WITH_EXCEPT_START, + &&TARGET_BINARY_OP, + &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_LIST, - &&TARGET_BUILD_SET, &&TARGET_BUILD_MAP, - &&TARGET_LOAD_ATTR, + &&TARGET_BUILD_SET, + &&TARGET_BUILD_SLICE, + &&TARGET_BUILD_STRING, + &&TARGET_BUILD_TUPLE, + &&TARGET_CALL, + &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_CALL_BUILTIN_CLASS, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, + &&TARGET_CALL_FUNCTION_EX, + &&TARGET_CALL_INTRINSIC_1, + &&TARGET_CALL_INTRINSIC_2, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + &&TARGET_CALL_NO_KW_ALLOC_AND_ENTER_INIT, + &&TARGET_CALL_NO_KW_BUILTIN_FAST, + &&TARGET_CALL_NO_KW_BUILTIN_O, + &&TARGET_CALL_NO_KW_ISINSTANCE, + &&TARGET_CALL_NO_KW_LEN, + &&TARGET_CALL_NO_KW_LIST_APPEND, + &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST, + &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, + &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, + &&TARGET_CALL_NO_KW_STR_1, + &&TARGET_CALL_NO_KW_TUPLE_1, + &&TARGET_CALL_NO_KW_TYPE_1, + &&TARGET_CALL_PY_EXACT_ARGS, + &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_COMPARE_OP, - &&TARGET_IMPORT_NAME, - &&TARGET_IMPORT_FROM, - &&TARGET_JUMP_FORWARD, + &&TARGET_COMPARE_OP_FLOAT, + &&TARGET_COMPARE_OP_INT, &&TARGET_COMPARE_OP_STR, - &&TARGET_FOR_ITER_LIST, - &&TARGET_FOR_ITER_TUPLE, - &&TARGET_POP_JUMP_IF_FALSE, - &&TARGET_POP_JUMP_IF_TRUE, - &&TARGET_LOAD_GLOBAL, - &&TARGET_IS_OP, &&TARGET_CONTAINS_OP, - &&TARGET_RERAISE, + &&TARGET_CONVERT_VALUE, &&TARGET_COPY, - &&TARGET_RETURN_CONST, - &&TARGET_BINARY_OP, - &&TARGET_SEND, - &&TARGET_LOAD_FAST, - &&TARGET_STORE_FAST, + &&TARGET_COPY_FREE_VARS, + &&TARGET_DELETE_ATTR, + &&TARGET_DELETE_DEREF, &&TARGET_DELETE_FAST, - &&TARGET_LOAD_FAST_CHECK, - &&TARGET_POP_JUMP_IF_NOT_NONE, - &&TARGET_POP_JUMP_IF_NONE, - &&TARGET_RAISE_VARARGS, - &&TARGET_GET_AWAITABLE, + &&TARGET_DELETE_GLOBAL, + &&TARGET_DELETE_NAME, + &&TARGET_DICT_MERGE, + &&TARGET_DICT_UPDATE, + &&TARGET_ENTER_EXECUTOR, + &&TARGET_EXTENDED_ARG, + &&TARGET_FOR_ITER, + &&TARGET_FOR_ITER_GEN, + &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_RANGE, - &&TARGET_BUILD_SLICE, + &&TARGET_FOR_ITER_TUPLE, + &&TARGET_GET_AWAITABLE, + &&TARGET_IMPORT_FROM, + &&TARGET_IMPORT_NAME, + &&TARGET_IS_OP, + &&TARGET_JUMP_BACKWARD, &&TARGET_JUMP_BACKWARD_NO_INTERRUPT, - &&TARGET_MAKE_CELL, - &&TARGET_FOR_ITER_GEN, + &&TARGET_JUMP_FORWARD, + &&TARGET_KW_NAMES, + &&TARGET_LIST_APPEND, + &&TARGET_LIST_EXTEND, + &&TARGET_LOAD_ATTR, + &&TARGET_LOAD_ATTR_CLASS, + &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + &&TARGET_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + &&TARGET_LOAD_ATTR_PROPERTY, + &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_CONST, &&TARGET_LOAD_DEREF, - &&TARGET_STORE_DEREF, - &&TARGET_DELETE_DEREF, - &&TARGET_JUMP_BACKWARD, - &&TARGET_LOAD_SUPER_ATTR, - &&TARGET_CALL_FUNCTION_EX, + &&TARGET_LOAD_FAST, &&TARGET_LOAD_FAST_AND_CLEAR, - &&TARGET_EXTENDED_ARG, - &&TARGET_LIST_APPEND, - &&TARGET_SET_ADD, + &&TARGET_LOAD_FAST_CHECK, + &&TARGET_LOAD_FAST_LOAD_FAST, + &&TARGET_LOAD_FROM_DICT_OR_DEREF, + &&TARGET_LOAD_FROM_DICT_OR_GLOBALS, + &&TARGET_LOAD_GLOBAL, + &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_NAME, + &&TARGET_LOAD_SUPER_ATTR, + &&TARGET_LOAD_SUPER_ATTR_ATTR, + &&TARGET_LOAD_SUPER_ATTR_METHOD, + &&TARGET_MAKE_CELL, &&TARGET_MAP_ADD, - &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, - &&TARGET_COPY_FREE_VARS, - &&TARGET_YIELD_VALUE, - &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_CALL_PY_EXACT_ARGS, - &&TARGET_CALL_PY_WITH_DEFAULTS, - &&TARGET_CALL_NO_KW_TYPE_1, - &&TARGET_BUILD_CONST_KEY_MAP, - &&TARGET_BUILD_STRING, - &&TARGET_CONVERT_VALUE, - &&TARGET_CALL_NO_KW_STR_1, - &&TARGET_CALL_NO_KW_TUPLE_1, - &&TARGET_CALL_BUILTIN_CLASS, - &&TARGET_LIST_EXTEND, + &&TARGET_POP_JUMP_IF_FALSE, + &&TARGET_POP_JUMP_IF_NONE, + &&TARGET_POP_JUMP_IF_NOT_NONE, + &&TARGET_POP_JUMP_IF_TRUE, + &&TARGET_RAISE_VARARGS, + &&TARGET_RERAISE, + &&TARGET_RESUME, + &&TARGET_RETURN_CONST, + &&TARGET_SEND, + &&TARGET_SEND_GEN, + &&TARGET_SET_ADD, + &&TARGET_SET_FUNCTION_ATTRIBUTE, &&TARGET_SET_UPDATE, - &&TARGET_DICT_MERGE, - &&TARGET_DICT_UPDATE, - &&TARGET_CALL_NO_KW_BUILTIN_O, - &&TARGET_CALL_NO_KW_BUILTIN_FAST, - &&TARGET_LOAD_FAST_LOAD_FAST, + &&TARGET_STORE_ATTR, + &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_DEREF, + &&TARGET_STORE_FAST, &&TARGET_STORE_FAST_LOAD_FAST, &&TARGET_STORE_FAST_STORE_FAST, - &&TARGET_CALL, - &&TARGET_KW_NAMES, - &&TARGET_CALL_INTRINSIC_1, - &&TARGET_CALL_INTRINSIC_2, - &&TARGET_LOAD_FROM_DICT_OR_GLOBALS, - &&TARGET_LOAD_FROM_DICT_OR_DEREF, - &&TARGET_SET_FUNCTION_ATTRIBUTE, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, - &&TARGET_CALL_NO_KW_LEN, - &&TARGET_CALL_NO_KW_ISINSTANCE, - &&TARGET_CALL_NO_KW_LIST_APPEND, - &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, - &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, - &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, - &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST, - &&TARGET_CALL_NO_KW_ALLOC_AND_ENTER_INIT, + &&TARGET_STORE_GLOBAL, + &&TARGET_STORE_NAME, + &&TARGET_SWAP, + &&TARGET_UNPACK_EX, + &&TARGET_UNPACK_SEQUENCE, + &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_UNPACK_SEQUENCE_TUPLE, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, + &&TARGET_YIELD_VALUE, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -229,30 +230,28 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&TARGET_ENTER_EXECUTOR, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, - &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, - &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, &&TARGET_INSTRUMENTED_RESUME, - &&TARGET_INSTRUMENTED_CALL, + &&TARGET_INSTRUMENTED_END_FOR, + &&TARGET_INSTRUMENTED_END_SEND, &&TARGET_INSTRUMENTED_RETURN_VALUE, + &&TARGET_INSTRUMENTED_RETURN_CONST, &&TARGET_INSTRUMENTED_YIELD_VALUE, + &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, + &&TARGET_INSTRUMENTED_FOR_ITER, + &&TARGET_INSTRUMENTED_CALL, &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, + &&TARGET_INSTRUMENTED_INSTRUCTION, &&TARGET_INSTRUMENTED_JUMP_FORWARD, &&TARGET_INSTRUMENTED_JUMP_BACKWARD, - &&TARGET_INSTRUMENTED_RETURN_CONST, - &&TARGET_INSTRUMENTED_FOR_ITER, - &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, &&TARGET_INSTRUMENTED_POP_JUMP_IF_TRUE, - &&TARGET_INSTRUMENTED_END_FOR, - &&TARGET_INSTRUMENTED_END_SEND, - &&TARGET_INSTRUMENTED_INSTRUCTION, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_FALSE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_NONE, + &&TARGET_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, &&TARGET_INSTRUMENTED_LINE, - &&_unknown_opcode -}; + &&_unknown_opcode}; diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index ce609bd089874..8dbb7bfa69a92 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -22,7 +22,7 @@ verbose = False # This must be kept in sync with opcode.py -RESUME = 151 +RESUME = 166 def isprintable(b: bytes) -> bool: return all(0x20 <= c < 0x7f for c in b) @@ -297,10 +297,12 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_linetable = {co_linetable},") self.write(f"._co_cached = NULL,") self.write(f".co_code_adaptive = {co_code_adaptive},") - for i, op in enumerate(code.co_code[::2]): + first_traceable = 0 + for op in code.co_code[::2]: if op == RESUME: - self.write(f"._co_firsttraceable = {i},") break + first_traceable += 1 + self.write(f"._co_firsttraceable = {first_traceable},") name_as_code = f"(PyCodeObject *)&{name}" self.finis.append(f"_PyStaticCode_Fini({name_as_code});") self.inits.append(f"_PyStaticCode_Init({name_as_code})") diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 67f4a2c2d5d76..344709a05184c 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -27,27 +27,6 @@ #endif /* !Py_OPCODE_H */ """ -opcode_ids_h_header = f""" -// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} - -#ifndef Py_OPCODE_IDS_H -#define Py_OPCODE_IDS_H -#ifdef __cplusplus -extern "C" {{ -#endif - - -/* Instruction opcodes for compiled code */ -""".lstrip() - -opcode_ids_h_footer = """ - -#ifdef __cplusplus -} -#endif -#endif /* !Py_OPCODE_IDS_H */ -""" - internal_header = f""" // Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} @@ -83,52 +62,10 @@ def get_python_module_dict(filename): return mod def main(opcode_py, - _opcode_metadata_py='Lib/_opcode_metadata.py', - opcode_ids_h='Include/opcode_ids.h', opcode_h='Include/opcode.h', - opcode_targets_h='Python/opcode_targets.h', internal_opcode_h='Include/internal/pycore_opcode.h'): - _opcode_metadata = get_python_module_dict(_opcode_metadata_py) - opcode = get_python_module_dict(opcode_py) - opmap = opcode['opmap'] - opname = opcode['opname'] - - MIN_INSTRUMENTED_OPCODE = opcode["MIN_INSTRUMENTED_OPCODE"] - - NUM_OPCODES = len(opname) - used = [ False ] * len(opname) - next_op = 1 - - for name, op in opmap.items(): - used[op] = True - - specialized_opmap = {} - opname_including_specialized = opname.copy() - for name in _opcode_metadata['_specialized_instructions']: - while used[next_op]: - next_op += 1 - specialized_opmap[name] = next_op - opname_including_specialized[next_op] = name - used[next_op] = True - - with open(opcode_ids_h, 'w') as fobj: - fobj.write(opcode_ids_h_header) - - for name in opname: - if name in opmap: - op = opmap[name] - if op == MIN_INSTRUMENTED_OPCODE: - fobj.write(DEFINE.format("MIN_INSTRUMENTED_OPCODE", MIN_INSTRUMENTED_OPCODE)) - - fobj.write(DEFINE.format(name, op)) - - - for name, op in specialized_opmap.items(): - fobj.write(DEFINE.format(name, op)) - - fobj.write(opcode_ids_h_footer) with open(opcode_h, 'w') as fobj: fobj.write(opcode_h_header) @@ -143,7 +80,6 @@ def main(opcode_py, iobj.write(internal_header) iobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n") - iobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n") iobj.write("\n#ifdef NEED_OPCODE_TABLES\n") iobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n") @@ -151,52 +87,12 @@ def main(opcode_py, iobj.write(f" [{name}] = {entries},\n") iobj.write("};\n") - deoptcodes = {} - for basic, op in opmap.items(): - if op < 256: - deoptcodes[basic] = basic - for basic, family in _opcode_metadata["_specializations"].items(): - for specialized in family: - deoptcodes[specialized] = basic - iobj.write("\nconst uint8_t _PyOpcode_Deopt[256] = {\n") - for opt, deopt in sorted(deoptcodes.items()): - iobj.write(f" [{opt}] = {deopt},\n") - iobj.write("};\n") - iobj.write("#endif // NEED_OPCODE_TABLES\n") - - iobj.write("\n") - iobj.write(f"\nextern const char *const _PyOpcode_OpName[{NUM_OPCODES}];\n") - iobj.write("\n#ifdef NEED_OPCODE_TABLES\n") - iobj.write(f"const char *const _PyOpcode_OpName[{NUM_OPCODES}] = {{\n") - for op, name in enumerate(opname_including_specialized): - if name[0] != "<": - op = name - iobj.write(f''' [{op}] = "{name}",\n''') - iobj.write("};\n") iobj.write("#endif // NEED_OPCODE_TABLES\n") - iobj.write("\n") - iobj.write("#define EXTRA_CASES \\\n") - for i, flag in enumerate(used): - if not flag: - iobj.write(f" case {i}: \\\n") - iobj.write(" ;\n") - iobj.write(internal_footer) - with open(opcode_targets_h, "w") as f: - targets = ["_unknown_opcode"] * 256 - for op, name in enumerate(opname_including_specialized): - if op < 256 and not name.startswith("<"): - targets[op] = f"TARGET_{name}" - - f.write("static void *opcode_targets[256] = {\n") - f.write(",\n".join([f" &&{s}" for s in targets])) - f.write("\n};\n") - print(f"{opcode_h} regenerated from {opcode_py}") if __name__ == '__main__': - main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], - sys.argv[5], sys.argv[6]) + main(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index e170e110f80cf..d991cb4690063 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -5,6 +5,7 @@ import argparse import contextlib +import itertools import os import posixpath import sys @@ -36,6 +37,12 @@ DEFAULT_INPUT = os.path.relpath(os.path.join(ROOT, "Python/bytecodes.c")) DEFAULT_OUTPUT = os.path.relpath(os.path.join(ROOT, "Python/generated_cases.c.h")) +DEFAULT_OPCODE_IDS_H_OUTPUT = os.path.relpath( + os.path.join(ROOT, "Include/opcode_ids.h") +) +DEFAULT_OPCODE_TARGETS_H_OUTPUT = os.path.relpath( + os.path.join(ROOT, "Python/opcode_targets.h") +) DEFAULT_METADATA_OUTPUT = os.path.relpath( os.path.join(ROOT, "Include/internal/pycore_opcode_metadata.h") ) @@ -86,6 +93,20 @@ arg_parser.add_argument( "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT ) +arg_parser.add_argument( + "-n", + "--opcode_ids_h", + type=str, + help="Header file with opcode number definitions", + default=DEFAULT_OPCODE_IDS_H_OUTPUT, +) +arg_parser.add_argument( + "-t", + "--opcode_targets_h", + type=str, + help="File with opcode targets for computed gotos", + default=DEFAULT_OPCODE_TARGETS_H_OUTPUT, +) arg_parser.add_argument( "-m", "--metadata", @@ -225,6 +246,129 @@ def write_provenance_header(self): self.out.write_raw(self.from_source_files()) self.out.write_raw(f"{self.out.comment} Do not edit!\n") + def assign_opcode_ids(self): + """Assign IDs to opcodes""" + + ops: list[(bool, str)] = [] # (has_arg, name) for each opcode + instrumented_ops: list[str] = [] + + for instr in itertools.chain( + [instr for instr in self.instrs.values() if instr.kind != "op"], + self.macro_instrs.values()): + + name = instr.name + if name.startswith('INSTRUMENTED_'): + instrumented_ops.append(name) + else: + ops.append((instr.instr_flags.HAS_ARG_FLAG, name)) + + # Special case: this instruction is implemented in ceval.c + # rather than bytecodes.c, so we need to add it explicitly + # here (at least until we add something to bytecodes.c to + # declare external instructions). + instrumented_ops.append('INSTRUMENTED_LINE') + + # assert lists are unique + assert len(set(ops)) == len(ops) + assert len(set(instrumented_ops)) == len(instrumented_ops) + + opname: list[str or None] = [None] * 512 + opmap: dict = {} + markers: dict = {} + + def map_op(op, name): + assert op < len(opname) + assert opname[op] is None + assert name not in opmap + opname[op] = name + opmap[name] = op + + + # 0 is reserved for cache entries. This helps debugging. + map_op(0, 'CACHE') + + # 17 is reserved as it is the initial value for the specializing counter. + # This helps catch cases where we attempt to execute a cache. + map_op(17, 'RESERVED') + + # 166 is RESUME - it is hard coded as such in Tools/build/deepfreeze.py + map_op(166, 'RESUME') + + next_opcode = 1 + + for has_arg, name in sorted(ops): + if name in opmap: + continue # an anchored name, like CACHE + while opname[next_opcode] is not None: + next_opcode += 1 + assert next_opcode < 255 + map_op(next_opcode, name) + + if has_arg and 'HAVE_ARGUMENT' not in markers: + markers['HAVE_ARGUMENT'] = next_opcode + + # Instrumented opcodes are at the end of the valid range + min_instrumented = 254 - (len(instrumented_ops) - 1) + assert next_opcode <= min_instrumented + markers['MIN_INSTRUMENTED_OPCODE'] = min_instrumented + for i, op in enumerate(instrumented_ops): + map_op(min_instrumented + i, op) + + # Pseudo opcodes are after the valid range + for i, op in enumerate(sorted(self.pseudos)): + map_op(256 + i, op) + + assert 255 not in opmap # 255 is reserved + self.opmap = opmap + self.markers = markers + + def write_opcode_ids(self, opcode_ids_h_filename, opcode_targets_filename): + """Write header file that defined the opcode IDs""" + + with open(opcode_ids_h_filename, "w") as f: + # Create formatter + self.out = Formatter(f, 0) + + self.write_provenance_header() + + self.out.emit("") + self.out.emit("#ifndef Py_OPCODE_IDS_H") + self.out.emit("#define Py_OPCODE_IDS_H") + self.out.emit("#ifdef __cplusplus") + self.out.emit("extern \"C\" {") + self.out.emit("#endif") + self.out.emit("") + self.out.emit("/* Instruction opcodes for compiled code */") + + def define(name, opcode): + self.out.emit(f"#define {name:<38} {opcode:>3}") + + all_pairs = [] + # the second item in the tuple sorts the markers before the ops + all_pairs.extend((i, 1, name) for (name, i) in self.markers.items()) + all_pairs.extend((i, 2, name) for (name, i) in self.opmap.items()) + for i, _, name in sorted(all_pairs): + assert name is not None + define(name, i) + + self.out.emit("") + self.out.emit("#ifdef __cplusplus") + self.out.emit("}") + self.out.emit("#endif") + self.out.emit("#endif /* !Py_OPCODE_IDS_H */") + + with open(opcode_targets_filename, "w") as f: + # Create formatter + self.out = Formatter(f, 0) + + with self.out.block("static void *opcode_targets[256] =", ";"): + targets = ["_unknown_opcode"] * 256 + for name, op in self.opmap.items(): + if op < 256: + targets[op] = f"TARGET_{name}" + f.write(",\n".join([f" &&{s}" for s in targets])) + + def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> None: """Write instruction metadata to output file.""" @@ -378,12 +522,46 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No ): self.write_uop_items(lambda name, counter: f'[{name}] = "{name}",') + with self.metadata_item( + f"const char *const _PyOpcode_OpName[{1 + max(self.opmap.values())}]", "=", ";" + ): + for name in self.opmap: + self.out.emit(f'[{name}] = "{name}",') + + deoptcodes = {} + for name, op in self.opmap.items(): + if op < 256: + deoptcodes[name] = name + for name, family in self.families.items(): + for m in family.members: + deoptcodes[m] = name + # special case: + deoptcodes['BINARY_OP_INPLACE_ADD_UNICODE'] = 'BINARY_OP' + + with self.metadata_item( + f"const uint8_t _PyOpcode_Deopt[256]", "=", ";" + ): + for opt, deopt in sorted(deoptcodes.items()): + self.out.emit(f"[{opt}] = {deopt},") + + self.out.emit("") + self.out.emit("#define EXTRA_CASES \\") + valid_opcodes = set(self.opmap.values()) + with self.out.indent(): + for op in range(256): + if op not in valid_opcodes: + self.out.emit(f"case {op}: \\") + self.out.emit(" ;\n") + with open(pymetadata_filename, "w") as f: # Create formatter self.out = Formatter(f, 0, comment="#") self.write_provenance_header() + # emit specializations + specialized_ops = set() + self.out.emit("") self.out.emit("_specializations = {") for name, family in self.families.items(): @@ -392,6 +570,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No with self.out.indent(): for m in family.members: self.out.emit(f'"{m}",') + specialized_ops.update(family.members) self.out.emit(f"],") self.out.emit("}") @@ -402,14 +581,26 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No '_specializations["BINARY_OP"].append(' '"BINARY_OP_INPLACE_ADD_UNICODE")' ) + specialized_ops.add("BINARY_OP_INPLACE_ADD_UNICODE") - # Make list of specialized instructions + ops = sorted((id, name) for (name, id) in self.opmap.items()) + # emit specialized opmap self.out.emit("") - self.out.emit( - "_specialized_instructions = [" - "opcode for family in _specializations.values() for opcode in family" - "]" - ) + with self.out.block("_specialized_opmap ="): + for op, name in ops: + if name in specialized_ops: + self.out.emit(f"'{name}': {op},") + + # emit opmap + self.out.emit("") + with self.out.block("opmap ="): + for op, name in ops: + if name not in specialized_ops: + self.out.emit(f"'{name}': {op},") + + for name in ['MIN_INSTRUMENTED_OPCODE', 'HAVE_ARGUMENT']: + self.out.emit(f"{name} = {self.markers[name]}") + def write_pseudo_instrs(self) -> None: """Write the IS_PSEUDO_INSTR macro""" @@ -683,6 +874,9 @@ def main(): # These raise OSError if output can't be written a.write_instructions(args.output, args.emit_line_directives) + + a.assign_opcode_ids() + a.write_opcode_ids(args.opcode_ids_h, args.opcode_targets_h) a.write_metadata(args.metadata, args.pymetadata) a.write_executor_instructions(args.executor_cases, args.emit_line_directives) a.write_abstract_interpreter_instructions(args.abstract_interpreter_cases, diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index f798b2f772d08..2d198506fb5c6 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -17,7 +17,7 @@ DEFAULT_DIR = "/tmp/py_stats/" #Create list of all instruction names -specialized = iter(opcode._specialized_instructions) +specialized = iter(opcode._specialized_opmap.keys()) opname = ["<0>"] for name in opcode.opname[1:]: if name.startswith("<"): @@ -244,7 +244,7 @@ def categorized_counts(opcode_stats): specialized = 0 not_specialized = 0 specialized_instructions = { - op for op in opcode._specialized_instructions + op for op in opcode._specialized_opmap.keys() if "__" not in op} for i, opcode_stat in enumerate(opcode_stats): if "execution_count" not in opcode_stat: From webhook-mailer at python.org Wed Aug 16 19:17:26 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 23:17:26 -0000 Subject: [Python-checkins] [3.12] GH-92584: Drop reference to Distutils in ``site.USER_BASE`` (GH-108031) (#108039) Message-ID: https://github.com/python/cpython/commit/305169e795172baf4e65cde20f563cbafa274606 commit: 305169e795172baf4e65cde20f563cbafa274606 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-17T01:17:23+02:00 summary: [3.12] GH-92584: Drop reference to Distutils in ``site.USER_BASE`` (GH-108031) (#108039) GH-92584: Drop reference to Distutils in ``site.USER_BASE`` (GH-108031) Drop reference to Distutils in ``site.USER_BASE`` (cherry picked from commit f2a9dfdee9de381e4adf29a7f1e2aec56580bfda) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/site.rst diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 44f90a3b9e496..ff9e9107f9d16 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -189,7 +189,7 @@ Module contents :func:`getuserbase` hasn't been called yet. Default value is :file:`~/.local` for UNIX and macOS non-framework builds, :file:`~/Library/Python/{X.Y}` for macOS framework builds, and - :file:`{%APPDATA%}\\Python` for Windows. This value is used by Distutils to + :file:`{%APPDATA%}\\Python` for Windows. This value is used to compute the installation directories for scripts, data files, Python modules, etc. for the :ref:`user installation scheme `. See also :envvar:`PYTHONUSERBASE`. From webhook-mailer at python.org Wed Aug 16 19:18:04 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 23:18:04 -0000 Subject: [Python-checkins] [3.12] Remove Sphinx problem matcher to avoid annotating unchanged files (GH-108005) (#108049) Message-ID: https://github.com/python/cpython/commit/6bab8ef3ad4a3cbc1f096e3b81d7d06c1714622b commit: 6bab8ef3ad4a3cbc1f096e3b81d7d06c1714622b branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-17T01:18:01+02:00 summary: [3.12] Remove Sphinx problem matcher to avoid annotating unchanged files (GH-108005) (#108049) Remove Sphinx problem matcher to avoid annotating unchanged files (GH-108005) (cherry picked from commit 0d7f5d3ba3641f8c7d32facbb177bf70ee7520d1) Co-authored-by: Hugo van Kemenade files: D .github/problem-matchers/sphinx.json M .github/workflows/reusable-docs.yml diff --git a/.github/problem-matchers/sphinx.json b/.github/problem-matchers/sphinx.json deleted file mode 100644 index 09848608a7b03..0000000000000 --- a/.github/problem-matchers/sphinx.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "problemMatcher": [ - { - "owner": "sphinx-problem-matcher", - "pattern": [ - { - "regexp": "^(.*):(\\d+):\\s+(\\w*):\\s+(.*)$", - "file": 1, - "line": 2, - "severity": 3, - "message": 4 - } - ] - }, - { - "owner": "sphinx-problem-matcher-loose", - "pattern": [ - { - "_comment": "A bit of a looser pattern, doesn't look for line numbers, just looks for file names relying on them to start with / and end with .rst", - "regexp": "(\/.*\\.rst):\\s+(\\w*):\\s+(.*)$", - "file": 1, - "severity": 2, - "message": 3 - } - ] - }, - { - "owner": "sphinx-problem-matcher-loose-no-severity", - "pattern": [ - { - "_comment": "Looks for file names ending with .rst and line numbers but without severity", - "regexp": "^(.*\\.rst):(\\d+):(.*)$", - "file": 1, - "line": 2, - "message": 3 - } - ] - } - ] -} \ No newline at end of file diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index b39d8cea6421e..77142a64a4b22 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -18,8 +18,6 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout at v3 - - name: Register Sphinx problem matcher - run: echo "::add-matcher::.github/problem-matchers/sphinx.json" - name: 'Set up Python' uses: actions/setup-python at v4 with: @@ -83,8 +81,6 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout at v3 - - name: Register Sphinx problem matcher - run: echo "::add-matcher::.github/problem-matchers/sphinx.json" - uses: actions/cache at v3 with: path: ~/.cache/pip From webhook-mailer at python.org Wed Aug 16 19:18:27 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 23:18:27 -0000 Subject: [Python-checkins] [3.12] gh-91051: fix type watcher test to be robust to existing watcher (GH-107989) (#108053) Message-ID: https://github.com/python/cpython/commit/84a4370e31632fb04590131ed24329c60cf2f356 commit: 84a4370e31632fb04590131ed24329c60cf2f356 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-17T01:18:24+02:00 summary: [3.12] gh-91051: fix type watcher test to be robust to existing watcher (GH-107989) (#108053) gh-91051: fix type watcher test to be robust to existing watcher (GH-107989) (cherry picked from commit fce93c80ae2d792b8ca443b044e28abbf28bb89a) Co-authored-by: Carl Meyer files: M Lib/test/test_capi/test_watchers.py diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 10b76e163bfb2..6b8855ec219d2 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -351,12 +351,10 @@ def test_clear_unassigned_watcher_id(self): self.clear_watcher(1) def test_no_more_ids_available(self): - contexts = [self.watcher() for i in range(self.TYPE_MAX_WATCHERS)] - with ExitStack() as stack: - for ctx in contexts: - stack.enter_context(ctx) - with self.assertRaisesRegex(RuntimeError, r"no more type watcher IDs"): - self.add_watcher() + with self.assertRaisesRegex(RuntimeError, r"no more type watcher IDs"): + with ExitStack() as stack: + for _ in range(self.TYPE_MAX_WATCHERS + 1): + stack.enter_context(self.watcher()) class TestCodeObjectWatchers(unittest.TestCase): From webhook-mailer at python.org Wed Aug 16 19:19:01 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 23:19:01 -0000 Subject: [Python-checkins] [3.12] GH-92584: Remove reference to Distutils in ``cx_Freeze``'s description (GH-108047) (#108057) Message-ID: https://github.com/python/cpython/commit/2a00cf2db8a19533dc7d71276ce6a77f6a103c65 commit: 2a00cf2db8a19533dc7d71276ce6a77f6a103c65 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-17T01:18:57+02:00 summary: [3.12] GH-92584: Remove reference to Distutils in ``cx_Freeze``'s description (GH-108047) (#108057) GH-92584: Remove reference to Distutils in ``cx_Freeze``'s description (GH-108047) Remove reference to Distutils in ``cx_Freeze``'s description (cherry picked from commit 57fcf96e4f21b8955b3ae4b4d70e4b756949712f) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/using/windows.rst diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index b7e4b2b48a601..1c305d983dfa4 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -1246,8 +1246,8 @@ shipped with PyWin32. It is an embeddable IDE with a built-in debugger. cx_Freeze --------- -`cx_Freeze `_ is a ``distutils`` -extension which wraps Python scripts into executable Windows programs +`cx_Freeze `_ +wraps Python scripts into executable Windows programs (:file:`{*}.exe` files). When you have done this, you can distribute your application without requiring your users to install Python. From webhook-mailer at python.org Wed Aug 16 19:19:51 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 23:19:51 -0000 Subject: [Python-checkins] [3.12] gh-106242: Fix path truncation in os.path.normpath (GH-106816) (#107981) Message-ID: https://github.com/python/cpython/commit/ede98958810b76694cf756d305b564cd6adc1a48 commit: ede98958810b76694cf756d305b564cd6adc1a48 branch: 3.12 author: Steve Dower committer: Yhg1s date: 2023-08-17T01:19:48+02:00 summary: [3.12] gh-106242: Fix path truncation in os.path.normpath (GH-106816) (#107981) * gh-106242: Fix path truncation in os.path.normpath (GH-106816) * gh-106242: Minor fixup to avoid compiler warnings --------- Co-authored-by: Finn Womack Co-authored-by: Serhiy Storchaka files: A Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst M Include/internal/pycore_fileutils.h M Lib/test/test_genericpath.py M Modules/posixmodule.c M Python/fileutils.c diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index ef6642d00f1b5..7c2b6ec0bffef 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -252,7 +252,8 @@ extern int _Py_add_relfile(wchar_t *dirname, const wchar_t *relfile, size_t bufsize); extern size_t _Py_find_basename(const wchar_t *filename); -PyAPI_FUNC(wchar_t *) _Py_normpath(wchar_t *path, Py_ssize_t size); +PyAPI_FUNC(wchar_t*) _Py_normpath(wchar_t *path, Py_ssize_t size); +extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *length); // The Windows Games API family does not provide these functions // so provide our own implementations. Remove them in case they get added diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 489044f8090d3..4f311c2d498e9 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -460,6 +460,10 @@ def test_normpath_issue5827(self): for path in ('', '.', '/', '\\', '///foo/.//bar//'): self.assertIsInstance(self.pathmodule.normpath(path), str) + def test_normpath_issue106242(self): + for path in ('\x00', 'foo\x00bar', '\x00\x00', '\x00foo', 'foo\x00'): + self.assertEqual(self.pathmodule.normpath(path), path) + def test_abspath_issue3426(self): # Check that abspath returns unicode when the arg is unicode # with both ASCII and non-ASCII cwds. diff --git a/Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst b/Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst new file mode 100644 index 0000000000000..44237a9f15708 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-14-23-11-11.gh-issue-106242.71HMym.rst @@ -0,0 +1 @@ +Fixes :func:`os.path.normpath` to handle embedded null characters without truncating the path. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 342f393b1f0f9..b9f45c0ce5543 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5275,7 +5275,9 @@ os__path_normpath_impl(PyObject *module, PyObject *path) if (!buffer) { return NULL; } - PyObject *result = PyUnicode_FromWideChar(_Py_normpath(buffer, len), -1); + Py_ssize_t norm_len; + wchar_t *norm_path = _Py_normpath_and_size(buffer, len, &norm_len); + PyObject *result = PyUnicode_FromWideChar(norm_path, norm_len); PyMem_Free(buffer); return result; } diff --git a/Python/fileutils.c b/Python/fileutils.c index f137ee936502c..268ffa3d61a47 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2377,12 +2377,14 @@ _Py_find_basename(const wchar_t *filename) path, which will be within the original buffer. Guaranteed to not make the path longer, and will not fail. 'size' is the length of the path, if known. If -1, the first null character will be assumed - to be the end of the path. */ + to be the end of the path. 'normsize' will be set to contain the + length of the resulting normalized path. */ wchar_t * -_Py_normpath(wchar_t *path, Py_ssize_t size) +_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) { assert(path != NULL); - if (!path[0] || size == 0) { + if ((size < 0 && !path[0]) || size == 0) { + *normsize = 0; return path; } wchar_t *pEnd = size >= 0 ? &path[size] : NULL; @@ -2431,11 +2433,7 @@ _Py_normpath(wchar_t *path, Py_ssize_t size) *p2++ = lastC = *p1; } } - if (sepCount) { - minP2 = p2; // Invalid path - } else { - minP2 = p2 - 1; // Absolute path has SEP at minP2 - } + minP2 = p2 - 1; } #else // Skip past two leading SEPs @@ -2495,13 +2493,28 @@ _Py_normpath(wchar_t *path, Py_ssize_t size) while (--p2 != minP2 && *p2 == SEP) { *p2 = L'\0'; } + } else { + --p2; } + *normsize = p2 - path + 1; #undef SEP_OR_END #undef IS_SEP #undef IS_END return path; } +/* In-place path normalisation. Returns the start of the normalized + path, which will be within the original buffer. Guaranteed to not + make the path longer, and will not fail. 'size' is the length of + the path, if known. If -1, the first null character will be assumed + to be the end of the path. */ +wchar_t * +_Py_normpath(wchar_t *path, Py_ssize_t size) +{ + Py_ssize_t norm_length; + return _Py_normpath_and_size(path, size, &norm_length); +} + /* Get the current directory. buflen is the buffer size in wide characters including the null character. Decode the path from the locale encoding. From webhook-mailer at python.org Wed Aug 16 19:20:27 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 23:20:27 -0000 Subject: [Python-checkins] [3.12] gh-107298: Fix C API Buffer documentation (GH-108011). (#108048) Message-ID: https://github.com/python/cpython/commit/277bf80238c944f4edada9b5f1885e3567f5fc63 commit: 277bf80238c944f4edada9b5f1885e3567f5fc63 branch: 3.12 author: Serhiy Storchaka committer: Yhg1s date: 2023-08-17T01:20:23+02:00 summary: [3.12] gh-107298: Fix C API Buffer documentation (GH-108011). (#108048) (cherry picked from commit c2941cba7a986e6158eebb2a0bf33906dcd78616) Co-authored-by: Victor Stinner files: M Doc/c-api/buffer.rst M Doc/c-api/typeobj.rst M Doc/tools/.nitignore diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index ba391a5279f20..e572815ffd625 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -163,13 +163,6 @@ a buffer, see :c:func:`PyObject_GetBuffer`. and :c:member:`~Py_buffer.suboffsets` MUST be ``NULL``. The maximum number of dimensions is given by :c:macro:`PyBUF_MAX_NDIM`. - .. :c:macro:: PyBUF_MAX_NDIM - - The maximum number of dimensions the memory represents. - Exporters MUST respect this limit, consumers of multi-dimensional - buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions. - Currently set to 64. - .. c:member:: Py_ssize_t *shape An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim` @@ -221,6 +214,17 @@ a buffer, see :c:func:`PyObject_GetBuffer`. freed when the buffer is released. The consumer MUST NOT alter this value. + +Constants: + +.. c:macro:: PyBUF_MAX_NDIM + + The maximum number of dimensions the memory represents. + Exporters MUST respect this limit, consumers of multi-dimensional + buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions. + Currently set to 64. + + .. _buffer-request-types: Buffer request types @@ -444,7 +448,7 @@ Buffer-related functions Send a request to *exporter* to fill in *view* as specified by *flags*. If the exporter cannot provide a buffer of the exact type, it MUST raise - :c:data:`PyExc_BufferError`, set ``view->obj`` to ``NULL`` and + :exc:`BufferError`, set ``view->obj`` to ``NULL`` and return ``-1``. On success, fill in *view*, set ``view->obj`` to a new reference @@ -531,7 +535,7 @@ Buffer-related functions and :c:macro:`PyBUF_WRITABLE` is set in *flags*. On success, set ``view->obj`` to a new reference to *exporter* and - return 0. Otherwise, raise :c:data:`PyExc_BufferError`, set + return 0. Otherwise, raise :exc:`BufferError`, set ``view->obj`` to ``NULL`` and return ``-1``; If this function is used as part of a :ref:`getbufferproc `, diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index faa183e27fcfa..d394ce10504b0 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -2443,7 +2443,7 @@ Buffer Object Structures Except for point (3), an implementation of this function MUST take these steps: - (1) Check if the request can be met. If not, raise :c:data:`PyExc_BufferError`, + (1) Check if the request can be met. If not, raise :exc:`BufferError`, set :c:expr:`view->obj` to ``NULL`` and return ``-1``. (2) Fill in the requested fields. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 8e9ac17135ebe..9a0a6c2561d12 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -4,7 +4,6 @@ # to help avoid merge conflicts. Doc/c-api/arg.rst -Doc/c-api/buffer.rst Doc/c-api/datetime.rst Doc/c-api/descriptor.rst Doc/c-api/dict.rst From webhook-mailer at python.org Wed Aug 16 19:20:52 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 23:20:52 -0000 Subject: [Python-checkins] [3.12] GH-92584: Remove references to Distutils in ``PYTHONUSERBASE`` (GH-108040) (#108060) Message-ID: https://github.com/python/cpython/commit/589bc198d01aa8c128ad403cb0c9399689047a82 commit: 589bc198d01aa8c128ad403cb0c9399689047a82 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-17T01:20:49+02:00 summary: [3.12] GH-92584: Remove references to Distutils in ``PYTHONUSERBASE`` (GH-108040) (#108060) GH-92584: Remove references to Distutils in ``PYTHONUSERBASE`` (GH-108040) Remove references to Distutils in ``PYTHONUSERBASE`` (cherry picked from commit 636ca313b2f7ce09a311889995778dccae8ebe40) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/using/cmdline.rst M Misc/python.man diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 4bf67eb439ec6..23c89400f152b 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -811,8 +811,8 @@ conflict. Defines the :data:`user base directory `, which is used to compute the path of the :data:`user site-packages directory ` - and :ref:`Distutils installation paths ` for - ``python setup.py install --user``. + and :ref:`installation paths ` for + ``python -m pip install --user``. .. seealso:: diff --git a/Misc/python.man b/Misc/python.man index bf7cf767d164a..9f89c94adf502 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -592,8 +592,8 @@ works on Mac OS X. .IP PYTHONUSERBASE Defines the user base directory, which is used to compute the path of the user .IR site-packages -directory and Distutils installation paths for -.IR "python setup\.py install \-\-user" . +directory and installation paths for +.IR "python \-m pip install \-\-user" . .IP PYTHONPROFILEIMPORTTIME If this environment variable is set to a non-empty string, Python will show how long each import takes. This is exactly equivalent to setting From webhook-mailer at python.org Wed Aug 16 19:21:11 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 23:21:11 -0000 Subject: [Python-checkins] [3.12] GH-92584: Redirect macOS package installation to the PPUG (GH-108044) (#108058) Message-ID: https://github.com/python/cpython/commit/2cbb452b10a1adf3a564f91949815170446523f9 commit: 2cbb452b10a1adf3a564f91949815170446523f9 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-17T01:21:08+02:00 summary: [3.12] GH-92584: Redirect macOS package installation to the PPUG (GH-108044) (#108058) GH-92584: Redirect macOS package installation to the PPUG (GH-108044) (cherry picked from commit 902864256cb261428ae9682ca0ffddd597e1f894) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/using/mac.rst diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 6517827286216..eb1413af2cbc3 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -125,13 +125,9 @@ http://www.hashcollision.org/hkn/python/idle_intro/index.html. Installing Additional Python Packages ===================================== -There are several methods to install additional Python packages: +This section has moved to the `Python Packaging User Guide`_. -* Packages can be installed via the standard Python distutils mode (``python - setup.py install``). - -* Many packages can also be installed via the :program:`setuptools` extension - or :program:`pip` wrapper, see https://pip.pypa.io/. +.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/installing-packages/ GUI Programming on the Mac From webhook-mailer at python.org Wed Aug 16 19:26:47 2023 From: webhook-mailer at python.org (gvanrossum) Date: Wed, 16 Aug 2023 23:26:47 -0000 Subject: [Python-checkins] gh-106581: Split `CALL_PY_EXACT_ARGS` into uops (#107760) Message-ID: https://github.com/python/cpython/commit/dc8fdf5fd5f572e87152883f15f35c354fa4b4bf commit: dc8fdf5fd5f572e87152883f15f35c354fa4b4bf branch: main author: Guido van Rossum committer: gvanrossum date: 2023-08-16T16:26:43-07:00 summary: gh-106581: Split `CALL_PY_EXACT_ARGS` into uops (#107760) * Split `CALL_PY_EXACT_ARGS` into uops This is only the first step for doing `CALL` in Tier 2. The next step involves tracing into the called code object and back. After that we'll have to do the remaining `CALL` specialization. Finally we'll have to deal with `KW_NAMES`. Note: this moves setting `frame->return_offset` directly in front of `DISPATCH_INLINED()`, to make it easier to move it into `_PUSH_FRAME`. files: M Include/internal/pycore_opcode_metadata.h M Lib/test/test_capi/test_misc.py M Python/abstract_interp_cases.c.h M Python/bytecodes.c M Python/ceval.c M Python/ceval_macros.h M Python/executor.c M Python/executor_cases.c.h M Python/generated_cases.c.h M Python/optimizer.c M Tools/cases_generator/flags.py M Tools/cases_generator/generate_cases.py M Tools/cases_generator/instructions.py M Tools/cases_generator/stacking.py diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 7dbd78b32a895..afe8aa172b703 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -52,10 +52,16 @@ #define _ITER_CHECK_RANGE 328 #define _IS_ITER_EXHAUSTED_RANGE 329 #define _ITER_NEXT_RANGE 330 -#define _POP_JUMP_IF_FALSE 331 -#define _POP_JUMP_IF_TRUE 332 -#define JUMP_TO_TOP 333 -#define INSERT 334 +#define _CHECK_PEP_523 331 +#define _CHECK_FUNCTION_EXACT_ARGS 332 +#define _CHECK_STACK_SPACE 333 +#define _INIT_CALL_PY_EXACT_ARGS 334 +#define _PUSH_FRAME 335 +#define _POP_JUMP_IF_FALSE 336 +#define _POP_JUMP_IF_TRUE 337 +#define JUMP_TO_TOP 338 +#define SAVE_CURRENT_IP 339 +#define INSERT 340 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -473,6 +479,16 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return oparg + 2; case CALL_BOUND_METHOD_EXACT_ARGS: return oparg + 2; + case _CHECK_PEP_523: + return 0; + case _CHECK_FUNCTION_EXACT_ARGS: + return oparg + 2; + case _CHECK_STACK_SPACE: + return oparg + 2; + case _INIT_CALL_PY_EXACT_ARGS: + return oparg + 2; + case _PUSH_FRAME: + return 1; case CALL_PY_EXACT_ARGS: return oparg + 2; case CALL_PY_WITH_DEFAULTS: @@ -561,6 +577,8 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 0; case SAVE_IP: return 0; + case SAVE_CURRENT_IP: + return 0; case EXIT_TRACE: return 0; case INSERT: @@ -987,6 +1005,16 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case CALL_BOUND_METHOD_EXACT_ARGS: return 1; + case _CHECK_PEP_523: + return 0; + case _CHECK_FUNCTION_EXACT_ARGS: + return oparg + 2; + case _CHECK_STACK_SPACE: + return oparg + 2; + case _INIT_CALL_PY_EXACT_ARGS: + return 1; + case _PUSH_FRAME: + return 1; case CALL_PY_EXACT_ARGS: return 1; case CALL_PY_WITH_DEFAULTS: @@ -1075,6 +1103,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case SAVE_IP: return 0; + case SAVE_CURRENT_IP: + return 0; case EXIT_TRACE: return 0; case INSERT: @@ -1088,6 +1118,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, + INSTR_FMT_IBC0, INSTR_FMT_IBC00, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, @@ -1132,6 +1163,7 @@ struct opcode_macro_expansion { #define OPARG_CACHE_4 4 #define OPARG_TOP 5 #define OPARG_BOTTOM 6 +#define OPARG_SAVE_IP 7 #define OPCODE_METADATA_FMT(OP) (_PyOpcode_opcode_metadata[(OP)].instr_format) #define SAME_OPCODE_METADATA(OP1, OP2) \ @@ -1474,6 +1506,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { GET_YIELD_FROM_ITER, 0, 0 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { WITH_EXCEPT_START, 0, 0 } } }, [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { PUSH_EXC_INFO, 0, 0 } } }, + [CALL_PY_EXACT_ARGS] = { .nuops = 7, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { SAVE_IP, 7, 3 }, { SAVE_CURRENT_IP, 0, 0 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_NO_KW_TYPE_1] = { .nuops = 1, .uops = { { CALL_NO_KW_TYPE_1, 0, 0 } } }, [CALL_NO_KW_STR_1] = { .nuops = 1, .uops = { { CALL_NO_KW_STR_1, 0, 0 } } }, [CALL_NO_KW_TUPLE_1] = { .nuops = 1, .uops = { { CALL_NO_KW_TUPLE_1, 0, 0 } } }, @@ -1531,9 +1564,15 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", [_IS_ITER_EXHAUSTED_RANGE] = "_IS_ITER_EXHAUSTED_RANGE", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", + [_CHECK_PEP_523] = "_CHECK_PEP_523", + [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", + [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", + [_PUSH_FRAME] = "_PUSH_FRAME", [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", [JUMP_TO_TOP] = "JUMP_TO_TOP", + [SAVE_CURRENT_IP] = "SAVE_CURRENT_IP", [INSERT] = "INSERT", }; #endif // NEED_OPCODE_METADATA diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index c81212202d9ef..3dfbfdc26e741 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2618,6 +2618,23 @@ def testfunc(it): with self.assertRaises(StopIteration): next(it) + def test_call_py_exact_args(self): + def testfunc(n): + def dummy(x): + return x+1 + for i in range(n): + dummy(i) + + opt = _testinternalcapi.get_uop_optimizer() + with temporary_optimizer(opt): + testfunc(10) + + ex = get_first_executor(testfunc) + self.assertIsNotNone(ex) + uops = {opname for opname, _, _ in ex} + self.assertIn("_PUSH_FRAME", uops) + + if __name__ == "__main__": unittest.main() diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 6bfcf534646b1..eef071119bcd8 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -612,6 +612,30 @@ break; } + case _CHECK_PEP_523: { + break; + } + + case _CHECK_FUNCTION_EXACT_ARGS: { + break; + } + + case _CHECK_STACK_SPACE: { + break; + } + + case _INIT_CALL_PY_EXACT_ARGS: { + STACK_SHRINK(oparg); + STACK_SHRINK(1); + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + + case _PUSH_FRAME: { + PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); + break; + } + case CALL_NO_KW_TYPE_1: { STACK_SHRINK(oparg); STACK_SHRINK(1); @@ -751,6 +775,10 @@ break; } + case SAVE_CURRENT_IP: { + break; + } + case EXIT_TRACE: { break; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2a5ad2c942fb3..ae2923c65b330 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -956,13 +956,13 @@ dummy_func( { PyGenObject *gen = (PyGenObject *)receiver; _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->return_offset = oparg; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); + frame->return_offset = oparg; DISPATCH_INLINED(gen_frame); } if (Py_IsNone(v) && PyIter_Check(receiver)) { @@ -995,13 +995,13 @@ dummy_func( DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); STAT_INC(SEND, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->return_offset = oparg; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); + frame->return_offset = oparg; DISPATCH_INLINED(gen_frame); } @@ -2587,7 +2587,6 @@ dummy_func( DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->return_offset = oparg; _PyFrame_StackPush(gen_frame, Py_None); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; @@ -2595,6 +2594,7 @@ dummy_func( SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + frame->return_offset = oparg; DISPATCH_INLINED(gen_frame); } @@ -2949,32 +2949,72 @@ dummy_func( GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); } - inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { - ASSERT_KWNAMES_IS_NULL(); + op(_CHECK_PEP_523, (--)) { DEOPT_IF(tstate->interp->eval_frame, CALL); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } + } + + op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != argcount, CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + } + + op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + } + + op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); for (int i = 0; i < argcount; i++) { new_frame->localsplus[i] = args[i]; } - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); + } + + // The 'unused' output effect represents the return value + // (which will be pushed when the frame returns). + // It is needed so CALL_PY_EXACT_ARGS matches its family. + op(_PUSH_FRAME, (new_frame: _PyInterpreterFrame* -- unused)) { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. frame->return_offset = 0; - DISPATCH_INLINED(new_frame); + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + #if TIER_ONE + frame = cframe.current_frame = new_frame; + goto start_frame; + #endif + #if TIER_TWO + frame = tstate->cframe->current_frame = new_frame; + ERROR_IF(_Py_EnterRecursivePy(tstate), exit_unwind); + stack_pointer = _PyFrame_GetStackPointer(frame); + ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + #endif } + macro(CALL_PY_EXACT_ARGS) = + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_FUNCTION_EXACT_ARGS + + _CHECK_STACK_SPACE + + _INIT_CALL_PY_EXACT_ARGS + + SAVE_IP + // Tier 2 only; special-cased oparg + SAVE_CURRENT_IP + // Sets frame->prev_instr + _PUSH_FRAME; + inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { ASSERT_KWNAMES_IS_NULL(); DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -3735,6 +3775,16 @@ dummy_func( frame->prev_instr = ip_offset + oparg; } + op(SAVE_CURRENT_IP, (--)) { + #if TIER_ONE + frame->prev_instr = next_instr - 1; + #endif + #if TIER_TWO + // Relies on a preceding SAVE_IP + frame->prev_instr--; + #endif + } + op(EXIT_TRACE, (--)) { frame->prev_instr--; // Back up to just before destination _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/ceval.c b/Python/ceval.c index b966399a342d0..26e741ed7c754 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -602,11 +602,6 @@ int _Py_CheckRecursiveCallPy( return 0; } -static inline int _Py_EnterRecursivePy(PyThreadState *tstate) { - return (tstate->py_recursion_remaining-- <= 0) && - _Py_CheckRecursiveCallPy(tstate); -} - static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { tstate->py_recursion_remaining++; @@ -770,6 +765,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif { +#define TIER_ONE 1 #include "generated_cases.c.h" /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c, diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 8dc8b75448585..5e2db1e0b394e 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -364,3 +364,8 @@ static const convertion_func_ptr CONVERSION_FUNCTIONS[4] = { #else #define _Py_atomic_load_relaxed_int32(ATOMIC_VAL) _Py_atomic_load_relaxed(ATOMIC_VAL) #endif + +static inline int _Py_EnterRecursivePy(PyThreadState *tstate) { + return (tstate->py_recursion_remaining-- <= 0) && + _Py_CheckRecursiveCallPy(tstate); +} diff --git a/Python/executor.c b/Python/executor.c index 4a18618c0c6c0..5a571e6da4673 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -81,6 +81,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject OBJECT_STAT_INC(optimization_uops_executed); switch (opcode) { +#define TIER_TWO 2 #include "executor_cases.c.h" default: @@ -106,6 +107,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject pop_2_error: STACK_SHRINK(1); pop_1_error: +pop_1_exit_unwind: STACK_SHRINK(1); error: // On ERROR_IF we return NULL as the frame. diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 85d27777423ab..b3dd313353056 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -103,7 +103,6 @@ } case TO_BOOL: { - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -363,7 +362,6 @@ } case BINARY_SUBSCR: { - static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; PyObject *res; @@ -557,7 +555,6 @@ } case STORE_SUBSCR: { - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); PyObject *sub; PyObject *container; PyObject *v; @@ -862,7 +859,6 @@ } case UNPACK_SEQUENCE: { - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; seq = stack_pointer[-1]; #if ENABLE_SPECIALIZATION @@ -950,7 +946,6 @@ } case STORE_ATTR: { - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *v; owner = stack_pointer[-1]; @@ -1061,7 +1056,6 @@ } case LOAD_GLOBAL: { - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; #if ENABLE_SPECIALIZATION @@ -1554,7 +1548,6 @@ } case LOAD_ATTR: { - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); PyObject *owner; PyObject *attr; PyObject *self_or_null = NULL; @@ -1650,7 +1643,6 @@ } case COMPARE_OP: { - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); PyObject *right; PyObject *left; PyObject *res; @@ -2155,6 +2147,84 @@ break; } + case _CHECK_PEP_523: { + DEOPT_IF(tstate->interp->eval_frame, CALL); + break; + } + + case _CHECK_FUNCTION_EXACT_ARGS: { + PyObject *self_or_null; + PyObject *callable; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)operand; + ASSERT_KWNAMES_IS_NULL(); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + break; + } + + case _CHECK_STACK_SPACE: { + PyObject *callable; + callable = stack_pointer[-2 - oparg]; + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + break; + } + + case _INIT_CALL_PY_EXACT_ARGS: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + args = stack_pointer - oparg; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } + STACK_SHRINK(oparg); + STACK_SHRINK(1); + stack_pointer[-1] = (PyObject *)new_frame; + break; + } + + case _PUSH_FRAME: { + _PyInterpreterFrame *new_frame; + new_frame = (_PyInterpreterFrame *)stack_pointer[-1]; + STACK_SHRINK(1); + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + frame->return_offset = 0; + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + #if TIER_ONE + frame = cframe.current_frame = new_frame; + goto start_frame; + #endif + #if TIER_TWO + frame = tstate->cframe->current_frame = new_frame; + if (_Py_EnterRecursivePy(tstate)) goto pop_1_exit_unwind; + stack_pointer = _PyFrame_GetStackPointer(frame); + ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + #endif + break; + } + case CALL_NO_KW_TYPE_1: { PyObject **args; PyObject *null; @@ -2656,7 +2726,6 @@ } case BINARY_OP: { - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *rhs; PyObject *lhs; PyObject *res; @@ -2726,6 +2795,17 @@ break; } + case SAVE_CURRENT_IP: { + #if TIER_ONE + frame->prev_instr = next_instr - 1; + #endif + #if TIER_TWO + // Relies on a preceding SAVE_IP + frame->prev_instr--; + #endif + break; + } + case EXIT_TRACE: { frame->prev_instr--; // Back up to just before destination _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2661a39e047c4..11d560a6e77ad 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1191,13 +1191,13 @@ { PyGenObject *gen = (PyGenObject *)receiver; _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->return_offset = oparg; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); + frame->return_offset = oparg; DISPATCH_INLINED(gen_frame); } if (Py_IsNone(v) && PyIter_Check(receiver)) { @@ -1237,13 +1237,13 @@ DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); STAT_INC(SEND, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->return_offset = oparg; STACK_SHRINK(1); _PyFrame_StackPush(gen_frame, v); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; SKIP_OVER(INLINE_CACHE_ENTRIES_SEND); + frame->return_offset = oparg; DISPATCH_INLINED(gen_frame); } @@ -3343,7 +3343,6 @@ DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - frame->return_offset = oparg; _PyFrame_StackPush(gen_frame, Py_None); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; @@ -3351,6 +3350,7 @@ SKIP_OVER(INLINE_CACHE_ENTRIES_FOR_ITER); assert(next_instr[oparg].op.code == END_FOR || next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + frame->return_offset = oparg; DISPATCH_INLINED(gen_frame); STACK_GROW(1); } @@ -3764,38 +3764,83 @@ TARGET(CALL_PY_EXACT_ARGS) { PREDICTED(CALL_PY_EXACT_ARGS); - PyObject **args; PyObject *self_or_null; PyObject *callable; - args = stack_pointer - oparg; + PyObject **args; + _PyInterpreterFrame *new_frame; + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_EXACT_ARGS self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&next_instr[1].cache); - ASSERT_KWNAMES_IS_NULL(); - DEOPT_IF(tstate->interp->eval_frame, CALL); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; + { + uint32_t func_version = read_u32(&next_instr[1].cache); + ASSERT_KWNAMES_IS_NULL(); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + } + // _CHECK_STACK_SPACE + callable = stack_pointer[-2 - oparg]; + { + PyFunctionObject *func = (PyFunctionObject *)callable; + PyCodeObject *code = (PyCodeObject *)func->func_code; + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); } - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != argcount, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + // _INIT_CALL_PY_EXACT_ARGS + args = stack_pointer - oparg; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + int argcount = oparg; + if (self_or_null != NULL) { + args--; + argcount++; + } + STAT_INC(CALL, hit); + PyFunctionObject *func = (PyFunctionObject *)callable; + new_frame = _PyFrame_PushUnchecked(tstate, func, argcount); + for (int i = 0; i < argcount; i++) { + new_frame->localsplus[i] = args[i]; + } } - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - SKIP_OVER(INLINE_CACHE_ENTRIES_CALL); - frame->return_offset = 0; - DISPATCH_INLINED(new_frame); + // SAVE_CURRENT_IP + next_instr += 3; + { + #if TIER_ONE + frame->prev_instr = next_instr - 1; + #endif + #if TIER_TWO + // Relies on a preceding SAVE_IP + frame->prev_instr--; + #endif + } + // _PUSH_FRAME STACK_SHRINK(oparg); - STACK_SHRINK(1); + STACK_SHRINK(2); + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + frame->return_offset = 0; + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + #if TIER_ONE + frame = cframe.current_frame = new_frame; + goto start_frame; + #endif + #if TIER_TWO + frame = tstate->cframe->current_frame = new_frame; + if (_Py_EnterRecursivePy(tstate)) goto pop_1_exit_unwind; + stack_pointer = _PyFrame_GetStackPointer(frame); + ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + #endif + } } TARGET(CALL_PY_WITH_DEFAULTS) { diff --git a/Python/optimizer.c b/Python/optimizer.c index d3ac2424038ef..559c4ae987263 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -606,6 +606,10 @@ translate_bytecode_to_trace( case OPARG_BOTTOM: // Second half of super-instr oparg = orig_oparg & 0xF; break; + case OPARG_SAVE_IP: // op==SAVE_IP; oparg=next instr + oparg = INSTR_IP(instr + offset, code); + break; + default: fprintf(stderr, "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", @@ -615,6 +619,11 @@ translate_bytecode_to_trace( Py_FatalError("garbled expansion"); } ADD_TO_TRACE(expansion->uops[i].uop, oparg, operand); + if (expansion->uops[i].uop == _PUSH_FRAME) { + assert(i + 1 == nuops); + ADD_TO_TRACE(SAVE_IP, 0, 0); + goto done; + } } break; } diff --git a/Tools/cases_generator/flags.py b/Tools/cases_generator/flags.py index f7ebdeb0d6567..962f003b194db 100644 --- a/Tools/cases_generator/flags.py +++ b/Tools/cases_generator/flags.py @@ -92,7 +92,7 @@ def variable_used_unspecialized(node: parsing.Node, name: str) -> bool: if text == "#if": if ( i + 1 < len(node.tokens) - and node.tokens[i + 1].text == "ENABLE_SPECIALIZATION" + and node.tokens[i + 1].text in ("ENABLE_SPECIALIZATION", "TIER_ONE") ): skipping = True elif text in ("#else", "#endif"): diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index d991cb4690063..f31b6658d6599 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -25,6 +25,7 @@ PseudoInstruction, StackEffect, OverriddenInstructionPlaceHolder, + TIER_ONE, TIER_TWO, ) import parsing @@ -65,6 +66,7 @@ "OPARG_CACHE_4": 4, "OPARG_TOP": 5, "OPARG_BOTTOM": 6, + "OPARG_SAVE_IP": 7, } INSTR_FMT_PREFIX = "INSTR_FMT_" @@ -501,7 +503,9 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No if instr.kind == "inst" and instr.is_viable_uop(): # Construct a dummy Component -- input/output mappings are not used part = Component(instr, instr.active_caches) - self.write_macro_expansions(instr.name, [part]) + self.write_macro_expansions( + instr.name, [part], instr.cache_offset + ) elif instr.kind == "inst" and variable_used( instr.inst, "oparg1" ): @@ -511,7 +515,9 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.write_super_expansions(instr.name) case parsing.Macro(): mac = self.macro_instrs[thing.name] - self.write_macro_expansions(mac.name, mac.parts) + self.write_macro_expansions( + mac.name, mac.parts, mac.cache_offset + ) case parsing.Pseudo(): pass case _: @@ -630,7 +636,9 @@ def add(name: str) -> None: if instr.kind == "op" and instr.is_viable_uop(): add(instr.name) - def write_macro_expansions(self, name: str, parts: MacroParts) -> None: + def write_macro_expansions( + self, name: str, parts: MacroParts, cache_offset: int + ) -> None: """Write the macro expansions for a macro-instruction.""" # TODO: Refactor to share code with write_cody(), is_viaible_uop(), etc. offset = 0 # Cache effect offset @@ -650,7 +658,10 @@ def write_macro_expansions(self, name: str, parts: MacroParts) -> None: ) return if not part.active_caches: - size, offset = OPARG_SIZES["OPARG_FULL"], 0 + if part.instr.name == "SAVE_IP": + size, offset = OPARG_SIZES["OPARG_SAVE_IP"], cache_offset + else: + size, offset = OPARG_SIZES["OPARG_FULL"], 0 else: # If this assert triggers, is_viable_uops() lied assert len(part.active_caches) == 1, (name, part.instr.name) @@ -753,7 +764,9 @@ def write_instructions( case parsing.Macro(): n_macros += 1 mac = self.macro_instrs[thing.name] - stacking.write_macro_instr(mac, self.out, self.families.get(mac.name)) + stacking.write_macro_instr( + mac, self.out, self.families.get(mac.name) + ) # self.write_macro(self.macro_instrs[thing.name]) case parsing.Pseudo(): pass @@ -789,7 +802,9 @@ def write_executor_instructions( n_instrs += 1 self.out.emit("") with self.out.block(f"case {thing.name}:"): - instr.write(self.out, tier=TIER_TWO) + stacking.write_single_instr( + instr, self.out, tier=TIER_TWO + ) if instr.check_eval_breaker: self.out.emit("CHECK_EVAL_BREAKER();") self.out.emit("break;") @@ -851,8 +866,13 @@ def write_instr(self, instr: Instruction) -> None: with self.out.block(f"TARGET({name})"): if instr.predicted: self.out.emit(f"PREDICTED({name});") - instr.write(self.out) + self.out.static_assert_family_size( + instr.name, instr.family, instr.cache_offset + ) + stacking.write_single_instr(instr, self.out, tier=TIER_ONE) if not instr.always_exits: + if instr.cache_offset: + self.out.emit(f"next_instr += {instr.cache_offset};") if instr.check_eval_breaker: self.out.emit("CHECK_EVAL_BREAKER();") self.out.emit(f"DISPATCH();") diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index a505df08fa265..9143ae0db7be8 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -59,7 +59,7 @@ class Instruction: block_line: int # First line of block in original code # Computed by constructor - always_exits: bool + always_exits: str # If the block always exits, its last line; else "" has_deopt: bool cache_offset: int cache_effects: list[parsing.CacheEffect] @@ -120,13 +120,13 @@ def __init__(self, inst: parsing.InstDef): def is_viable_uop(self) -> bool: """Whether this instruction is viable as a uop.""" dprint: typing.Callable[..., None] = lambda *args, **kwargs: None - # if self.name.startswith("CALL"): - # dprint = print + if "FRAME" in self.name: + dprint = print if self.name == "EXIT_TRACE": return True # This has 'return frame' but it's okay if self.always_exits: - dprint(f"Skipping {self.name} because it always exits") + dprint(f"Skipping {self.name} because it always exits: {self.always_exits}") return False if len(self.active_caches) > 1: # print(f"Skipping {self.name} because it has >1 cache entries") @@ -140,23 +140,6 @@ def is_viable_uop(self) -> bool: res = False return res - def write(self, out: Formatter, tier: Tiers = TIER_ONE) -> None: - """Write one instruction, sans prologue and epilogue.""" - - # Write a static assertion that a family's cache size is correct - out.static_assert_family_size(self.name, self.family, self.cache_offset) - - # Write input stack effect variable declarations and initializations - stacking.write_single_instr(self, out, tier) - - # Skip the rest if the block always exits - if self.always_exits: - return - - # Write cache effect - if tier == TIER_ONE and self.cache_offset: - out.emit(f"next_instr += {self.cache_offset};") - def write_body( self, out: Formatter, @@ -341,16 +324,16 @@ def extract_block_text(block: parsing.Block) -> tuple[list[str], bool, int]: return blocklines, check_eval_breaker, block_line -def always_exits(lines: list[str]) -> bool: +def always_exits(lines: list[str]) -> str: """Determine whether a block always ends in a return/goto/etc.""" if not lines: - return False + return "" line = lines[-1].rstrip() # Indent must match exactly (TODO: Do something better) if line[:12] != " " * 12: - return False + return "" line = line[12:] - return line.startswith( + if line.startswith( ( "goto ", "return ", @@ -359,4 +342,6 @@ def always_exits(lines: list[str]) -> bool: "Py_UNREACHABLE()", "ERROR_IF(true, ", ) - ) + ): + return line + return "" diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 31a21e026cb49..8361eb99f88a7 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -1,6 +1,7 @@ import dataclasses import typing +from flags import variable_used_unspecialized from formatting import ( Formatter, UNUSED, @@ -146,6 +147,8 @@ class EffectManager: # Track offsets from stack pointer min_offset: StackOffset final_offset: StackOffset + # Link to previous manager + pred: "EffectManager | None" = None def __init__( self, @@ -167,7 +170,8 @@ def __init__( self.pokes.append(StackItem(offset=self.final_offset.clone(), effect=eff)) self.final_offset.higher(eff) - if pred: + self.pred = pred + while pred: # Replace push(x) + pop(y) with copy(x, y). # Check that the sources and destinations are disjoint. sources: set[str] = set() @@ -192,6 +196,11 @@ def __init__( sources, destinations, ) + # See if we can get more copies of a earlier predecessor. + if self.peeks and not pred.pokes and not pred.peeks: + pred = pred.pred + else: + pred = None # Break def adjust_deeper(self, eff: StackEffect) -> None: for peek in self.peeks: @@ -295,6 +304,7 @@ def write_single_instr( [Component(instr, instr.active_caches)], out, tier, + 0, ) except AssertionError as err: raise AssertionError(f"Error writing instruction {instr.name}") from err @@ -303,37 +313,32 @@ def write_single_instr( def write_macro_instr( mac: MacroInstruction, out: Formatter, family: Family | None ) -> None: - parts = [part for part in mac.parts if isinstance(part, Component)] - - cache_adjust = 0 - for part in mac.parts: - match part: - case CacheEffect(size=size): - cache_adjust += size - case Component(instr=instr): - cache_adjust += instr.cache_offset - case _: - typing.assert_never(part) - + parts = [ + part + for part in mac.parts + if isinstance(part, Component) and part.instr.name != "SAVE_IP" + ] out.emit("") with out.block(f"TARGET({mac.name})"): if mac.predicted: out.emit(f"PREDICTED({mac.name});") - out.static_assert_family_size(mac.name, family, cache_adjust) + out.static_assert_family_size(mac.name, family, mac.cache_offset) try: - write_components(parts, out, TIER_ONE) + next_instr_is_set = write_components(parts, out, TIER_ONE, mac.cache_offset) except AssertionError as err: raise AssertionError(f"Error writing macro {mac.name}") from err - if cache_adjust: - out.emit(f"next_instr += {cache_adjust};") - out.emit("DISPATCH();") + if not parts[-1].instr.always_exits and not next_instr_is_set: + if mac.cache_offset: + out.emit(f"next_instr += {mac.cache_offset};") + out.emit("DISPATCH();") def write_components( parts: list[Component], out: Formatter, tier: Tiers, -) -> None: + cache_offset: int, +) -> bool: managers = get_managers(parts) all_vars: dict[str, StackEffect] = {} @@ -354,6 +359,7 @@ def write_components( for name, eff in all_vars.items(): out.declare(eff, None) + next_instr_is_set = False for mgr in managers: if len(parts) > 1: out.emit(f"// {mgr.instr.name}") @@ -374,13 +380,25 @@ def write_components( poke.as_stack_effect(lax=True), ) + if mgr.instr.name == "_PUSH_FRAME": + # Adjust stack to min_offset (input effects materialized) + out.stack_adjust(mgr.min_offset.deep, mgr.min_offset.high) + # Use clone() since adjust_inverse() mutates final_offset + mgr.adjust_inverse(mgr.final_offset.clone()) + + if mgr.instr.name == "SAVE_CURRENT_IP": + next_instr_is_set = True + if cache_offset: + out.emit(f"next_instr += {cache_offset};") + if len(parts) == 1: mgr.instr.write_body(out, 0, mgr.active_caches, tier) else: with out.block(""): mgr.instr.write_body(out, -4, mgr.active_caches, tier) - if mgr is managers[-1]: + if mgr is managers[-1] and not next_instr_is_set: + # TODO: Explain why this adjustment is needed. out.stack_adjust(mgr.final_offset.deep, mgr.final_offset.high) # Use clone() since adjust_inverse() mutates final_offset mgr.adjust_inverse(mgr.final_offset.clone()) @@ -392,6 +410,8 @@ def write_components( poke.effect, ) + return next_instr_is_set + def write_single_instr_for_abstract_interp( instr: Instruction, out: Formatter From webhook-mailer at python.org Wed Aug 16 19:30:11 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 16 Aug 2023 23:30:11 -0000 Subject: [Python-checkins] [3.12] GH-92584: Remove references to Distutils in configure.rst (GH-108043) (#108063) Message-ID: https://github.com/python/cpython/commit/931df0a47cfa5f6ed321c5bab6f0884bd524d1a0 commit: 931df0a47cfa5f6ed321c5bab6f0884bd524d1a0 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-17T01:30:08+02:00 summary: [3.12] GH-92584: Remove references to Distutils in configure.rst (GH-108043) (#108063) GH-92584: Remove references to Distutils in configure.rst (GH-108043) Remove references to Distutils in configure.rst (cherry picked from commit e88eb3775ecdcb3af6c6d694a935b7fa5f41e5ce) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/using/configure.rst diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 924e73dc54da2..441d346a1a38a 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -686,7 +686,6 @@ Main files of the build system * :file:`pyconfig.h` (created by :file:`configure`); * :file:`Modules/Setup`: C extensions built by the Makefile using :file:`Module/makesetup` shell script; -* :file:`setup.py`: C extensions built using the ``setuptools`` package. Main build steps ---------------- @@ -695,8 +694,7 @@ Main build steps * A static ``libpython`` library (``.a``) is created from objects files. * ``python.o`` and the static ``libpython`` library are linked into the final ``python`` program. -* C extensions are built by the Makefile (see :file:`Modules/Setup`) - and ``python setup.py build``. +* C extensions are built by the Makefile (see :file:`Modules/Setup`). Main Makefile targets --------------------- @@ -748,9 +746,6 @@ Example on Linux x86-64:: At the beginning of the files, C extensions are built as built-in modules. Extensions defined after the ``*shared*`` marker are built as dynamic libraries. -The :file:`setup.py` script only builds C extensions as shared libraries using -the :mod:`distutils` module. - The :c:macro:`PyAPI_FUNC()`, :c:macro:`PyAPI_API()` and :c:macro:`PyMODINIT_FUNC()` macros of :file:`Include/pyport.h` are defined differently depending if the ``Py_BUILD_CORE_MODULE`` macro is defined: @@ -784,7 +779,7 @@ Preprocessor flags headers in a nonstandard directory ````. Both :envvar:`CPPFLAGS` and :envvar:`LDFLAGS` need to contain the shell's - value for setup.py to be able to build extension modules using the + value to be able to build extension modules using the directories specified in the environment variables. .. envvar:: BASECPPFLAGS @@ -821,8 +816,8 @@ Compiler flags .. envvar:: CFLAGS_NODIST :envvar:`CFLAGS_NODIST` is used for building the interpreter and stdlib C - extensions. Use it when a compiler flag should *not* be part of the - distutils :envvar:`CFLAGS` once Python is installed (:issue:`21121`). + extensions. Use it when a compiler flag should *not* be part of + :envvar:`CFLAGS` once Python is installed (:gh:`65320`). In particular, :envvar:`CFLAGS` should not contain: @@ -952,7 +947,7 @@ Linker flags :envvar:`LDFLAGS_NODIST` is used in the same manner as :envvar:`CFLAGS_NODIST`. Use it when a linker flag should *not* be part of - the distutils :envvar:`LDFLAGS` once Python is installed (:issue:`35257`). + :envvar:`LDFLAGS` once Python is installed (:gh:`65320`). In particular, :envvar:`LDFLAGS` should not contain: @@ -974,7 +969,7 @@ Linker flags directory ````. Both :envvar:`CPPFLAGS` and :envvar:`LDFLAGS` need to contain the shell's - value for setup.py to be able to build extension modules using the + value to be able to build extension modules using the directories specified in the environment variables. .. envvar:: LIBS From webhook-mailer at python.org Thu Aug 17 02:44:09 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Thu, 17 Aug 2023 06:44:09 -0000 Subject: [Python-checkins] gh-107298: Add standard exceptions and warnings in the nitpick_ignore list (GH-108029) Message-ID: https://github.com/python/cpython/commit/c9d83f93d804b80ee14480466ebee63a6f97dac2 commit: c9d83f93d804b80ee14480466ebee63a6f97dac2 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-17T09:44:05+03:00 summary: gh-107298: Add standard exceptions and warnings in the nitpick_ignore list (GH-108029) files: M Doc/conf.py diff --git a/Doc/conf.py b/Doc/conf.py index 19e05e1aa8fe1..19eab1a9340e4 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -157,6 +157,77 @@ ('envvar', 'USER'), ('envvar', 'USERNAME'), ('envvar', 'USERPROFILE'), +] + +# Temporary undocumented names. +# In future this list must be empty. +nitpick_ignore += [ + # C API: Standard Python exception classes + ('c:data', 'PyExc_ArithmeticError'), + ('c:data', 'PyExc_AssertionError'), + ('c:data', 'PyExc_AttributeError'), + ('c:data', 'PyExc_BaseException'), + ('c:data', 'PyExc_BlockingIOError'), + ('c:data', 'PyExc_BrokenPipeError'), + ('c:data', 'PyExc_BufferError'), + ('c:data', 'PyExc_ChildProcessError'), + ('c:data', 'PyExc_ConnectionAbortedError'), + ('c:data', 'PyExc_ConnectionError'), + ('c:data', 'PyExc_ConnectionRefusedError'), + ('c:data', 'PyExc_ConnectionResetError'), + ('c:data', 'PyExc_EOFError'), + ('c:data', 'PyExc_Exception'), + ('c:data', 'PyExc_FileExistsError'), + ('c:data', 'PyExc_FileNotFoundError'), + ('c:data', 'PyExc_FloatingPointError'), + ('c:data', 'PyExc_GeneratorExit'), + ('c:data', 'PyExc_ImportError'), + ('c:data', 'PyExc_IndentationError'), + ('c:data', 'PyExc_IndexError'), + ('c:data', 'PyExc_InterruptedError'), + ('c:data', 'PyExc_IsADirectoryError'), + ('c:data', 'PyExc_KeyboardInterrupt'), + ('c:data', 'PyExc_KeyError'), + ('c:data', 'PyExc_LookupError'), + ('c:data', 'PyExc_MemoryError'), + ('c:data', 'PyExc_ModuleNotFoundError'), + ('c:data', 'PyExc_NameError'), + ('c:data', 'PyExc_NotADirectoryError'), + ('c:data', 'PyExc_NotImplementedError'), + ('c:data', 'PyExc_OSError'), + ('c:data', 'PyExc_OverflowError'), + ('c:data', 'PyExc_PermissionError'), + ('c:data', 'PyExc_ProcessLookupError'), + ('c:data', 'PyExc_RecursionError'), + ('c:data', 'PyExc_ReferenceError'), + ('c:data', 'PyExc_RuntimeError'), + ('c:data', 'PyExc_StopAsyncIteration'), + ('c:data', 'PyExc_StopIteration'), + ('c:data', 'PyExc_SyntaxError'), + ('c:data', 'PyExc_SystemError'), + ('c:data', 'PyExc_SystemExit'), + ('c:data', 'PyExc_TabError'), + ('c:data', 'PyExc_TimeoutError'), + ('c:data', 'PyExc_TypeError'), + ('c:data', 'PyExc_UnboundLocalError'), + ('c:data', 'PyExc_UnicodeDecodeError'), + ('c:data', 'PyExc_UnicodeEncodeError'), + ('c:data', 'PyExc_UnicodeError'), + ('c:data', 'PyExc_UnicodeTranslateError'), + ('c:data', 'PyExc_ValueError'), + ('c:data', 'PyExc_ZeroDivisionError'), + # C API: Standard Python warning classes + ('c:data', 'PyExc_BytesWarning'), + ('c:data', 'PyExc_DeprecationWarning'), + ('c:data', 'PyExc_FutureWarning'), + ('c:data', 'PyExc_ImportWarning'), + ('c:data', 'PyExc_PendingDeprecationWarning'), + ('c:data', 'PyExc_ResourceWarning'), + ('c:data', 'PyExc_RuntimeWarning'), + ('c:data', 'PyExc_SyntaxWarning'), + ('c:data', 'PyExc_UnicodeWarning'), + ('c:data', 'PyExc_UserWarning'), + ('c:data', 'PyExc_Warning'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. From webhook-mailer at python.org Thu Aug 17 02:45:53 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Thu, 17 Aug 2023 06:45:53 -0000 Subject: [Python-checkins] gh-105539: Explict resource management for connection objects in sqlite3 tests (#108017) Message-ID: https://github.com/python/cpython/commit/1344cfac43a1920c596b0e8718ca0567889e697b commit: 1344cfac43a1920c596b0e8718ca0567889e697b branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-17T08:45:48+02:00 summary: gh-105539: Explict resource management for connection objects in sqlite3 tests (#108017) - Use memory_database() helper - Move test utility functions to util.py - Add convenience memory database mixin - Add check() helper for closed connection tests files: A Lib/test/test_sqlite3/util.py M Lib/test/test_sqlite3/test_backup.py M Lib/test/test_sqlite3/test_dbapi.py M Lib/test/test_sqlite3/test_dump.py M Lib/test/test_sqlite3/test_factory.py M Lib/test/test_sqlite3/test_hooks.py M Lib/test/test_sqlite3/test_regression.py M Lib/test/test_sqlite3/test_transactions.py M Lib/test/test_sqlite3/test_userfunctions.py diff --git a/Lib/test/test_sqlite3/test_backup.py b/Lib/test/test_sqlite3/test_backup.py index 87ab29c54d65e..4584d976bce0c 100644 --- a/Lib/test/test_sqlite3/test_backup.py +++ b/Lib/test/test_sqlite3/test_backup.py @@ -1,6 +1,8 @@ import sqlite3 as sqlite import unittest +from .util import memory_database + class BackupTests(unittest.TestCase): def setUp(self): @@ -32,32 +34,32 @@ def test_bad_target_same_connection(self): self.cx.backup(self.cx) def test_bad_target_closed_connection(self): - bck = sqlite.connect(':memory:') - bck.close() - with self.assertRaises(sqlite.ProgrammingError): - self.cx.backup(bck) + with memory_database() as bck: + bck.close() + with self.assertRaises(sqlite.ProgrammingError): + self.cx.backup(bck) def test_bad_source_closed_connection(self): - bck = sqlite.connect(':memory:') - source = sqlite.connect(":memory:") - source.close() - with self.assertRaises(sqlite.ProgrammingError): - source.backup(bck) + with memory_database() as bck: + source = sqlite.connect(":memory:") + source.close() + with self.assertRaises(sqlite.ProgrammingError): + source.backup(bck) def test_bad_target_in_transaction(self): - bck = sqlite.connect(':memory:') - bck.execute('CREATE TABLE bar (key INTEGER)') - bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) - with self.assertRaises(sqlite.OperationalError) as cm: - self.cx.backup(bck) + with memory_database() as bck: + bck.execute('CREATE TABLE bar (key INTEGER)') + bck.executemany('INSERT INTO bar (key) VALUES (?)', [(3,), (4,)]) + with self.assertRaises(sqlite.OperationalError) as cm: + self.cx.backup(bck) def test_keyword_only_args(self): with self.assertRaises(TypeError): - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, 1) def test_simple(self): - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck) self.verify_backup(bck) @@ -67,7 +69,7 @@ def test_progress(self): def progress(status, remaining, total): journal.append(status) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=1, progress=progress) self.verify_backup(bck) @@ -81,7 +83,7 @@ def test_progress_all_pages_at_once_1(self): def progress(status, remaining, total): journal.append(remaining) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, progress=progress) self.verify_backup(bck) @@ -94,7 +96,7 @@ def test_progress_all_pages_at_once_2(self): def progress(status, remaining, total): journal.append(remaining) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=-1, progress=progress) self.verify_backup(bck) @@ -103,7 +105,7 @@ def progress(status, remaining, total): def test_non_callable_progress(self): with self.assertRaises(TypeError) as cm: - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=1, progress='bar') self.assertEqual(str(cm.exception), 'progress argument must be a callable') @@ -116,7 +118,7 @@ def progress(status, remaining, total): self.cx.commit() journal.append(remaining) - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, pages=1, progress=progress) self.verify_backup(bck) @@ -140,12 +142,12 @@ def progress(status, remaining, total): self.assertEqual(str(err.exception), 'nearly out of space') def test_database_source_name(self): - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='main') - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='temp') with self.assertRaises(sqlite.OperationalError) as cm: - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='non-existing') self.assertIn("unknown database", str(cm.exception)) @@ -153,7 +155,7 @@ def test_database_source_name(self): self.cx.execute('CREATE TABLE attached_db.foo (key INTEGER)') self.cx.executemany('INSERT INTO attached_db.foo (key) VALUES (?)', [(3,), (4,)]) self.cx.commit() - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, name='attached_db') self.verify_backup(bck) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index c9a9e1353938c..df3c2ea8d1dbd 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -33,26 +33,13 @@ SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess, is_emscripten, is_wasi ) +from test.support import gc_collect from test.support import threading_helper from _testcapi import INT_MAX, ULLONG_MAX from os import SEEK_SET, SEEK_CUR, SEEK_END from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE, unlink, temp_dir, FakePath - -# Helper for temporary memory databases -def memory_database(*args, **kwargs): - cx = sqlite.connect(":memory:", *args, **kwargs) - return contextlib.closing(cx) - - -# Temporarily limit a database connection parameter - at contextlib.contextmanager -def cx_limit(cx, category=sqlite.SQLITE_LIMIT_SQL_LENGTH, limit=128): - try: - _prev = cx.setlimit(category, limit) - yield limit - finally: - cx.setlimit(category, _prev) +from .util import memory_database, cx_limit class ModuleTests(unittest.TestCase): @@ -326,9 +313,9 @@ def test_extended_error_code_on_exception(self): self.assertEqual(exc.sqlite_errorname, "SQLITE_CONSTRAINT_CHECK") def test_disallow_instantiation(self): - cx = sqlite.connect(":memory:") - check_disallow_instantiation(self, type(cx("select 1"))) - check_disallow_instantiation(self, sqlite.Blob) + with memory_database() as cx: + check_disallow_instantiation(self, type(cx("select 1"))) + check_disallow_instantiation(self, sqlite.Blob) def test_complete_statement(self): self.assertFalse(sqlite.complete_statement("select t")) @@ -342,6 +329,7 @@ def setUp(self): cu = self.cx.cursor() cu.execute("create table test(id integer primary key, name text)") cu.execute("insert into test(name) values (?)", ("foo",)) + cu.close() def tearDown(self): self.cx.close() @@ -412,21 +400,22 @@ def test_exceptions(self): def test_in_transaction(self): # Can't use db from setUp because we want to test initial state. - cx = sqlite.connect(":memory:") - cu = cx.cursor() - self.assertEqual(cx.in_transaction, False) - cu.execute("create table transactiontest(id integer primary key, name text)") - self.assertEqual(cx.in_transaction, False) - cu.execute("insert into transactiontest(name) values (?)", ("foo",)) - self.assertEqual(cx.in_transaction, True) - cu.execute("select name from transactiontest where name=?", ["foo"]) - row = cu.fetchone() - self.assertEqual(cx.in_transaction, True) - cx.commit() - self.assertEqual(cx.in_transaction, False) - cu.execute("select name from transactiontest where name=?", ["foo"]) - row = cu.fetchone() - self.assertEqual(cx.in_transaction, False) + with memory_database() as cx: + cu = cx.cursor() + self.assertEqual(cx.in_transaction, False) + cu.execute("create table transactiontest(id integer primary key, name text)") + self.assertEqual(cx.in_transaction, False) + cu.execute("insert into transactiontest(name) values (?)", ("foo",)) + self.assertEqual(cx.in_transaction, True) + cu.execute("select name from transactiontest where name=?", ["foo"]) + row = cu.fetchone() + self.assertEqual(cx.in_transaction, True) + cx.commit() + self.assertEqual(cx.in_transaction, False) + cu.execute("select name from transactiontest where name=?", ["foo"]) + row = cu.fetchone() + self.assertEqual(cx.in_transaction, False) + cu.close() def test_in_transaction_ro(self): with self.assertRaises(AttributeError): @@ -450,10 +439,9 @@ def test_connection_exceptions(self): self.assertIs(getattr(sqlite, exc), getattr(self.cx, exc)) def test_interrupt_on_closed_db(self): - cx = sqlite.connect(":memory:") - cx.close() + self.cx.close() with self.assertRaises(sqlite.ProgrammingError): - cx.interrupt() + self.cx.interrupt() def test_interrupt(self): self.assertIsNone(self.cx.interrupt()) @@ -521,29 +509,29 @@ def test_connection_init_good_isolation_levels(self): self.assertEqual(cx.isolation_level, level) def test_connection_reinit(self): - db = ":memory:" - cx = sqlite.connect(db) - cx.text_factory = bytes - cx.row_factory = sqlite.Row - cu = cx.cursor() - cu.execute("create table foo (bar)") - cu.executemany("insert into foo (bar) values (?)", - ((str(v),) for v in range(4))) - cu.execute("select bar from foo") - - rows = [r for r in cu.fetchmany(2)] - self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) - self.assertEqual([r[0] for r in rows], [b"0", b"1"]) - - cx.__init__(db) - cx.execute("create table foo (bar)") - cx.executemany("insert into foo (bar) values (?)", - ((v,) for v in ("a", "b", "c", "d"))) - - # This uses the old database, old row factory, but new text factory - rows = [r for r in cu.fetchall()] - self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) - self.assertEqual([r[0] for r in rows], ["2", "3"]) + with memory_database() as cx: + cx.text_factory = bytes + cx.row_factory = sqlite.Row + cu = cx.cursor() + cu.execute("create table foo (bar)") + cu.executemany("insert into foo (bar) values (?)", + ((str(v),) for v in range(4))) + cu.execute("select bar from foo") + + rows = [r for r in cu.fetchmany(2)] + self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) + self.assertEqual([r[0] for r in rows], [b"0", b"1"]) + + cx.__init__(":memory:") + cx.execute("create table foo (bar)") + cx.executemany("insert into foo (bar) values (?)", + ((v,) for v in ("a", "b", "c", "d"))) + + # This uses the old database, old row factory, but new text factory + rows = [r for r in cu.fetchall()] + self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) + self.assertEqual([r[0] for r in rows], ["2", "3"]) + cu.close() def test_connection_bad_reinit(self): cx = sqlite.connect(":memory:") @@ -591,11 +579,11 @@ def test_connect_positional_arguments(self): "parameters in Python 3.15." ) with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - sqlite.connect(":memory:", 1.0) + cx = sqlite.connect(":memory:", 1.0) + cx.close() self.assertEqual(cm.filename, __file__) - class UninitialisedConnectionTests(unittest.TestCase): def setUp(self): self.cx = sqlite.Connection.__new__(sqlite.Connection) @@ -1571,12 +1559,12 @@ def run(con, err): except sqlite.Error: err.append("multi-threading not allowed") - con = sqlite.connect(":memory:", check_same_thread=False) - err = [] - t = threading.Thread(target=run, kwargs={"con": con, "err": err}) - t.start() - t.join() - self.assertEqual(len(err), 0, "\n".join(err)) + with memory_database(check_same_thread=False) as con: + err = [] + t = threading.Thread(target=run, kwargs={"con": con, "err": err}) + t.start() + t.join() + self.assertEqual(len(err), 0, "\n".join(err)) class ConstructorTests(unittest.TestCase): @@ -1602,9 +1590,16 @@ def test_binary(self): b = sqlite.Binary(b"\0'") class ExtensionTests(unittest.TestCase): + def setUp(self): + self.con = sqlite.connect(":memory:") + self.cur = self.con.cursor() + + def tearDown(self): + self.cur.close() + self.con.close() + def test_script_string_sql(self): - con = sqlite.connect(":memory:") - cur = con.cursor() + cur = self.cur cur.executescript(""" -- bla bla /* a stupid comment */ @@ -1616,40 +1611,40 @@ def test_script_string_sql(self): self.assertEqual(res, 5) def test_script_syntax_error(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(sqlite.OperationalError): - cur.executescript("create table test(x); asdf; create table test2(x)") + self.cur.executescript(""" + CREATE TABLE test(x); + asdf; + CREATE TABLE test2(x) + """) def test_script_error_normal(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(sqlite.OperationalError): - cur.executescript("create table test(sadfsadfdsa); select foo from hurz;") + self.cur.executescript(""" + CREATE TABLE test(sadfsadfdsa); + SELECT foo FROM hurz; + """) def test_cursor_executescript_as_bytes(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(TypeError): - cur.executescript(b"create table test(foo); insert into test(foo) values (5);") + self.cur.executescript(b""" + CREATE TABLE test(foo); + INSERT INTO test(foo) VALUES (5); + """) def test_cursor_executescript_with_null_characters(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(ValueError): - cur.executescript(""" - create table a(i);\0 - insert into a(i) values (5); - """) + self.cur.executescript(""" + CREATE TABLE a(i);\0 + INSERT INTO a(i) VALUES (5); + """) def test_cursor_executescript_with_surrogates(self): - con = sqlite.connect(":memory:") - cur = con.cursor() with self.assertRaises(UnicodeEncodeError): - cur.executescript(""" - create table a(s); - insert into a(s) values ('\ud8ff'); - """) + self.cur.executescript(""" + CREATE TABLE a(s); + INSERT INTO a(s) VALUES ('\ud8ff'); + """) def test_cursor_executescript_too_large_script(self): msg = "query string is too large" @@ -1659,19 +1654,18 @@ def test_cursor_executescript_too_large_script(self): cx.executescript("select 'too large'".ljust(lim+1)) def test_cursor_executescript_tx_control(self): - con = sqlite.connect(":memory:") + con = self.con con.execute("begin") self.assertTrue(con.in_transaction) con.executescript("select 1") self.assertFalse(con.in_transaction) def test_connection_execute(self): - con = sqlite.connect(":memory:") - result = con.execute("select 5").fetchone()[0] + result = self.con.execute("select 5").fetchone()[0] self.assertEqual(result, 5, "Basic test of Connection.execute") def test_connection_executemany(self): - con = sqlite.connect(":memory:") + con = self.con con.execute("create table test(foo)") con.executemany("insert into test(foo) values (?)", [(3,), (4,)]) result = con.execute("select foo from test order by foo").fetchall() @@ -1679,47 +1673,44 @@ def test_connection_executemany(self): self.assertEqual(result[1][0], 4, "Basic test of Connection.executemany") def test_connection_executescript(self): - con = sqlite.connect(":memory:") - con.executescript("create table test(foo); insert into test(foo) values (5);") + con = self.con + con.executescript(""" + CREATE TABLE test(foo); + INSERT INTO test(foo) VALUES (5); + """) result = con.execute("select foo from test").fetchone()[0] self.assertEqual(result, 5, "Basic test of Connection.executescript") + class ClosedConTests(unittest.TestCase): + def check(self, fn, *args, **kwds): + regex = "Cannot operate on a closed database." + with self.assertRaisesRegex(sqlite.ProgrammingError, regex): + fn(*args, **kwds) + + def setUp(self): + self.con = sqlite.connect(":memory:") + self.cur = self.con.cursor() + self.con.close() + def test_closed_con_cursor(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - cur = con.cursor() + self.check(self.con.cursor) def test_closed_con_commit(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - con.commit() + self.check(self.con.commit) def test_closed_con_rollback(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - con.rollback() + self.check(self.con.rollback) def test_closed_cur_execute(self): - con = sqlite.connect(":memory:") - cur = con.cursor() - con.close() - with self.assertRaises(sqlite.ProgrammingError): - cur.execute("select 4") + self.check(self.cur.execute, "select 4") def test_closed_create_function(self): - con = sqlite.connect(":memory:") - con.close() - def f(x): return 17 - with self.assertRaises(sqlite.ProgrammingError): - con.create_function("foo", 1, f) + def f(x): + return 17 + self.check(self.con.create_function, "foo", 1, f) def test_closed_create_aggregate(self): - con = sqlite.connect(":memory:") - con.close() class Agg: def __init__(self): pass @@ -1727,29 +1718,21 @@ def step(self, x): pass def finalize(self): return 17 - with self.assertRaises(sqlite.ProgrammingError): - con.create_aggregate("foo", 1, Agg) + self.check(self.con.create_aggregate, "foo", 1, Agg) def test_closed_set_authorizer(self): - con = sqlite.connect(":memory:") - con.close() def authorizer(*args): return sqlite.DENY - with self.assertRaises(sqlite.ProgrammingError): - con.set_authorizer(authorizer) + self.check(self.con.set_authorizer, authorizer) def test_closed_set_progress_callback(self): - con = sqlite.connect(":memory:") - con.close() - def progress(): pass - with self.assertRaises(sqlite.ProgrammingError): - con.set_progress_handler(progress, 100) + def progress(): + pass + self.check(self.con.set_progress_handler, progress, 100) def test_closed_call(self): - con = sqlite.connect(":memory:") - con.close() - with self.assertRaises(sqlite.ProgrammingError): - con() + self.check(self.con) + class ClosedCurTests(unittest.TestCase): def test_closed(self): diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index d0c24b9c60e61..5f6811fb5cc0a 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -2,16 +2,12 @@ import unittest import sqlite3 as sqlite -from .test_dbapi import memory_database +from .util import memory_database +from .util import MemoryDatabaseMixin -class DumpTests(unittest.TestCase): - def setUp(self): - self.cx = sqlite.connect(":memory:") - self.cu = self.cx.cursor() - def tearDown(self): - self.cx.close() +class DumpTests(MemoryDatabaseMixin, unittest.TestCase): def test_table_dump(self): expected_sqls = [ diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index d63589483e104..a7c4417862aff 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -24,6 +24,9 @@ import sqlite3 as sqlite from collections.abc import Sequence +from .util import memory_database +from .util import MemoryDatabaseMixin + def dict_factory(cursor, row): d = {} @@ -45,10 +48,12 @@ class OkFactory(sqlite.Connection): def __init__(self, *args, **kwargs): sqlite.Connection.__init__(self, *args, **kwargs) - for factory in DefectFactory, OkFactory: - with self.subTest(factory=factory): - con = sqlite.connect(":memory:", factory=factory) - self.assertIsInstance(con, factory) + with memory_database(factory=OkFactory) as con: + self.assertIsInstance(con, OkFactory) + regex = "Base Connection.__init__ not called." + with self.assertRaisesRegex(sqlite.ProgrammingError, regex): + with memory_database(factory=DefectFactory) as con: + self.assertIsInstance(con, DefectFactory) def test_connection_factory_relayed_call(self): # gh-95132: keyword args must not be passed as positional args @@ -57,9 +62,9 @@ def __init__(self, *args, **kwargs): kwargs["isolation_level"] = None super(Factory, self).__init__(*args, **kwargs) - con = sqlite.connect(":memory:", factory=Factory) - self.assertIsNone(con.isolation_level) - self.assertIsInstance(con, Factory) + with memory_database(factory=Factory) as con: + self.assertIsNone(con.isolation_level) + self.assertIsInstance(con, Factory) def test_connection_factory_as_positional_arg(self): class Factory(sqlite.Connection): @@ -74,18 +79,13 @@ def __init__(self, *args, **kwargs): r"parameters in Python 3.15." ) with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - con = sqlite.connect(":memory:", 5.0, 0, None, True, Factory) + with memory_database(5.0, 0, None, True, Factory) as con: + self.assertIsNone(con.isolation_level) + self.assertIsInstance(con, Factory) self.assertEqual(cm.filename, __file__) - self.assertIsNone(con.isolation_level) - self.assertIsInstance(con, Factory) -class CursorFactoryTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") - - def tearDown(self): - self.con.close() +class CursorFactoryTests(MemoryDatabaseMixin, unittest.TestCase): def test_is_instance(self): cur = self.con.cursor() @@ -103,9 +103,8 @@ def test_invalid_factory(self): # invalid callable returning non-cursor self.assertRaises(TypeError, self.con.cursor, lambda con: None) -class RowFactoryTestsBackwardsCompat(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") + +class RowFactoryTestsBackwardsCompat(MemoryDatabaseMixin, unittest.TestCase): def test_is_produced_by_factory(self): cur = self.con.cursor(factory=MyCursor) @@ -114,12 +113,8 @@ def test_is_produced_by_factory(self): self.assertIsInstance(row, dict) cur.close() - def tearDown(self): - self.con.close() -class RowFactoryTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") +class RowFactoryTests(MemoryDatabaseMixin, unittest.TestCase): def test_custom_factory(self): self.con.row_factory = lambda cur, row: list(row) @@ -265,12 +260,8 @@ class FakeCursor(str): self.assertRaises(TypeError, self.con.cursor, FakeCursor) self.assertRaises(TypeError, sqlite.Row, FakeCursor(), ()) - def tearDown(self): - self.con.close() -class TextFactoryTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") +class TextFactoryTests(MemoryDatabaseMixin, unittest.TestCase): def test_unicode(self): austria = "?sterreich" @@ -291,15 +282,17 @@ def test_custom(self): self.assertEqual(type(row[0]), str, "type of row[0] must be unicode") self.assertTrue(row[0].endswith("reich"), "column must contain original data") - def tearDown(self): - self.con.close() class TextFactoryTestsWithEmbeddedZeroBytes(unittest.TestCase): + def setUp(self): self.con = sqlite.connect(":memory:") self.con.execute("create table test (value text)") self.con.execute("insert into test (value) values (?)", ("a\x00b",)) + def tearDown(self): + self.con.close() + def test_string(self): # text_factory defaults to str row = self.con.execute("select value from test").fetchone() @@ -325,9 +318,6 @@ def test_custom(self): self.assertIs(type(row[0]), bytes) self.assertEqual(row[0], b"a\x00b") - def tearDown(self): - self.con.close() - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/test_hooks.py b/Lib/test/test_sqlite3/test_hooks.py index 89230c08cc914..33f0af99532a1 100644 --- a/Lib/test/test_sqlite3/test_hooks.py +++ b/Lib/test/test_sqlite3/test_hooks.py @@ -26,34 +26,31 @@ from test.support.os_helper import TESTFN, unlink -from test.test_sqlite3.test_dbapi import memory_database, cx_limit -from test.test_sqlite3.test_userfunctions import with_tracebacks +from .util import memory_database, cx_limit, with_tracebacks +from .util import MemoryDatabaseMixin -class CollationTests(unittest.TestCase): +class CollationTests(MemoryDatabaseMixin, unittest.TestCase): + def test_create_collation_not_string(self): - con = sqlite.connect(":memory:") with self.assertRaises(TypeError): - con.create_collation(None, lambda x, y: (x > y) - (x < y)) + self.con.create_collation(None, lambda x, y: (x > y) - (x < y)) def test_create_collation_not_callable(self): - con = sqlite.connect(":memory:") with self.assertRaises(TypeError) as cm: - con.create_collation("X", 42) + self.con.create_collation("X", 42) self.assertEqual(str(cm.exception), 'parameter must be callable') def test_create_collation_not_ascii(self): - con = sqlite.connect(":memory:") - con.create_collation("coll?", lambda x, y: (x > y) - (x < y)) + self.con.create_collation("coll?", lambda x, y: (x > y) - (x < y)) def test_create_collation_bad_upper(self): class BadUpperStr(str): def upper(self): return None - con = sqlite.connect(":memory:") mycoll = lambda x, y: -((x > y) - (x < y)) - con.create_collation(BadUpperStr("mycoll"), mycoll) - result = con.execute(""" + self.con.create_collation(BadUpperStr("mycoll"), mycoll) + result = self.con.execute(""" select x from ( select 'a' as x union @@ -68,8 +65,7 @@ def mycoll(x, y): # reverse order return -((x > y) - (x < y)) - con = sqlite.connect(":memory:") - con.create_collation("mycoll", mycoll) + self.con.create_collation("mycoll", mycoll) sql = """ select x from ( select 'a' as x @@ -79,21 +75,20 @@ def mycoll(x, y): select 'c' as x ) order by x collate mycoll """ - result = con.execute(sql).fetchall() + result = self.con.execute(sql).fetchall() self.assertEqual(result, [('c',), ('b',), ('a',)], msg='the expected order was not returned') - con.create_collation("mycoll", None) + self.con.create_collation("mycoll", None) with self.assertRaises(sqlite.OperationalError) as cm: - result = con.execute(sql).fetchall() + result = self.con.execute(sql).fetchall() self.assertEqual(str(cm.exception), 'no such collation sequence: mycoll') def test_collation_returns_large_integer(self): def mycoll(x, y): # reverse order return -((x > y) - (x < y)) * 2**32 - con = sqlite.connect(":memory:") - con.create_collation("mycoll", mycoll) + self.con.create_collation("mycoll", mycoll) sql = """ select x from ( select 'a' as x @@ -103,7 +98,7 @@ def mycoll(x, y): select 'c' as x ) order by x collate mycoll """ - result = con.execute(sql).fetchall() + result = self.con.execute(sql).fetchall() self.assertEqual(result, [('c',), ('b',), ('a',)], msg="the expected order was not returned") @@ -112,7 +107,7 @@ def test_collation_register_twice(self): Register two different collation functions under the same name. Verify that the last one is actually used. """ - con = sqlite.connect(":memory:") + con = self.con con.create_collation("mycoll", lambda x, y: (x > y) - (x < y)) con.create_collation("mycoll", lambda x, y: -((x > y) - (x < y))) result = con.execute(""" @@ -126,25 +121,26 @@ def test_deregister_collation(self): Register a collation, then deregister it. Make sure an error is raised if we try to use it. """ - con = sqlite.connect(":memory:") + con = self.con con.create_collation("mycoll", lambda x, y: (x > y) - (x < y)) con.create_collation("mycoll", None) with self.assertRaises(sqlite.OperationalError) as cm: con.execute("select 'a' as x union select 'b' as x order by x collate mycoll") self.assertEqual(str(cm.exception), 'no such collation sequence: mycoll') -class ProgressTests(unittest.TestCase): + +class ProgressTests(MemoryDatabaseMixin, unittest.TestCase): + def test_progress_handler_used(self): """ Test that the progress handler is invoked once it is set. """ - con = sqlite.connect(":memory:") progress_calls = [] def progress(): progress_calls.append(None) return 0 - con.set_progress_handler(progress, 1) - con.execute(""" + self.con.set_progress_handler(progress, 1) + self.con.execute(""" create table foo(a, b) """) self.assertTrue(progress_calls) @@ -153,7 +149,7 @@ def test_opcode_count(self): """ Test that the opcode argument is respected. """ - con = sqlite.connect(":memory:") + con = self.con progress_calls = [] def progress(): progress_calls.append(None) @@ -176,11 +172,10 @@ def test_cancel_operation(self): """ Test that returning a non-zero value stops the operation in progress. """ - con = sqlite.connect(":memory:") def progress(): return 1 - con.set_progress_handler(progress, 1) - curs = con.cursor() + self.con.set_progress_handler(progress, 1) + curs = self.con.cursor() self.assertRaises( sqlite.OperationalError, curs.execute, @@ -190,7 +185,7 @@ def test_clear_handler(self): """ Test that setting the progress handler to None clears the previously set handler. """ - con = sqlite.connect(":memory:") + con = self.con action = 0 def progress(): nonlocal action @@ -203,31 +198,30 @@ def progress(): @with_tracebacks(ZeroDivisionError, name="bad_progress") def test_error_in_progress_handler(self): - con = sqlite.connect(":memory:") def bad_progress(): 1 / 0 - con.set_progress_handler(bad_progress, 1) + self.con.set_progress_handler(bad_progress, 1) with self.assertRaises(sqlite.OperationalError): - con.execute(""" + self.con.execute(""" create table foo(a, b) """) @with_tracebacks(ZeroDivisionError, name="bad_progress") def test_error_in_progress_handler_result(self): - con = sqlite.connect(":memory:") class BadBool: def __bool__(self): 1 / 0 def bad_progress(): return BadBool() - con.set_progress_handler(bad_progress, 1) + self.con.set_progress_handler(bad_progress, 1) with self.assertRaises(sqlite.OperationalError): - con.execute(""" + self.con.execute(""" create table foo(a, b) """) -class TraceCallbackTests(unittest.TestCase): +class TraceCallbackTests(MemoryDatabaseMixin, unittest.TestCase): + @contextlib.contextmanager def check_stmt_trace(self, cx, expected): try: @@ -242,12 +236,11 @@ def test_trace_callback_used(self): """ Test that the trace callback is invoked once it is set. """ - con = sqlite.connect(":memory:") traced_statements = [] def trace(statement): traced_statements.append(statement) - con.set_trace_callback(trace) - con.execute("create table foo(a, b)") + self.con.set_trace_callback(trace) + self.con.execute("create table foo(a, b)") self.assertTrue(traced_statements) self.assertTrue(any("create table foo" in stmt for stmt in traced_statements)) @@ -255,7 +248,7 @@ def test_clear_trace_callback(self): """ Test that setting the trace callback to None clears the previously set callback. """ - con = sqlite.connect(":memory:") + con = self.con traced_statements = [] def trace(statement): traced_statements.append(statement) @@ -269,7 +262,7 @@ def test_unicode_content(self): Test that the statement can contain unicode literals. """ unicode_value = '\xf6\xe4\xfc\xd6\xc4\xdc\xdf\u20ac' - con = sqlite.connect(":memory:") + con = self.con traced_statements = [] def trace(statement): traced_statements.append(statement) diff --git a/Lib/test/test_sqlite3/test_regression.py b/Lib/test/test_sqlite3/test_regression.py index 7e8221e7227e6..db4e13222da9d 100644 --- a/Lib/test/test_sqlite3/test_regression.py +++ b/Lib/test/test_sqlite3/test_regression.py @@ -28,15 +28,12 @@ from test import support from unittest.mock import patch -from test.test_sqlite3.test_dbapi import memory_database, cx_limit +from .util import memory_database, cx_limit +from .util import MemoryDatabaseMixin -class RegressionTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") - def tearDown(self): - self.con.close() +class RegressionTests(MemoryDatabaseMixin, unittest.TestCase): def test_pragma_user_version(self): # This used to crash pysqlite because this pragma command returns NULL for the column name @@ -45,28 +42,24 @@ def test_pragma_user_version(self): def test_pragma_schema_version(self): # This still crashed pysqlite <= 2.2.1 - con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_COLNAMES) - try: + with memory_database(detect_types=sqlite.PARSE_COLNAMES) as con: cur = self.con.cursor() cur.execute("pragma schema_version") - finally: - cur.close() - con.close() def test_statement_reset(self): # pysqlite 2.1.0 to 2.2.0 have the problem that not all statements are # reset before a rollback, but only those that are still in the # statement cache. The others are not accessible from the connection object. - con = sqlite.connect(":memory:", cached_statements=5) - cursors = [con.cursor() for x in range(5)] - cursors[0].execute("create table test(x)") - for i in range(10): - cursors[0].executemany("insert into test(x) values (?)", [(x,) for x in range(10)]) + with memory_database(cached_statements=5) as con: + cursors = [con.cursor() for x in range(5)] + cursors[0].execute("create table test(x)") + for i in range(10): + cursors[0].executemany("insert into test(x) values (?)", [(x,) for x in range(10)]) - for i in range(5): - cursors[i].execute(" " * i + "select x from test") + for i in range(5): + cursors[i].execute(" " * i + "select x from test") - con.rollback() + con.rollback() def test_column_name_with_spaces(self): cur = self.con.cursor() @@ -81,17 +74,15 @@ def test_statement_finalization_on_close_db(self): # cache when closing the database. statements that were still # referenced in cursors weren't closed and could provoke " # "OperationalError: Unable to close due to unfinalised statements". - con = sqlite.connect(":memory:") cursors = [] # default statement cache size is 100 for i in range(105): - cur = con.cursor() + cur = self.con.cursor() cursors.append(cur) cur.execute("select 1 x union select " + str(i)) - con.close() def test_on_conflict_rollback(self): - con = sqlite.connect(":memory:") + con = self.con con.execute("create table foo(x, unique(x) on conflict rollback)") con.execute("insert into foo(x) values (1)") try: @@ -126,16 +117,16 @@ def test_type_map_usage(self): a statement. This test exhibits the problem. """ SELECT = "select * from foo" - con = sqlite.connect(":memory:",detect_types=sqlite.PARSE_DECLTYPES) - cur = con.cursor() - cur.execute("create table foo(bar timestamp)") - with self.assertWarnsRegex(DeprecationWarning, "adapter"): - cur.execute("insert into foo(bar) values (?)", (datetime.datetime.now(),)) - cur.execute(SELECT) - cur.execute("drop table foo") - cur.execute("create table foo(bar integer)") - cur.execute("insert into foo(bar) values (5)") - cur.execute(SELECT) + with memory_database(detect_types=sqlite.PARSE_DECLTYPES) as con: + cur = con.cursor() + cur.execute("create table foo(bar timestamp)") + with self.assertWarnsRegex(DeprecationWarning, "adapter"): + cur.execute("insert into foo(bar) values (?)", (datetime.datetime.now(),)) + cur.execute(SELECT) + cur.execute("drop table foo") + cur.execute("create table foo(bar integer)") + cur.execute("insert into foo(bar) values (5)") + cur.execute(SELECT) def test_bind_mutating_list(self): # Issue41662: Crash when mutate a list of parameters during iteration. @@ -144,11 +135,11 @@ def __conform__(self, protocol): parameters.clear() return "..." parameters = [X(), 0] - con = sqlite.connect(":memory:",detect_types=sqlite.PARSE_DECLTYPES) - con.execute("create table foo(bar X, baz integer)") - # Should not crash - with self.assertRaises(IndexError): - con.execute("insert into foo(bar, baz) values (?, ?)", parameters) + with memory_database(detect_types=sqlite.PARSE_DECLTYPES) as con: + con.execute("create table foo(bar X, baz integer)") + # Should not crash + with self.assertRaises(IndexError): + con.execute("insert into foo(bar, baz) values (?, ?)", parameters) def test_error_msg_decode_error(self): # When porting the module to Python 3.0, the error message about @@ -173,7 +164,7 @@ def upper(self): def __del__(self): con.isolation_level = "" - con = sqlite.connect(":memory:") + con = self.con con.isolation_level = None for level in "", "DEFERRED", "IMMEDIATE", "EXCLUSIVE": with self.subTest(level=level): @@ -204,8 +195,7 @@ class Cursor(sqlite.Cursor): def __init__(self, con): pass - con = sqlite.connect(":memory:") - cur = Cursor(con) + cur = Cursor(self.con) with self.assertRaises(sqlite.ProgrammingError): cur.execute("select 4+5").fetchall() with self.assertRaisesRegex(sqlite.ProgrammingError, @@ -238,7 +228,9 @@ def test_auto_commit(self): 2.5.3 introduced a regression so that these could no longer be created. """ - con = sqlite.connect(":memory:", isolation_level=None) + with memory_database(isolation_level=None) as con: + self.assertIsNone(con.isolation_level) + self.assertFalse(con.in_transaction) def test_pragma_autocommit(self): """ @@ -273,9 +265,7 @@ def test_recursive_cursor_use(self): Recursively using a cursor, such as when reusing it from a generator led to segfaults. Now we catch recursive cursor usage and raise a ProgrammingError. """ - con = sqlite.connect(":memory:") - - cur = con.cursor() + cur = self.con.cursor() cur.execute("create table a (bar)") cur.execute("create table b (baz)") @@ -295,29 +285,30 @@ def test_convert_timestamp_microsecond_padding(self): since the microsecond string "456" actually represents "456000". """ - con = sqlite.connect(":memory:", detect_types=sqlite.PARSE_DECLTYPES) - cur = con.cursor() - cur.execute("CREATE TABLE t (x TIMESTAMP)") + with memory_database(detect_types=sqlite.PARSE_DECLTYPES) as con: + cur = con.cursor() + cur.execute("CREATE TABLE t (x TIMESTAMP)") - # Microseconds should be 456000 - cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.456')") + # Microseconds should be 456000 + cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.456')") - # Microseconds should be truncated to 123456 - cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.123456789')") + # Microseconds should be truncated to 123456 + cur.execute("INSERT INTO t (x) VALUES ('2012-04-04 15:06:00.123456789')") - cur.execute("SELECT * FROM t") - with self.assertWarnsRegex(DeprecationWarning, "converter"): - values = [x[0] for x in cur.fetchall()] + cur.execute("SELECT * FROM t") + with self.assertWarnsRegex(DeprecationWarning, "converter"): + values = [x[0] for x in cur.fetchall()] - self.assertEqual(values, [ - datetime.datetime(2012, 4, 4, 15, 6, 0, 456000), - datetime.datetime(2012, 4, 4, 15, 6, 0, 123456), - ]) + self.assertEqual(values, [ + datetime.datetime(2012, 4, 4, 15, 6, 0, 456000), + datetime.datetime(2012, 4, 4, 15, 6, 0, 123456), + ]) def test_invalid_isolation_level_type(self): # isolation level is a string, not an integer - self.assertRaises(TypeError, - sqlite.connect, ":memory:", isolation_level=123) + regex = "isolation_level must be str or None" + with self.assertRaisesRegex(TypeError, regex): + memory_database(isolation_level=123).__enter__() def test_null_character(self): @@ -333,7 +324,7 @@ def test_null_character(self): cur.execute, query) def test_surrogates(self): - con = sqlite.connect(":memory:") + con = self.con self.assertRaises(UnicodeEncodeError, con, "select '\ud8ff'") self.assertRaises(UnicodeEncodeError, con, "select '\udcff'") cur = con.cursor() @@ -359,7 +350,7 @@ def test_commit_cursor_reset(self): to return rows multiple times when fetched from cursors after commit. See issues 10513 and 23129 for details. """ - con = sqlite.connect(":memory:") + con = self.con con.executescript(""" create table t(c); create table t2(c); @@ -391,10 +382,9 @@ def test_bpo31770(self): """ def callback(*args): pass - con = sqlite.connect(":memory:") - cur = sqlite.Cursor(con) + cur = sqlite.Cursor(self.con) ref = weakref.ref(cur, callback) - cur.__init__(con) + cur.__init__(self.con) del cur # The interpreter shouldn't crash when ref is collected. del ref @@ -425,6 +415,7 @@ def test_return_empty_bytestring(self): def test_table_lock_cursor_replace_stmt(self): with memory_database() as con: + con = self.con cur = con.cursor() cur.execute("create table t(t)") cur.executemany("insert into t values(?)", diff --git a/Lib/test/test_sqlite3/test_transactions.py b/Lib/test/test_sqlite3/test_transactions.py index 5d211dd47b0b6..b7b231d222585 100644 --- a/Lib/test/test_sqlite3/test_transactions.py +++ b/Lib/test/test_sqlite3/test_transactions.py @@ -28,7 +28,8 @@ from test.support.os_helper import TESTFN, unlink from test.support.script_helper import assert_python_ok -from test.test_sqlite3.test_dbapi import memory_database +from .util import memory_database +from .util import MemoryDatabaseMixin TIMEOUT = LOOPBACK_TIMEOUT / 10 @@ -132,14 +133,14 @@ def test_locking(self): def test_rollback_cursor_consistency(self): """Check that cursors behave correctly after rollback.""" - con = sqlite.connect(":memory:") - cur = con.cursor() - cur.execute("create table test(x)") - cur.execute("insert into test(x) values (5)") - cur.execute("select 1 union select 2 union select 3") + with memory_database() as con: + cur = con.cursor() + cur.execute("create table test(x)") + cur.execute("insert into test(x) values (5)") + cur.execute("select 1 union select 2 union select 3") - con.rollback() - self.assertEqual(cur.fetchall(), [(1,), (2,), (3,)]) + con.rollback() + self.assertEqual(cur.fetchall(), [(1,), (2,), (3,)]) def test_multiple_cursors_and_iternext(self): # gh-94028: statements are cleared and reset in cursor iternext. @@ -218,10 +219,7 @@ def test_no_duplicate_rows_after_rollback_new_query(self): -class SpecialCommandTests(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") - self.cur = self.con.cursor() +class SpecialCommandTests(MemoryDatabaseMixin, unittest.TestCase): def test_drop_table(self): self.cur.execute("create table test(i)") @@ -233,14 +231,8 @@ def test_pragma(self): self.cur.execute("insert into test(i) values (5)") self.cur.execute("pragma count_changes=1") - def tearDown(self): - self.cur.close() - self.con.close() - -class TransactionalDDL(unittest.TestCase): - def setUp(self): - self.con = sqlite.connect(":memory:") +class TransactionalDDL(MemoryDatabaseMixin, unittest.TestCase): def test_ddl_does_not_autostart_transaction(self): # For backwards compatibility reasons, DDL statements should not @@ -268,9 +260,6 @@ def test_transactional_ddl(self): with self.assertRaises(sqlite.OperationalError): self.con.execute("select * from test") - def tearDown(self): - self.con.close() - class IsolationLevelFromInit(unittest.TestCase): CREATE = "create table t(t)" diff --git a/Lib/test/test_sqlite3/test_userfunctions.py b/Lib/test/test_sqlite3/test_userfunctions.py index 05c2fb3aa6f8f..5d12636dcd2b6 100644 --- a/Lib/test/test_sqlite3/test_userfunctions.py +++ b/Lib/test/test_sqlite3/test_userfunctions.py @@ -21,54 +21,15 @@ # misrepresented as being the original software. # 3. This notice may not be removed or altered from any source distribution. -import contextlib -import functools -import io -import re import sys import unittest import sqlite3 as sqlite from unittest.mock import Mock, patch -from test.support import bigmemtest, catch_unraisable_exception, gc_collect - -from test.test_sqlite3.test_dbapi import cx_limit - - -def with_tracebacks(exc, regex="", name=""): - """Convenience decorator for testing callback tracebacks.""" - def decorator(func): - _regex = re.compile(regex) if regex else None - @functools.wraps(func) - def wrapper(self, *args, **kwargs): - with catch_unraisable_exception() as cm: - # First, run the test with traceback enabled. - with check_tracebacks(self, cm, exc, _regex, name): - func(self, *args, **kwargs) - - # Then run the test with traceback disabled. - func(self, *args, **kwargs) - return wrapper - return decorator - - - at contextlib.contextmanager -def check_tracebacks(self, cm, exc, regex, obj_name): - """Convenience context manager for testing callback tracebacks.""" - sqlite.enable_callback_tracebacks(True) - try: - buf = io.StringIO() - with contextlib.redirect_stderr(buf): - yield - - self.assertEqual(cm.unraisable.exc_type, exc) - if regex: - msg = str(cm.unraisable.exc_value) - self.assertIsNotNone(regex.search(msg)) - if obj_name: - self.assertEqual(cm.unraisable.object.__name__, obj_name) - finally: - sqlite.enable_callback_tracebacks(False) +from test.support import bigmemtest, gc_collect + +from .util import cx_limit, memory_database +from .util import with_tracebacks, check_tracebacks def func_returntext(): @@ -405,19 +366,19 @@ def test_func_deterministic_keyword_only(self): def test_function_destructor_via_gc(self): # See bpo-44304: The destructor of the user function can # crash if is called without the GIL from the gc functions - dest = sqlite.connect(':memory:') def md5sum(t): return - dest.create_function("md5", 1, md5sum) - x = dest("create table lang (name, first_appeared)") - del md5sum, dest + with memory_database() as dest: + dest.create_function("md5", 1, md5sum) + x = dest("create table lang (name, first_appeared)") + del md5sum, dest - y = [x] - y.append(y) + y = [x] + y.append(y) - del x,y - gc_collect() + del x,y + gc_collect() @with_tracebacks(OverflowError) def test_func_return_too_large_int(self): @@ -514,6 +475,10 @@ def setUp(self): """ self.con.create_window_function("sumint", 1, WindowSumInt) + def tearDown(self): + self.cur.close() + self.con.close() + def test_win_sum_int(self): self.cur.execute(self.query % "sumint") self.assertEqual(self.cur.fetchall(), self.expected) @@ -634,6 +599,7 @@ def setUp(self): """) cur.execute("insert into test(t, i, f, n, b) values (?, ?, ?, ?, ?)", ("foo", 5, 3.14, None, memoryview(b"blob"),)) + cur.close() self.con.create_aggregate("nostep", 1, AggrNoStep) self.con.create_aggregate("nofinalize", 1, AggrNoFinalize) @@ -646,9 +612,7 @@ def setUp(self): self.con.create_aggregate("aggtxt", 1, AggrText) def tearDown(self): - #self.cur.close() - #self.con.close() - pass + self.con.close() def test_aggr_error_on_create(self): with self.assertRaises(sqlite.OperationalError): @@ -775,7 +739,7 @@ def setUp(self): self.con.set_authorizer(self.authorizer_cb) def tearDown(self): - pass + self.con.close() def test_table_access(self): with self.assertRaises(sqlite.DatabaseError) as cm: diff --git a/Lib/test/test_sqlite3/util.py b/Lib/test/test_sqlite3/util.py new file mode 100644 index 0000000000000..505406c437b63 --- /dev/null +++ b/Lib/test/test_sqlite3/util.py @@ -0,0 +1,78 @@ +import contextlib +import functools +import io +import re +import sqlite3 +import test.support +import unittest + + +# Helper for temporary memory databases +def memory_database(*args, **kwargs): + cx = sqlite3.connect(":memory:", *args, **kwargs) + return contextlib.closing(cx) + + +# Temporarily limit a database connection parameter + at contextlib.contextmanager +def cx_limit(cx, category=sqlite3.SQLITE_LIMIT_SQL_LENGTH, limit=128): + try: + _prev = cx.setlimit(category, limit) + yield limit + finally: + cx.setlimit(category, _prev) + + +def with_tracebacks(exc, regex="", name=""): + """Convenience decorator for testing callback tracebacks.""" + def decorator(func): + _regex = re.compile(regex) if regex else None + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + with test.support.catch_unraisable_exception() as cm: + # First, run the test with traceback enabled. + with check_tracebacks(self, cm, exc, _regex, name): + func(self, *args, **kwargs) + + # Then run the test with traceback disabled. + func(self, *args, **kwargs) + return wrapper + return decorator + + + at contextlib.contextmanager +def check_tracebacks(self, cm, exc, regex, obj_name): + """Convenience context manager for testing callback tracebacks.""" + sqlite3.enable_callback_tracebacks(True) + try: + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + yield + + self.assertEqual(cm.unraisable.exc_type, exc) + if regex: + msg = str(cm.unraisable.exc_value) + self.assertIsNotNone(regex.search(msg)) + if obj_name: + self.assertEqual(cm.unraisable.object.__name__, obj_name) + finally: + sqlite3.enable_callback_tracebacks(False) + + +class MemoryDatabaseMixin: + + def setUp(self): + self.con = sqlite3.connect(":memory:") + self.cur = self.con.cursor() + + def tearDown(self): + self.cur.close() + self.con.close() + + @property + def cx(self): + return self.con + + @property + def cu(self): + return self.cur From webhook-mailer at python.org Thu Aug 17 02:54:56 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Thu, 17 Aug 2023 06:54:56 -0000 Subject: [Python-checkins] [3.11] gh-107298: Add standard exceptions and warnings in the nitpick_ignore list (GH-108029) (GH-108071) Message-ID: https://github.com/python/cpython/commit/b8e62cfb01f072cc18d32c23e38b3f1107f19a7f commit: b8e62cfb01f072cc18d32c23e38b3f1107f19a7f branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: serhiy-storchaka date: 2023-08-17T06:54:53Z summary: [3.11] gh-107298: Add standard exceptions and warnings in the nitpick_ignore list (GH-108029) (GH-108071) (cherry picked from commit c9d83f93d804b80ee14480466ebee63a6f97dac2) Co-authored-by: Serhiy Storchaka files: M Doc/conf.py diff --git a/Doc/conf.py b/Doc/conf.py index d9e7c9f11e7c8..cfe34244fee39 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -149,6 +149,77 @@ ('envvar', 'USER'), ('envvar', 'USERNAME'), ('envvar', 'USERPROFILE'), +] + +# Temporary undocumented names. +# In future this list must be empty. +nitpick_ignore += [ + # C API: Standard Python exception classes + ('c:data', 'PyExc_ArithmeticError'), + ('c:data', 'PyExc_AssertionError'), + ('c:data', 'PyExc_AttributeError'), + ('c:data', 'PyExc_BaseException'), + ('c:data', 'PyExc_BlockingIOError'), + ('c:data', 'PyExc_BrokenPipeError'), + ('c:data', 'PyExc_BufferError'), + ('c:data', 'PyExc_ChildProcessError'), + ('c:data', 'PyExc_ConnectionAbortedError'), + ('c:data', 'PyExc_ConnectionError'), + ('c:data', 'PyExc_ConnectionRefusedError'), + ('c:data', 'PyExc_ConnectionResetError'), + ('c:data', 'PyExc_EOFError'), + ('c:data', 'PyExc_Exception'), + ('c:data', 'PyExc_FileExistsError'), + ('c:data', 'PyExc_FileNotFoundError'), + ('c:data', 'PyExc_FloatingPointError'), + ('c:data', 'PyExc_GeneratorExit'), + ('c:data', 'PyExc_ImportError'), + ('c:data', 'PyExc_IndentationError'), + ('c:data', 'PyExc_IndexError'), + ('c:data', 'PyExc_InterruptedError'), + ('c:data', 'PyExc_IsADirectoryError'), + ('c:data', 'PyExc_KeyboardInterrupt'), + ('c:data', 'PyExc_KeyError'), + ('c:data', 'PyExc_LookupError'), + ('c:data', 'PyExc_MemoryError'), + ('c:data', 'PyExc_ModuleNotFoundError'), + ('c:data', 'PyExc_NameError'), + ('c:data', 'PyExc_NotADirectoryError'), + ('c:data', 'PyExc_NotImplementedError'), + ('c:data', 'PyExc_OSError'), + ('c:data', 'PyExc_OverflowError'), + ('c:data', 'PyExc_PermissionError'), + ('c:data', 'PyExc_ProcessLookupError'), + ('c:data', 'PyExc_RecursionError'), + ('c:data', 'PyExc_ReferenceError'), + ('c:data', 'PyExc_RuntimeError'), + ('c:data', 'PyExc_StopAsyncIteration'), + ('c:data', 'PyExc_StopIteration'), + ('c:data', 'PyExc_SyntaxError'), + ('c:data', 'PyExc_SystemError'), + ('c:data', 'PyExc_SystemExit'), + ('c:data', 'PyExc_TabError'), + ('c:data', 'PyExc_TimeoutError'), + ('c:data', 'PyExc_TypeError'), + ('c:data', 'PyExc_UnboundLocalError'), + ('c:data', 'PyExc_UnicodeDecodeError'), + ('c:data', 'PyExc_UnicodeEncodeError'), + ('c:data', 'PyExc_UnicodeError'), + ('c:data', 'PyExc_UnicodeTranslateError'), + ('c:data', 'PyExc_ValueError'), + ('c:data', 'PyExc_ZeroDivisionError'), + # C API: Standard Python warning classes + ('c:data', 'PyExc_BytesWarning'), + ('c:data', 'PyExc_DeprecationWarning'), + ('c:data', 'PyExc_FutureWarning'), + ('c:data', 'PyExc_ImportWarning'), + ('c:data', 'PyExc_PendingDeprecationWarning'), + ('c:data', 'PyExc_ResourceWarning'), + ('c:data', 'PyExc_RuntimeWarning'), + ('c:data', 'PyExc_SyntaxWarning'), + ('c:data', 'PyExc_UnicodeWarning'), + ('c:data', 'PyExc_UserWarning'), + ('c:data', 'PyExc_Warning'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. From webhook-mailer at python.org Thu Aug 17 04:16:04 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Thu, 17 Aug 2023 08:16:04 -0000 Subject: [Python-checkins] gh-107298: Fix some references in the C API documentation (GH-108072) Message-ID: https://github.com/python/cpython/commit/f51f0466c07eabc6177c2f64f70c952dada050e8 commit: f51f0466c07eabc6177c2f64f70c952dada050e8 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-17T11:16:00+03:00 summary: gh-107298: Fix some references in the C API documentation (GH-108072) files: M Doc/c-api/typeobj.rst M Doc/extending/extending.rst M Doc/using/configure.rst M Doc/whatsnew/2.2.rst M Doc/whatsnew/2.3.rst M Doc/whatsnew/2.4.rst M Doc/whatsnew/2.6.rst M Doc/whatsnew/2.7.rst M Doc/whatsnew/3.0.rst M Doc/whatsnew/3.1.rst M Doc/whatsnew/3.2.rst diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index d394ce10504b0..cd037b4de882e 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1697,7 +1697,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) to a pointer, are valid C99 address constants. However, the unary '&' operator applied to a non-static variable - like :c:func:`PyBaseObject_Type` is not required to produce an address + like :c:data:`PyBaseObject_Type` is not required to produce an address constant. Compilers may support this (gcc does), MSVC does not. Both compilers are strictly standard conforming in this particular behavior. diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index f58b4f28113e8..1ee7f28b2ba22 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -242,7 +242,7 @@ needed to ensure that it will not be discarded, causing :c:data:`!SpamError` to become a dangling pointer. Should it become a dangling pointer, C code which raises the exception could cause a core dump or other unintended side effects. -We discuss the use of ``PyMODINIT_FUNC`` as a function return type later in this +We discuss the use of :c:macro:`PyMODINIT_FUNC` as a function return type later in this sample. The :exc:`!spam.error` exception can be raised in your extension module using a @@ -363,7 +363,7 @@ only non-\ ``static`` item defined in the module file:: return PyModule_Create(&spammodule); } -Note that PyMODINIT_FUNC declares the function as ``PyObject *`` return type, +Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type, declares any special linkage declarations required by the platform, and for C++ declares the function as ``extern "C"``. diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 441d346a1a38a..f4adea82a87c3 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -746,8 +746,8 @@ Example on Linux x86-64:: At the beginning of the files, C extensions are built as built-in modules. Extensions defined after the ``*shared*`` marker are built as dynamic libraries. -The :c:macro:`PyAPI_FUNC()`, :c:macro:`PyAPI_API()` and -:c:macro:`PyMODINIT_FUNC()` macros of :file:`Include/pyport.h` are defined +The :c:macro:`PyAPI_FUNC()`, :c:macro:`PyAPI_DATA()` and +:c:macro:`PyMODINIT_FUNC` macros of :file:`Include/pyport.h` are defined differently depending if the ``Py_BUILD_CORE_MODULE`` macro is defined: * Use ``Py_EXPORTED_SYMBOL`` if the ``Py_BUILD_CORE_MODULE`` is defined diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 44e9bd8d492bf..7de48a4026303 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -1109,7 +1109,7 @@ code, none of the changes described here will affect you very much. definition tables to simplify implementation of methods with no arguments or a single untyped argument. Calling such methods is more efficient than calling a corresponding method that uses :c:macro:`METH_VARARGS`. Also, the old - :c:macro:`METH_OLDARGS` style of writing C methods is now officially deprecated. + :c:macro:`!METH_OLDARGS` style of writing C methods is now officially deprecated. * Two new wrapper functions, :c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf` were added to provide cross-platform implementations for the relatively new diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index a96c1061455e0..2120ee49f7d6b 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1886,7 +1886,7 @@ Changes to Python's build process and to the C API include: (:file:`libpython2.3.so`) by supplying :option:`!--enable-shared` when running Python's :program:`configure` script. (Contributed by Ondrej Palkovsky.) -* The :c:macro:`DL_EXPORT` and :c:macro:`DL_IMPORT` macros are now deprecated. +* The :c:macro:`!DL_EXPORT` and :c:macro:`!DL_IMPORT` macros are now deprecated. Initialization functions for Python extension modules should now be declared using the new macro :c:macro:`PyMODINIT_FUNC`, while the Python core will generally use the :c:macro:`PyAPI_FUNC` and :c:macro:`PyAPI_DATA` macros. diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 9e8a9e6a622d0..40b404f87e515 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -1476,7 +1476,7 @@ Some of the changes to Python's build process and to the C API are: :c:func:`PyArg_ParseTupleAndKeywords` but takes a :c:type:`va_list` instead of a number of arguments. (Contributed by Greg Chapman.) -* A new method flag, :c:macro:`METH_COEXISTS`, allows a function defined in slots +* A new method flag, :c:macro:`METH_COEXIST`, allows a function defined in slots to co-exist with a :c:type:`PyCFunction` having the same name. This can halve the access time for a method such as :meth:`set.__contains__`. (Contributed by Raymond Hettinger.) diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index ad899d53886c5..5c46dc8952154 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -3060,9 +3060,9 @@ Changes to Python's build process and to the C API include: * Some macros were renamed in both 3.0 and 2.6 to make it clearer that they are macros, - not functions. :c:macro:`Py_Size()` became :c:macro:`Py_SIZE()`, - :c:macro:`Py_Type()` became :c:macro:`Py_TYPE()`, and - :c:macro:`Py_Refcnt()` became :c:macro:`Py_REFCNT()`. + not functions. :c:macro:`!Py_Size()` became :c:macro:`Py_SIZE()`, + :c:macro:`!Py_Type()` became :c:macro:`Py_TYPE()`, and + :c:macro:`!Py_Refcnt()` became :c:macro:`Py_REFCNT()`. The mixed-case macros are still available in Python 2.6 for backward compatibility. (:issue:`1629`) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 4b5a31f8a8481..00001f92b51c8 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2287,10 +2287,10 @@ object, and then get the ``void *`` pointer, which will usually point to an array of pointers to the module's various API functions. There is an existing data type already used for this, -:c:type:`PyCObject`, but it doesn't provide type safety. Evil code +:c:type:`!PyCObject`, but it doesn't provide type safety. Evil code written in pure Python could cause a segmentation fault by taking a -:c:type:`PyCObject` from module A and somehow substituting it for the -:c:type:`PyCObject` in module B. Capsules know their own name, +:c:type:`!PyCObject` from module A and somehow substituting it for the +:c:type:`!PyCObject` in module B. Capsules know their own name, and getting the pointer requires providing the name: .. code-block:: c @@ -2310,10 +2310,10 @@ detect the mismatched name and return false. Refer to :ref:`using-capsules` for more information on using these objects. Python 2.7 now uses capsules internally to provide various -extension-module APIs, but the :c:func:`PyCObject_AsVoidPtr` was +extension-module APIs, but the :c:func:`!PyCObject_AsVoidPtr` was modified to handle capsules, preserving compile-time compatibility -with the :c:type:`CObject` interface. Use of -:c:func:`PyCObject_AsVoidPtr` will signal a +with the :c:type:`!PyCObject` interface. Use of +:c:func:`!PyCObject_AsVoidPtr` will signal a :exc:`PendingDeprecationWarning`, which is silent by default. Implemented in Python 3.1 and backported to 2.7 by Larry Hastings; diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index b767bbe177abe..58d42bd94cb61 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -875,7 +875,7 @@ to the C API. * Renamed the boolean conversion C-level slot and method: ``nb_nonzero`` is now ``nb_bool``. -* Removed :c:macro:`METH_OLDARGS` and :c:macro:`WITH_CYCLE_GC` from the C API. +* Removed :c:macro:`!METH_OLDARGS` and :c:macro:`!WITH_CYCLE_GC` from the C API. .. ====================================================================== diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index c399f007fd63f..3c1c9c3c4bc60 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -510,7 +510,7 @@ Changes to Python's build process and to the C API include: (Contributed by Mark Dickinson; :issue:`5914`.) -* Added :c:type:`PyCapsule` as a replacement for the :c:type:`PyCObject` API. +* Added :c:type:`PyCapsule` as a replacement for the :c:type:`!PyCObject` API. The principal difference is that the new type has a well defined interface for passing typing safety information and a less complicated signature for calling a destructor. The old type had a problematic API and is now diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index ed1c1770fb0f5..56ac5c4c0a1c1 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -2658,7 +2658,7 @@ require changes to your code: * "t#" format has been removed: use "s#" or "s*" instead * "w" and "w#" formats has been removed: use "w*" instead -* The :c:type:`PyCObject` type, deprecated in 3.1, has been removed. To wrap +* The :c:type:`!PyCObject` type, deprecated in 3.1, has been removed. To wrap opaque C pointers in Python objects, the :c:type:`PyCapsule` API should be used instead; the new type has a well-defined interface for passing typing safety information and a less complicated signature for calling a destructor. From webhook-mailer at python.org Thu Aug 17 04:30:51 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Thu, 17 Aug 2023 08:30:51 -0000 Subject: [Python-checkins] [3.11] gh-107298: Fix some references in the C API documentation (GH-108072) (GH-108076) Message-ID: https://github.com/python/cpython/commit/358b1acc69c09d110eecfca8fa6b72cea376c5f7 commit: 358b1acc69c09d110eecfca8fa6b72cea376c5f7 branch: 3.11 author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-17T08:30:47Z summary: [3.11] gh-107298: Fix some references in the C API documentation (GH-108072) (GH-108076) (cherry picked from commit f51f0466c07eabc6177c2f64f70c952dada050e8) files: M Doc/c-api/typeobj.rst M Doc/extending/extending.rst M Doc/using/configure.rst M Doc/whatsnew/2.2.rst M Doc/whatsnew/2.3.rst M Doc/whatsnew/2.4.rst M Doc/whatsnew/2.6.rst M Doc/whatsnew/2.7.rst M Doc/whatsnew/3.0.rst M Doc/whatsnew/3.1.rst M Doc/whatsnew/3.2.rst diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index c2ed4e4391c89..62711802a783a 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1634,7 +1634,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) to a pointer, are valid C99 address constants. However, the unary '&' operator applied to a non-static variable - like :c:func:`PyBaseObject_Type` is not required to produce an address + like :c:data:`PyBaseObject_Type` is not required to produce an address constant. Compilers may support this (gcc does), MSVC does not. Both compilers are strictly standard conforming in this particular behavior. diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 72e30a6882807..68f8e0c667465 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -242,7 +242,7 @@ needed to ensure that it will not be discarded, causing :c:data:`!SpamError` to become a dangling pointer. Should it become a dangling pointer, C code which raises the exception could cause a core dump or other unintended side effects. -We discuss the use of ``PyMODINIT_FUNC`` as a function return type later in this +We discuss the use of :c:macro:`PyMODINIT_FUNC` as a function return type later in this sample. The :exc:`!spam.error` exception can be raised in your extension module using a @@ -363,7 +363,7 @@ only non-\ ``static`` item defined in the module file:: return PyModule_Create(&spammodule); } -Note that PyMODINIT_FUNC declares the function as ``PyObject *`` return type, +Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type, declares any special linkage declarations required by the platform, and for C++ declares the function as ``extern "C"``. diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 468228f12cfa0..25ba5084d5daa 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -679,8 +679,8 @@ Extensions defined after the ``*shared*`` marker are built as dynamic libraries. The :file:`setup.py` script only builds C extensions as shared libraries using the :mod:`distutils` module. -The :c:macro:`PyAPI_FUNC()`, :c:macro:`PyAPI_API()` and -:c:macro:`PyMODINIT_FUNC()` macros of :file:`Include/pyport.h` are defined +The :c:macro:`PyAPI_FUNC()`, :c:macro:`PyAPI_DATA()` and +:c:macro:`PyMODINIT_FUNC` macros of :file:`Include/pyport.h` are defined differently depending if the ``Py_BUILD_CORE_MODULE`` macro is defined: * Use ``Py_EXPORTED_SYMBOL`` if the ``Py_BUILD_CORE_MODULE`` is defined diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 44e9bd8d492bf..7de48a4026303 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -1109,7 +1109,7 @@ code, none of the changes described here will affect you very much. definition tables to simplify implementation of methods with no arguments or a single untyped argument. Calling such methods is more efficient than calling a corresponding method that uses :c:macro:`METH_VARARGS`. Also, the old - :c:macro:`METH_OLDARGS` style of writing C methods is now officially deprecated. + :c:macro:`!METH_OLDARGS` style of writing C methods is now officially deprecated. * Two new wrapper functions, :c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf` were added to provide cross-platform implementations for the relatively new diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index 68a8917a943e7..4390f245d3ab4 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1886,7 +1886,7 @@ Changes to Python's build process and to the C API include: (:file:`libpython2.3.so`) by supplying :option:`!--enable-shared` when running Python's :program:`configure` script. (Contributed by Ondrej Palkovsky.) -* The :c:macro:`DL_EXPORT` and :c:macro:`DL_IMPORT` macros are now deprecated. +* The :c:macro:`!DL_EXPORT` and :c:macro:`!DL_IMPORT` macros are now deprecated. Initialization functions for Python extension modules should now be declared using the new macro :c:macro:`PyMODINIT_FUNC`, while the Python core will generally use the :c:macro:`PyAPI_FUNC` and :c:macro:`PyAPI_DATA` macros. diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index a00c4360b9c68..4272928fb6789 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -1476,7 +1476,7 @@ Some of the changes to Python's build process and to the C API are: :c:func:`PyArg_ParseTupleAndKeywords` but takes a :c:type:`va_list` instead of a number of arguments. (Contributed by Greg Chapman.) -* A new method flag, :c:macro:`METH_COEXISTS`, allows a function defined in slots +* A new method flag, :c:macro:`METH_COEXIST`, allows a function defined in slots to co-exist with a :c:type:`PyCFunction` having the same name. This can halve the access time for a method such as :meth:`set.__contains__`. (Contributed by Raymond Hettinger.) diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 2b8fa1546a588..563fa461d2a86 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -3060,9 +3060,9 @@ Changes to Python's build process and to the C API include: * Some macros were renamed in both 3.0 and 2.6 to make it clearer that they are macros, - not functions. :c:macro:`Py_Size()` became :c:macro:`Py_SIZE()`, - :c:macro:`Py_Type()` became :c:macro:`Py_TYPE()`, and - :c:macro:`Py_Refcnt()` became :c:macro:`Py_REFCNT()`. + not functions. :c:macro:`!Py_Size()` became :c:macro:`Py_SIZE()`, + :c:macro:`!Py_Type()` became :c:macro:`Py_TYPE()`, and + :c:macro:`!Py_Refcnt()` became :c:macro:`Py_REFCNT()`. The mixed-case macros are still available in Python 2.6 for backward compatibility. (:issue:`1629`) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 09798ca82d261..bc62a2155f1b6 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2286,10 +2286,10 @@ object, and then get the ``void *`` pointer, which will usually point to an array of pointers to the module's various API functions. There is an existing data type already used for this, -:c:type:`PyCObject`, but it doesn't provide type safety. Evil code +:c:type:`!PyCObject`, but it doesn't provide type safety. Evil code written in pure Python could cause a segmentation fault by taking a -:c:type:`PyCObject` from module A and somehow substituting it for the -:c:type:`PyCObject` in module B. Capsules know their own name, +:c:type:`!PyCObject` from module A and somehow substituting it for the +:c:type:`!PyCObject` in module B. Capsules know their own name, and getting the pointer requires providing the name: .. code-block:: c @@ -2309,10 +2309,10 @@ detect the mismatched name and return false. Refer to :ref:`using-capsules` for more information on using these objects. Python 2.7 now uses capsules internally to provide various -extension-module APIs, but the :c:func:`PyCObject_AsVoidPtr` was +extension-module APIs, but the :c:func:`!PyCObject_AsVoidPtr` was modified to handle capsules, preserving compile-time compatibility -with the :c:type:`CObject` interface. Use of -:c:func:`PyCObject_AsVoidPtr` will signal a +with the :c:type:`!PyCObject` interface. Use of +:c:func:`!PyCObject_AsVoidPtr` will signal a :exc:`PendingDeprecationWarning`, which is silent by default. Implemented in Python 3.1 and backported to 2.7 by Larry Hastings; diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 6f2a155abe10b..65fcc071a5cf5 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -875,7 +875,7 @@ to the C API. * Renamed the boolean conversion C-level slot and method: ``nb_nonzero`` is now ``nb_bool``. -* Removed :c:macro:`METH_OLDARGS` and :c:macro:`WITH_CYCLE_GC` from the C API. +* Removed :c:macro:`!METH_OLDARGS` and :c:macro:`!WITH_CYCLE_GC` from the C API. .. ====================================================================== diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index e4365d44928b5..ceef622f9f8da 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -510,7 +510,7 @@ Changes to Python's build process and to the C API include: (Contributed by Mark Dickinson; :issue:`5914`.) -* Added :c:type:`PyCapsule` as a replacement for the :c:type:`PyCObject` API. +* Added :c:type:`PyCapsule` as a replacement for the :c:type:`!PyCObject` API. The principal difference is that the new type has a well defined interface for passing typing safety information and a less complicated signature for calling a destructor. The old type had a problematic API and is now diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 01f5cf5204bcf..de355c19ac0e4 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -2656,7 +2656,7 @@ require changes to your code: * "t#" format has been removed: use "s#" or "s*" instead * "w" and "w#" formats has been removed: use "w*" instead -* The :c:type:`PyCObject` type, deprecated in 3.1, has been removed. To wrap +* The :c:type:`!PyCObject` type, deprecated in 3.1, has been removed. To wrap opaque C pointers in Python objects, the :c:type:`PyCapsule` API should be used instead; the new type has a well-defined interface for passing typing safety information and a less complicated signature for calling a destructor. From webhook-mailer at python.org Thu Aug 17 06:01:18 2023 From: webhook-mailer at python.org (hugovk) Date: Thu, 17 Aug 2023 10:01:18 -0000 Subject: [Python-checkins] GH-107987: Remove the Distributing Python Modules guide (#108016) Message-ID: https://github.com/python/cpython/commit/33e6e3fec02ff3035dec52692542d3dd10124bef commit: 33e6e3fec02ff3035dec52692542d3dd10124bef branch: main author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: hugovk date: 2023-08-17T13:01:14+03:00 summary: GH-107987: Remove the Distributing Python Modules guide (#108016) files: M Doc/conf.py M Doc/contents.rst M Doc/distributing/index.rst M Doc/installing/index.rst M Doc/tutorial/venv.rst diff --git a/Doc/conf.py b/Doc/conf.py index 19eab1a9340e4..16fd8bf257179 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -354,8 +354,6 @@ latex_documents = [ ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), - ('distributing/index', 'distributing.tex', - 'Distributing Python Modules', _stdauthor, 'manual'), ('extending/index', 'extending.tex', 'Extending and Embedding Python', _stdauthor, 'manual'), ('installing/index', 'installing.tex', diff --git a/Doc/contents.rst b/Doc/contents.rst index 649a1344a0b26..24ceacb0076b5 100644 --- a/Doc/contents.rst +++ b/Doc/contents.rst @@ -11,7 +11,6 @@ library/index.rst extending/index.rst c-api/index.rst - distributing/index.rst installing/index.rst howto/index.rst faq/index.rst diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index d237f8f082d87..2430564b45d6d 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -1,174 +1,19 @@ +:orphan: + +.. This page is retained solely for existing links to /distributing/index.html. + Direct readers to the PPUG instead. + .. _distributing-index: ############################### Distributing Python Modules ############################### -:Email: distutils-sig at python.org - - -As a popular open source development project, Python has an active -supporting community of contributors and users that also make their software -available for other Python developers to use under open source license terms. - -This allows Python users to share and collaborate effectively, benefiting -from the solutions others have already created to common (and sometimes -even rare!) problems, as well as potentially contributing their own -solutions to the common pool. - -This guide covers the distribution part of the process. For a guide to -installing other Python projects, refer to the -:ref:`installation guide `. - .. note:: - For corporate and other institutional users, be aware that many - organisations have their own policies around using and contributing to - open source software. Please take such policies into account when making - use of the distribution and installation tools provided with Python. - - -Key terms -========= - -* the `Python Package Index `__ is a public - repository of open source licensed packages made available for use by - other Python users -* the `Python Packaging Authority - `__ are the group of - developers and documentation authors responsible for the maintenance and - evolution of the standard packaging tools and the associated metadata and - file format standards. They maintain a variety of tools, documentation - and issue trackers on `GitHub `__. -* ``distutils`` is the original build and distribution system first added - to the Python standard library in 1998. While direct use of ``distutils`` - is being phased out, it still laid the foundation for the current packaging - and distribution infrastructure, and it not only remains part of the - standard library, but its name lives on in other ways (such as the name - of the mailing list used to coordinate Python packaging standards - development). -* `setuptools`_ is a (largely) drop-in replacement for ``distutils`` first - published in 2004. Its most notable addition over the unmodified - ``distutils`` tools was the ability to declare dependencies on other - packages. It is currently recommended as a more regularly updated - alternative to ``distutils`` that offers consistent support for more - recent packaging standards across a wide range of Python versions. -* `wheel`_ (in this context) is a project that adds the ``bdist_wheel`` - command to ``distutils``/`setuptools`_. This produces a cross platform - binary packaging format (called "wheels" or "wheel files" and defined in - :pep:`427`) that allows Python libraries, even those including binary - extensions, to be installed on a system without needing to be built - locally. - -.. _setuptools: https://setuptools.readthedocs.io/en/latest/ -.. _wheel: https://wheel.readthedocs.io/ - -Open source licensing and collaboration -======================================= - -In most parts of the world, software is automatically covered by copyright. -This means that other developers require explicit permission to copy, use, -modify and redistribute the software. - -Open source licensing is a way of explicitly granting such permission in a -relatively consistent way, allowing developers to share and collaborate -efficiently by making common solutions to various problems freely available. -This leaves many developers free to spend more time focusing on the problems -that are relatively unique to their specific situation. - -The distribution tools provided with Python are designed to make it -reasonably straightforward for developers to make their own contributions -back to that common pool of software if they choose to do so. - -The same distribution tools can also be used to distribute software within -an organisation, regardless of whether that software is published as open -source software or not. - - -Installing the tools -==================== - -The standard library does not include build tools that support modern -Python packaging standards, as the core development team has found that it -is important to have standard tools that work consistently, even on older -versions of Python. - -The currently recommended build and distribution tools can be installed -by invoking the ``pip`` module at the command line:: - - python -m pip install setuptools wheel twine - -.. note:: - - For POSIX users (including macOS and Linux users), these instructions - assume the use of a :term:`virtual environment`. - - For Windows users, these instructions assume that the option to - adjust the system PATH environment variable was selected when installing - Python. - -The Python Packaging User Guide includes more details on the `currently -recommended tools`_. - -.. _currently recommended tools: https://packaging.python.org/guides/tool-recommendations/#packaging-tool-recommendations - -.. index:: - single: Python Package Index (PyPI) - single: PyPI; (see Python Package Index (PyPI)) - -.. _publishing-python-packages: - -Reading the Python Packaging User Guide -======================================= - -The Python Packaging User Guide covers the various key steps and elements -involved in creating and publishing a project: - -* `Project structure`_ -* `Building and packaging the project`_ -* `Uploading the project to the Python Package Index`_ -* `The .pypirc file`_ - -.. _Project structure: https://packaging.python.org/tutorials/packaging-projects/#packaging-python-projects -.. _Building and packaging the project: https://packaging.python.org/tutorials/packaging-projects/#creating-the-package-files -.. _Uploading the project to the Python Package Index: https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives -.. _The .pypirc file: https://packaging.python.org/specifications/pypirc/ - - -How do I...? -============ - -These are quick answers or links for some common tasks. - -... choose a name for my project? ---------------------------------- - -This isn't an easy topic, but here are a few tips: - -* check the Python Package Index to see if the name is already in use -* check popular hosting sites like GitHub, Bitbucket, etc to see if there - is already a project with that name -* check what comes up in a web search for the name you're considering -* avoid particularly common words, especially ones with multiple meanings, - as they can make it difficult for users to find your software when - searching for it - - -... create and distribute binary extensions? --------------------------------------------- - -This is actually quite a complex topic, with a variety of alternatives -available depending on exactly what you're aiming to achieve. See the -Python Packaging User Guide for more information and recommendations. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions - `__ - -.. other topics: + Information and guidance on distributing Python modules and packages + has been moved to the `Python Packaging User Guide`_, + and the tutorial on `packaging Python projects`_. - Once the Development & Deployment part of PPUG is fleshed out, some of - those sections should be linked from new questions here (most notably, - we should have a question about avoiding depending on PyPI that links to - https://packaging.python.org/en/latest/mirrors/) + .. _Python Packaging User Guide: https://packaging.python.org/ + .. _packaging Python projects: https://packaging.python.org/en/latest/tutorials/packaging-projects/ diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index 5aec5178d48f3..a46c1caefe4d8 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -19,7 +19,9 @@ solutions to the common pool. This guide covers the installation part of the process. For a guide to creating and sharing your own Python projects, refer to the -:ref:`distribution guide `. +`Python packaging user guide`_. + +.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/packaging-projects/ .. note:: diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index d1bba098d7d23..a6dead2eac11f 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -207,4 +207,6 @@ necessary packages with ``install -r``: ``pip`` has many more options. Consult the :ref:`installing-index` guide for complete documentation for ``pip``. When you've written a package and want to make it available on the Python Package Index, -consult the :ref:`distributing-index` guide. +consult the `Python packaging user guide`_. + +.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/packaging-projects/ From webhook-mailer at python.org Thu Aug 17 06:16:06 2023 From: webhook-mailer at python.org (markshannon) Date: Thu, 17 Aug 2023 10:16:06 -0000 Subject: [Python-checkins] GH-108035: Remove the `_PyCFrame` struct as it is no longer needed for performance. (GH-108036) Message-ID: https://github.com/python/cpython/commit/006e44f9502308ec3d14424ad8bd774046f2be8e commit: 006e44f9502308ec3d14424ad8bd774046f2be8e branch: main author: Mark Shannon committer: markshannon date: 2023-08-17T11:16:03+01:00 summary: GH-108035: Remove the `_PyCFrame` struct as it is no longer needed for performance. (GH-108036) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-11-16-18-19.gh-issue-108035.e2msOD.rst M Include/cpython/pystate.h M Include/internal/pycore_frame.h M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Modules/_xxsubinterpretersmodule.c M Objects/genobject.c M Objects/typevarobject.c M Python/bytecodes.c M Python/ceval.c M Python/ceval_macros.h M Python/executor.c M Python/executor_cases.c.h M Python/frame.c M Python/generated_cases.c.h M Python/instrumentation.c M Python/intrinsics.c M Python/pylifecycle.c M Python/pystate.c M Python/sysmodule.c M Python/traceback.c diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 56e473cc5e42d..2a53838314c9d 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -29,24 +29,6 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *); #define PyTrace_C_RETURN 6 #define PyTrace_OPCODE 7 -// Internal structure: you should not use it directly, but use public functions -// like PyThreadState_EnterTracing() and PyThreadState_LeaveTracing(). -typedef struct _PyCFrame { - /* This struct will be threaded through the C stack - * allowing fast access to per-thread state that needs - * to be accessed quickly by the interpreter, but can - * be modified outside of the interpreter. - * - * WARNING: This makes data on the C stack accessible from - * heap objects. Care must be taken to maintain stack - * discipline and make sure that instances of this struct cannot - * accessed outside of their lifetime. - */ - /* Pointer to the currently executing frame (it can be NULL) */ - struct _PyInterpreterFrame *current_frame; - struct _PyCFrame *previous; -} _PyCFrame; - typedef struct _err_stackitem { /* This struct represents a single execution context where we might * be currently handling an exception. It is a per-coroutine state @@ -123,9 +105,8 @@ struct _ts { int tracing; int what_event; /* The event currently being monitored, if any. */ - /* Pointer to current _PyCFrame in the C stack frame of the currently, - * or most recently, executing _PyEval_EvalFrameDefault. */ - _PyCFrame *cframe; + /* Pointer to currently executing frame. */ + struct _PyInterpreterFrame *current_frame; Py_tracefunc c_profilefunc; Py_tracefunc c_tracefunc; @@ -211,8 +192,6 @@ struct _ts { /* The thread's exception stack entry. (Always the last entry.) */ _PyErr_StackItem exc_state; - /* The bottom-most frame on the stack. */ - _PyCFrame root_cframe; }; /* WASI has limited call stack. Python's recursion limit depends on code diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 5ff20ef845ab1..0dc2a1814cb1d 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -196,7 +196,7 @@ _PyFrame_GetFirstComplete(_PyInterpreterFrame *frame) static inline _PyInterpreterFrame * _PyThreadState_GetFrame(PyThreadState *tstate) { - return _PyFrame_GetFirstComplete(tstate->cframe->current_frame); + return _PyFrame_GetFirstComplete(tstate->current_frame); } /* For use by _PyFrame_GetFrameObject diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 0ec86ee6c50ca..63b485027486b 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -80,7 +80,7 @@ typedef struct _Py_DebugOffsets { off_t prev; off_t next; off_t interp; - off_t cframe; + off_t current_frame; off_t thread_id; off_t native_thread_id; } thread_state; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index e89d368be07aa..ea29c69f59acc 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -45,7 +45,7 @@ extern PyTypeObject _PyExc_MemoryError; .prev = offsetof(PyThreadState, prev), \ .next = offsetof(PyThreadState, next), \ .interp = offsetof(PyThreadState, interp), \ - .cframe = offsetof(PyThreadState, cframe), \ + .current_frame = offsetof(PyThreadState, current_frame), \ .thread_id = offsetof(PyThreadState, thread_id), \ .native_thread_id = offsetof(PyThreadState, native_thread_id), \ }, \ @@ -56,10 +56,6 @@ extern PyTypeObject _PyExc_MemoryError; .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ .owner = offsetof(_PyInterpreterFrame, owner), \ }, \ - .cframe = { \ - .current_frame = offsetof(_PyCFrame, current_frame), \ - .previous = offsetof(_PyCFrame, previous), \ - }, \ .code_object = { \ .filename = offsetof(PyCodeObject, co_filename), \ .name = offsetof(PyCodeObject, co_name), \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-11-16-18-19.gh-issue-108035.e2msOD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-11-16-18-19.gh-issue-108035.e2msOD.rst new file mode 100644 index 0000000000000..fc2369ddabb83 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-11-16-18-19.gh-issue-108035.e2msOD.rst @@ -0,0 +1,4 @@ +Remove the ``_PyCFrame`` struct, moving the pointer to the current intepreter frame +back to the threadstate, as it was for 3.10 and earlier. The ``_PyCFrame`` +existed as a performance optimization for tracing. Since PEP 669 has been +implemented, this optimization no longer applies. diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 31373f8fdf8c7..ea91e70cad991 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -369,7 +369,7 @@ _is_running(PyInterpreterState *interp) } assert(!PyErr_Occurred()); - struct _PyInterpreterFrame *frame = tstate->cframe->current_frame; + struct _PyInterpreterFrame *frame = tstate->current_frame; if (frame == NULL) { return 0; } diff --git a/Objects/genobject.c b/Objects/genobject.c index 65782be182cd7..10c55efb1b1e9 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -476,9 +476,9 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, will be reported correctly to the user. */ /* XXX We should probably be updating the current frame somewhere in ceval.c. */ - _PyInterpreterFrame *prev = tstate->cframe->current_frame; + _PyInterpreterFrame *prev = tstate->current_frame; frame->previous = prev; - tstate->cframe->current_frame = frame; + tstate->current_frame = frame; /* Close the generator that we are currently iterating with 'yield from' or awaiting on with 'await'. */ PyFrameState state = gen->gi_frame_state; @@ -486,7 +486,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, ret = _gen_throw((PyGenObject *)yf, close_on_genexit, typ, val, tb); gen->gi_frame_state = state; - tstate->cframe->current_frame = prev; + tstate->current_frame = prev; frame->previous = NULL; } else { /* `yf` is an iterator or a coroutine-like object. */ @@ -938,7 +938,7 @@ _Py_MakeCoro(PyFunctionObject *func) if (origin_depth == 0) { ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; } else { - _PyInterpreterFrame *frame = tstate->cframe->current_frame; + _PyInterpreterFrame *frame = tstate->current_frame; assert(frame); assert(_PyFrame_IsIncomplete(frame)); frame = _PyFrame_GetFirstComplete(frame->previous); diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index e09e6a62553cf..66122e36283ad 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -107,7 +107,7 @@ make_union(PyObject *self, PyObject *other) static PyObject * caller(void) { - _PyInterpreterFrame *f = _PyThreadState_GET()->cframe->current_frame; + _PyInterpreterFrame *f = _PyThreadState_GET()->current_frame; if (f == NULL) { Py_RETURN_NONE; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ae2923c65b330..6f17472e04e5e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -72,7 +72,6 @@ dummy_func( _PyInterpreterFrame *frame, unsigned char opcode, unsigned int oparg, - _PyCFrame cframe, _Py_CODEUNIT *next_instr, PyObject **stack_pointer, PyObject *kwnames, @@ -134,8 +133,7 @@ dummy_func( } inst(RESUME, (--)) { - assert(tstate->cframe == &cframe); - assert(frame == cframe.current_frame); + assert(frame == tstate->current_frame); /* Possibly combine this with eval breaker */ if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); @@ -752,9 +750,8 @@ dummy_func( inst(INTERPRETER_EXIT, (retval --)) { assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous cframe and return. */ - tstate->cframe = cframe.previous; - assert(tstate->cframe->current_frame == frame->previous); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; @@ -768,7 +765,7 @@ dummy_func( assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -787,7 +784,7 @@ dummy_func( assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -803,7 +800,7 @@ dummy_func( assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -823,7 +820,7 @@ dummy_func( assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -1019,7 +1016,7 @@ dummy_func( gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; - frame = cframe.current_frame = frame->previous; + frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; @@ -1038,7 +1035,7 @@ dummy_func( gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; - frame = cframe.current_frame = frame->previous; + frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; @@ -2207,10 +2204,10 @@ dummy_func( OBJECT_STAT_INC(optimization_attempts); frame = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer); if (frame == NULL) { - frame = cframe.current_frame; + frame = tstate->current_frame; goto resume_with_error; } - assert(frame == cframe.current_frame); + assert(frame == tstate->current_frame); here[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) -1); goto resume_frame; } @@ -2238,7 +2235,7 @@ dummy_func( Py_INCREF(executor); frame = executor->execute(executor, frame, stack_pointer); if (frame == NULL) { - frame = cframe.current_frame; + frame = tstate->current_frame; goto resume_with_error; } goto resume_frame; @@ -2993,12 +2990,11 @@ dummy_func( _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; #if TIER_ONE - frame = cframe.current_frame = new_frame; goto start_frame; #endif #if TIER_TWO - frame = tstate->cframe->current_frame = new_frame; ERROR_IF(_Py_EnterRecursivePy(tstate), exit_unwind); stack_pointer = _PyFrame_GetStackPointer(frame); ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; @@ -3135,7 +3131,7 @@ dummy_func( /* Link frames */ init_frame->previous = shim; shim->previous = frame; - frame = cframe.current_frame = init_frame; + frame = tstate->current_frame = init_frame; CALL_STAT_INC(inlined_py_calls); /* Account for pushing the extra frame. * We don't check recursion depth here, @@ -3598,7 +3594,7 @@ dummy_func( assert(frame != &entry_frame); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); - frame = cframe.current_frame = prev; + frame = tstate->current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; } diff --git a/Python/ceval.c b/Python/ceval.c index 26e741ed7c754..1e2262c1f18c3 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -656,17 +656,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int int lltrace = 0; #endif - _PyCFrame cframe; _PyInterpreterFrame entry_frame; PyObject *kwnames = NULL; // Borrowed reference. Reset by CALL instructions. - /* WARNING: Because the _PyCFrame lives on the C stack, - * but can be accessed from a heap allocated object (tstate) - * strict stack discipline must be maintained. - */ - _PyCFrame *prev_cframe = tstate->cframe; - cframe.previous = prev_cframe; - tstate->cframe = &cframe; + #ifdef Py_DEBUG /* Set these to invalid but identifiable values for debugging. */ @@ -682,9 +675,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry_frame.owner = FRAME_OWNED_BY_CSTACK; entry_frame.return_offset = 0; /* Push frame */ - entry_frame.previous = prev_cframe->current_frame; + entry_frame.previous = tstate->current_frame; frame->previous = &entry_frame; - cframe.current_frame = frame; + tstate->current_frame = frame; tstate->c_recursion_remaining -= (PY_EVAL_C_STACK_UNITS - 1); if (_Py_EnterRecursiveCallTstate(tstate, "")) { @@ -924,13 +917,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->return_offset = 0; if (frame == &entry_frame) { - /* Restore previous cframe and exit */ - tstate->cframe = cframe.previous; - assert(tstate->cframe->current_frame == frame->previous); + /* Restore previous frame and exit */ + tstate->current_frame = frame->previous; tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return NULL; } @@ -2297,7 +2289,7 @@ int PyEval_MergeCompilerFlags(PyCompilerFlags *cf) { PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *current_frame = tstate->cframe->current_frame; + _PyInterpreterFrame *current_frame = tstate->current_frame; int result = cf->cf_flags != 0; if (current_frame != NULL) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 5e2db1e0b394e..08f19cd9a397f 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -109,7 +109,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); \ frame->prev_instr = next_instr - 1; \ (NEW_FRAME)->previous = frame; \ - frame = cframe.current_frame = (NEW_FRAME); \ + frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ goto start_frame; \ } while (0) diff --git a/Python/executor.c b/Python/executor.c index 5a571e6da4673..88c039da8539e 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -111,7 +111,7 @@ _PyUopExecute(_PyExecutorObject *executor, _PyInterpreterFrame *frame, PyObject STACK_SHRINK(1); error: // On ERROR_IF we return NULL as the frame. - // The caller recovers the frame from cframe.current_frame. + // The caller recovers the frame from tstate->current_frame. DPRINTF(2, "Error: [Opcode %d, operand %" PRIu64 "]\n", opcode, operand); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(self); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b3dd313353056..9fbf026f164a6 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2212,12 +2212,11 @@ _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; #if TIER_ONE - frame = cframe.current_frame = new_frame; goto start_frame; #endif #if TIER_TWO - frame = tstate->cframe->current_frame = new_frame; if (_Py_EnterRecursivePy(tstate)) goto pop_1_exit_unwind; stack_pointer = _PyFrame_GetStackPointer(frame); ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; diff --git a/Python/frame.c b/Python/frame.c index 581e4f95710fe..fbfa54398c72b 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -124,7 +124,7 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) _PyFrame_GetGenerator(frame)->gi_frame_state == FRAME_CLEARED); // GH-99729: Clearing this frame can expose the stack (via finalizers). It's // crucial that this frame has been unlinked, and is no longer visible: - assert(_PyThreadState_GET()->cframe->current_frame != frame); + assert(_PyThreadState_GET()->current_frame != frame); if (frame->frame_obj) { PyFrameObject *f = frame->frame_obj; frame->frame_obj = NULL; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 11d560a6e77ad..80af8a7bcd56d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,8 +8,7 @@ } TARGET(RESUME) { - assert(tstate->cframe == &cframe); - assert(frame == cframe.current_frame); + assert(frame == tstate->current_frame); /* Possibly combine this with eval breaker */ if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); @@ -961,9 +960,8 @@ retval = stack_pointer[-1]; assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); - /* Restore previous cframe and return. */ - tstate->cframe = cframe.previous; - assert(tstate->cframe->current_frame == frame->previous); + /* Restore previous frame and return. */ + tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS; return retval; @@ -980,7 +978,7 @@ assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -1002,7 +1000,7 @@ assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -1019,7 +1017,7 @@ assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -1039,7 +1037,7 @@ assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; - frame = cframe.current_frame = dying->previous; + frame = tstate->current_frame = dying->previous; _PyEvalFrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); @@ -1263,7 +1261,7 @@ gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; - frame = cframe.current_frame = frame->previous; + frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; @@ -1284,7 +1282,7 @@ gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; - frame = cframe.current_frame = frame->previous; + frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; _PyFrame_StackPush(frame, retval); goto resume_frame; @@ -2911,10 +2909,10 @@ OBJECT_STAT_INC(optimization_attempts); frame = _PyOptimizer_BackEdge(frame, here, next_instr, stack_pointer); if (frame == NULL) { - frame = cframe.current_frame; + frame = tstate->current_frame; goto resume_with_error; } - assert(frame == cframe.current_frame); + assert(frame == tstate->current_frame); here[1].cache &= ((1 << OPTIMIZER_BITS_IN_COUNTER) -1); goto resume_frame; } @@ -2933,7 +2931,7 @@ Py_INCREF(executor); frame = executor->execute(executor, frame, stack_pointer); if (frame == NULL) { - frame = cframe.current_frame; + frame = tstate->current_frame; goto resume_with_error; } goto resume_frame; @@ -3830,12 +3828,11 @@ _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; #if TIER_ONE - frame = cframe.current_frame = new_frame; goto start_frame; #endif #if TIER_TWO - frame = tstate->cframe->current_frame = new_frame; if (_Py_EnterRecursivePy(tstate)) goto pop_1_exit_unwind; stack_pointer = _PyFrame_GetStackPointer(frame); ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; @@ -4014,7 +4011,7 @@ /* Link frames */ init_frame->previous = shim; shim->previous = frame; - frame = cframe.current_frame = init_frame; + frame = tstate->current_frame = init_frame; CALL_STAT_INC(inlined_py_calls); /* Account for pushing the extra frame. * We don't check recursion depth here, @@ -4636,7 +4633,7 @@ assert(frame != &entry_frame); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); - frame = cframe.current_frame = prev; + frame = tstate->current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 48befed4ea838..a7a5b4a5dc5f6 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1677,7 +1677,7 @@ instrument_all_executing_code_objects(PyInterpreterState *interp) { PyThreadState* ts = PyInterpreterState_ThreadHead(interp); HEAD_UNLOCK(runtime); while (ts) { - _PyInterpreterFrame *frame = ts->cframe->current_frame; + _PyInterpreterFrame *frame = ts->current_frame; while (frame) { if (frame->owner != FRAME_OWNED_BY_CSTACK) { if (_Py_Instrument(_PyFrame_GetCode(frame), interp)) { diff --git a/Python/intrinsics.c b/Python/intrinsics.c index 61a8e75872d2e..5267c10238e62 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -120,7 +120,7 @@ import_all_from(PyThreadState *tstate, PyObject *locals, PyObject *v) static PyObject * import_star(PyThreadState* tstate, PyObject *from) { - _PyInterpreterFrame *frame = tstate->cframe->current_frame; + _PyInterpreterFrame *frame = tstate->current_frame; if (_PyFrame_FastToLocalsWithError(frame) < 0) { return NULL; } @@ -142,7 +142,7 @@ import_star(PyThreadState* tstate, PyObject *from) static PyObject * stopiteration_error(PyThreadState* tstate, PyObject *exc) { - _PyInterpreterFrame *frame = tstate->cframe->current_frame; + _PyInterpreterFrame *frame = tstate->current_frame; assert(frame->owner == FRAME_OWNED_BY_GENERATOR); assert(PyExceptionInstance_Check(exc)); const char *msg = NULL; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 0de3abf940789..90633960cb00e 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2147,7 +2147,7 @@ Py_EndInterpreter(PyThreadState *tstate) if (tstate != _PyThreadState_GET()) { Py_FatalError("thread is not current"); } - if (tstate->cframe->current_frame != NULL) { + if (tstate->current_frame != NULL) { Py_FatalError("thread still has a frame"); } interp->finalizing = 1; diff --git a/Python/pystate.c b/Python/pystate.c index 3a05cb0fa7988..01651d79f9acc 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1310,7 +1310,7 @@ init_threadstate(PyThreadState *tstate, // This is cleared when PyGILState_Ensure() creates the thread state. tstate->gilstate_counter = 1; - tstate->cframe = &tstate->root_cframe; + tstate->current_frame = NULL; tstate->datastack_chunk = NULL; tstate->datastack_top = NULL; tstate->datastack_limit = NULL; @@ -1452,7 +1452,7 @@ PyThreadState_Clear(PyThreadState *tstate) int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; - if (verbose && tstate->cframe->current_frame != NULL) { + if (verbose && tstate->current_frame != NULL) { /* bpo-20526: After the main thread calls _PyInterpreterState_SetFinalizing() in Py_FinalizeEx() (or in Py_EndInterpreter() for subinterpreters), @@ -1953,7 +1953,7 @@ _PyThread_CurrentFrames(void) for (i = runtime->interpreters.head; i != NULL; i = i->next) { PyThreadState *t; for (t = i->threads.head; t != NULL; t = t->next) { - _PyInterpreterFrame *frame = t->cframe->current_frame; + _PyInterpreterFrame *frame = t->current_frame; frame = _PyFrame_GetFirstComplete(frame); if (frame == NULL) { continue; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index be026d95ba7e7..f82901181f886 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1954,7 +1954,7 @@ sys__getframe_impl(PyObject *module, int depth) /*[clinic end generated code: output=d438776c04d59804 input=c1be8a6464b11ee5]*/ { PyThreadState *tstate = _PyThreadState_GET(); - _PyInterpreterFrame *frame = tstate->cframe->current_frame; + _PyInterpreterFrame *frame = tstate->current_frame; if (frame != NULL) { while (depth > 0) { @@ -2270,7 +2270,7 @@ sys__getframemodulename_impl(PyObject *module, int depth) if (PySys_Audit("sys._getframemodulename", "i", depth) < 0) { return NULL; } - _PyInterpreterFrame *f = _PyThreadState_GET()->cframe->current_frame; + _PyInterpreterFrame *f = _PyThreadState_GET()->current_frame; while (f && (_PyFrame_IsIncomplete(f) || depth-- > 0)) { f = f->previous; } diff --git a/Python/traceback.c b/Python/traceback.c index ca524b1b9af78..bddb8763a2f9b 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -1207,7 +1207,7 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) PUTS(fd, "Stack (most recent call first):\n"); } - frame = tstate->cframe->current_frame; + frame = tstate->current_frame; if (frame == NULL) { PUTS(fd, " \n"); return; From webhook-mailer at python.org Thu Aug 17 06:25:32 2023 From: webhook-mailer at python.org (iritkatriel) Date: Thu, 17 Aug 2023 10:25:32 -0000 Subject: [Python-checkins] gh-105481: fix out of date comment (#108079) Message-ID: https://github.com/python/cpython/commit/2da67a5789a9247ae6611185bf786b697aa04fbe commit: 2da67a5789a9247ae6611185bf786b697aa04fbe branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-17T10:25:28Z summary: gh-105481: fix out of date comment (#108079) files: M Tools/build/deepfreeze.py diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index 8dbb7bfa69a92..996d5f65a923b 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -21,7 +21,7 @@ verbose = False -# This must be kept in sync with opcode.py +# This must be kept in sync with Tools/cases_generator/generate_cases.py RESUME = 166 def isprintable(b: bytes) -> bool: From webhook-mailer at python.org Thu Aug 17 06:50:50 2023 From: webhook-mailer at python.org (Yhg1s) Date: Thu, 17 Aug 2023 10:50:50 -0000 Subject: [Python-checkins] [3.12] GH-107987: Remove the Distributing Python Modules guide (GH-108016) (#108081) Message-ID: https://github.com/python/cpython/commit/25763030074e678b7e2d17a868355131f52d9a6b commit: 25763030074e678b7e2d17a868355131f52d9a6b branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-17T12:50:46+02:00 summary: [3.12] GH-107987: Remove the Distributing Python Modules guide (GH-108016) (#108081) GH-107987: Remove the Distributing Python Modules guide (GH-108016) (cherry picked from commit 33e6e3fec02ff3035dec52692542d3dd10124bef) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/conf.py M Doc/contents.rst M Doc/distributing/index.rst M Doc/installing/index.rst M Doc/tutorial/venv.rst diff --git a/Doc/conf.py b/Doc/conf.py index 8224b248062f4..c1a7ab5225707 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -283,8 +283,6 @@ latex_documents = [ ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), - ('distributing/index', 'distributing.tex', - 'Distributing Python Modules', _stdauthor, 'manual'), ('extending/index', 'extending.tex', 'Extending and Embedding Python', _stdauthor, 'manual'), ('installing/index', 'installing.tex', diff --git a/Doc/contents.rst b/Doc/contents.rst index 464f93bdf85f9..59cc541d4cb99 100644 --- a/Doc/contents.rst +++ b/Doc/contents.rst @@ -11,7 +11,6 @@ library/index.rst extending/index.rst c-api/index.rst - distributing/index.rst installing/index.rst howto/index.rst faq/index.rst diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index d237f8f082d87..2430564b45d6d 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -1,174 +1,19 @@ +:orphan: + +.. This page is retained solely for existing links to /distributing/index.html. + Direct readers to the PPUG instead. + .. _distributing-index: ############################### Distributing Python Modules ############################### -:Email: distutils-sig at python.org - - -As a popular open source development project, Python has an active -supporting community of contributors and users that also make their software -available for other Python developers to use under open source license terms. - -This allows Python users to share and collaborate effectively, benefiting -from the solutions others have already created to common (and sometimes -even rare!) problems, as well as potentially contributing their own -solutions to the common pool. - -This guide covers the distribution part of the process. For a guide to -installing other Python projects, refer to the -:ref:`installation guide `. - .. note:: - For corporate and other institutional users, be aware that many - organisations have their own policies around using and contributing to - open source software. Please take such policies into account when making - use of the distribution and installation tools provided with Python. - - -Key terms -========= - -* the `Python Package Index `__ is a public - repository of open source licensed packages made available for use by - other Python users -* the `Python Packaging Authority - `__ are the group of - developers and documentation authors responsible for the maintenance and - evolution of the standard packaging tools and the associated metadata and - file format standards. They maintain a variety of tools, documentation - and issue trackers on `GitHub `__. -* ``distutils`` is the original build and distribution system first added - to the Python standard library in 1998. While direct use of ``distutils`` - is being phased out, it still laid the foundation for the current packaging - and distribution infrastructure, and it not only remains part of the - standard library, but its name lives on in other ways (such as the name - of the mailing list used to coordinate Python packaging standards - development). -* `setuptools`_ is a (largely) drop-in replacement for ``distutils`` first - published in 2004. Its most notable addition over the unmodified - ``distutils`` tools was the ability to declare dependencies on other - packages. It is currently recommended as a more regularly updated - alternative to ``distutils`` that offers consistent support for more - recent packaging standards across a wide range of Python versions. -* `wheel`_ (in this context) is a project that adds the ``bdist_wheel`` - command to ``distutils``/`setuptools`_. This produces a cross platform - binary packaging format (called "wheels" or "wheel files" and defined in - :pep:`427`) that allows Python libraries, even those including binary - extensions, to be installed on a system without needing to be built - locally. - -.. _setuptools: https://setuptools.readthedocs.io/en/latest/ -.. _wheel: https://wheel.readthedocs.io/ - -Open source licensing and collaboration -======================================= - -In most parts of the world, software is automatically covered by copyright. -This means that other developers require explicit permission to copy, use, -modify and redistribute the software. - -Open source licensing is a way of explicitly granting such permission in a -relatively consistent way, allowing developers to share and collaborate -efficiently by making common solutions to various problems freely available. -This leaves many developers free to spend more time focusing on the problems -that are relatively unique to their specific situation. - -The distribution tools provided with Python are designed to make it -reasonably straightforward for developers to make their own contributions -back to that common pool of software if they choose to do so. - -The same distribution tools can also be used to distribute software within -an organisation, regardless of whether that software is published as open -source software or not. - - -Installing the tools -==================== - -The standard library does not include build tools that support modern -Python packaging standards, as the core development team has found that it -is important to have standard tools that work consistently, even on older -versions of Python. - -The currently recommended build and distribution tools can be installed -by invoking the ``pip`` module at the command line:: - - python -m pip install setuptools wheel twine - -.. note:: - - For POSIX users (including macOS and Linux users), these instructions - assume the use of a :term:`virtual environment`. - - For Windows users, these instructions assume that the option to - adjust the system PATH environment variable was selected when installing - Python. - -The Python Packaging User Guide includes more details on the `currently -recommended tools`_. - -.. _currently recommended tools: https://packaging.python.org/guides/tool-recommendations/#packaging-tool-recommendations - -.. index:: - single: Python Package Index (PyPI) - single: PyPI; (see Python Package Index (PyPI)) - -.. _publishing-python-packages: - -Reading the Python Packaging User Guide -======================================= - -The Python Packaging User Guide covers the various key steps and elements -involved in creating and publishing a project: - -* `Project structure`_ -* `Building and packaging the project`_ -* `Uploading the project to the Python Package Index`_ -* `The .pypirc file`_ - -.. _Project structure: https://packaging.python.org/tutorials/packaging-projects/#packaging-python-projects -.. _Building and packaging the project: https://packaging.python.org/tutorials/packaging-projects/#creating-the-package-files -.. _Uploading the project to the Python Package Index: https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives -.. _The .pypirc file: https://packaging.python.org/specifications/pypirc/ - - -How do I...? -============ - -These are quick answers or links for some common tasks. - -... choose a name for my project? ---------------------------------- - -This isn't an easy topic, but here are a few tips: - -* check the Python Package Index to see if the name is already in use -* check popular hosting sites like GitHub, Bitbucket, etc to see if there - is already a project with that name -* check what comes up in a web search for the name you're considering -* avoid particularly common words, especially ones with multiple meanings, - as they can make it difficult for users to find your software when - searching for it - - -... create and distribute binary extensions? --------------------------------------------- - -This is actually quite a complex topic, with a variety of alternatives -available depending on exactly what you're aiming to achieve. See the -Python Packaging User Guide for more information and recommendations. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions - `__ - -.. other topics: + Information and guidance on distributing Python modules and packages + has been moved to the `Python Packaging User Guide`_, + and the tutorial on `packaging Python projects`_. - Once the Development & Deployment part of PPUG is fleshed out, some of - those sections should be linked from new questions here (most notably, - we should have a question about avoiding depending on PyPI that links to - https://packaging.python.org/en/latest/mirrors/) + .. _Python Packaging User Guide: https://packaging.python.org/ + .. _packaging Python projects: https://packaging.python.org/en/latest/tutorials/packaging-projects/ diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index 5aec5178d48f3..a46c1caefe4d8 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -19,7 +19,9 @@ solutions to the common pool. This guide covers the installation part of the process. For a guide to creating and sharing your own Python projects, refer to the -:ref:`distribution guide `. +`Python packaging user guide`_. + +.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/packaging-projects/ .. note:: diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index d1bba098d7d23..a6dead2eac11f 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -207,4 +207,6 @@ necessary packages with ``install -r``: ``pip`` has many more options. Consult the :ref:`installing-index` guide for complete documentation for ``pip``. When you've written a package and want to make it available on the Python Package Index, -consult the :ref:`distributing-index` guide. +consult the `Python packaging user guide`_. + +.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/packaging-projects/ From webhook-mailer at python.org Thu Aug 17 06:51:10 2023 From: webhook-mailer at python.org (Yhg1s) Date: Thu, 17 Aug 2023 10:51:10 -0000 Subject: [Python-checkins] [3.12] gh-108000: Test that `lambda` also has `__type_params__` (GH-108002) (#108019) Message-ID: https://github.com/python/cpython/commit/e0244e85d09a3d2a40211c587891d908c35d277c commit: e0244e85d09a3d2a40211c587891d908c35d277c branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-17T12:51:07+02:00 summary: [3.12] gh-108000: Test that `lambda` also has `__type_params__` (GH-108002) (#108019) gh-108000: Test that `lambda` also has `__type_params__` (GH-108002) (cherry picked from commit a8d440b3837273926af5ce996162b019290ddad5) Co-authored-by: Nikita Sobolev files: M Lib/test/test_funcattrs.py diff --git a/Lib/test/test_funcattrs.py b/Lib/test/test_funcattrs.py index e08d72877d8ae..35b473d5e9a0b 100644 --- a/Lib/test/test_funcattrs.py +++ b/Lib/test/test_funcattrs.py @@ -194,16 +194,19 @@ def test___qualname__(self): def test___type_params__(self): def generic[T](): pass def not_generic(): pass + lambda_ = lambda: ... T, = generic.__type_params__ self.assertIsInstance(T, typing.TypeVar) self.assertEqual(generic.__type_params__, (T,)) - self.assertEqual(not_generic.__type_params__, ()) - with self.assertRaises(TypeError): - del not_generic.__type_params__ - with self.assertRaises(TypeError): - not_generic.__type_params__ = 42 - not_generic.__type_params__ = (T,) - self.assertEqual(not_generic.__type_params__, (T,)) + for func in (not_generic, lambda_): + with self.subTest(func=func): + self.assertEqual(func.__type_params__, ()) + with self.assertRaises(TypeError): + del func.__type_params__ + with self.assertRaises(TypeError): + func.__type_params__ = 42 + func.__type_params__ = (T,) + self.assertEqual(func.__type_params__, (T,)) def test___code__(self): num_one, num_two = 7, 8 From webhook-mailer at python.org Thu Aug 17 06:52:12 2023 From: webhook-mailer at python.org (Yhg1s) Date: Thu, 17 Aug 2023 10:52:12 -0000 Subject: [Python-checkins] [3.12] gh-107298: Add standard exceptions and warnings in the nitpick_ignore list (GH-108029) (#108070) Message-ID: https://github.com/python/cpython/commit/125aab99ab3d11eaca8e4248548856a077355391 commit: 125aab99ab3d11eaca8e4248548856a077355391 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-17T12:52:08+02:00 summary: [3.12] gh-107298: Add standard exceptions and warnings in the nitpick_ignore list (GH-108029) (#108070) gh-107298: Add standard exceptions and warnings in the nitpick_ignore list (GH-108029) (cherry picked from commit c9d83f93d804b80ee14480466ebee63a6f97dac2) Co-authored-by: Serhiy Storchaka files: M Doc/conf.py diff --git a/Doc/conf.py b/Doc/conf.py index c1a7ab5225707..a8805bd4653a8 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -157,6 +157,77 @@ ('envvar', 'USER'), ('envvar', 'USERNAME'), ('envvar', 'USERPROFILE'), +] + +# Temporary undocumented names. +# In future this list must be empty. +nitpick_ignore += [ + # C API: Standard Python exception classes + ('c:data', 'PyExc_ArithmeticError'), + ('c:data', 'PyExc_AssertionError'), + ('c:data', 'PyExc_AttributeError'), + ('c:data', 'PyExc_BaseException'), + ('c:data', 'PyExc_BlockingIOError'), + ('c:data', 'PyExc_BrokenPipeError'), + ('c:data', 'PyExc_BufferError'), + ('c:data', 'PyExc_ChildProcessError'), + ('c:data', 'PyExc_ConnectionAbortedError'), + ('c:data', 'PyExc_ConnectionError'), + ('c:data', 'PyExc_ConnectionRefusedError'), + ('c:data', 'PyExc_ConnectionResetError'), + ('c:data', 'PyExc_EOFError'), + ('c:data', 'PyExc_Exception'), + ('c:data', 'PyExc_FileExistsError'), + ('c:data', 'PyExc_FileNotFoundError'), + ('c:data', 'PyExc_FloatingPointError'), + ('c:data', 'PyExc_GeneratorExit'), + ('c:data', 'PyExc_ImportError'), + ('c:data', 'PyExc_IndentationError'), + ('c:data', 'PyExc_IndexError'), + ('c:data', 'PyExc_InterruptedError'), + ('c:data', 'PyExc_IsADirectoryError'), + ('c:data', 'PyExc_KeyboardInterrupt'), + ('c:data', 'PyExc_KeyError'), + ('c:data', 'PyExc_LookupError'), + ('c:data', 'PyExc_MemoryError'), + ('c:data', 'PyExc_ModuleNotFoundError'), + ('c:data', 'PyExc_NameError'), + ('c:data', 'PyExc_NotADirectoryError'), + ('c:data', 'PyExc_NotImplementedError'), + ('c:data', 'PyExc_OSError'), + ('c:data', 'PyExc_OverflowError'), + ('c:data', 'PyExc_PermissionError'), + ('c:data', 'PyExc_ProcessLookupError'), + ('c:data', 'PyExc_RecursionError'), + ('c:data', 'PyExc_ReferenceError'), + ('c:data', 'PyExc_RuntimeError'), + ('c:data', 'PyExc_StopAsyncIteration'), + ('c:data', 'PyExc_StopIteration'), + ('c:data', 'PyExc_SyntaxError'), + ('c:data', 'PyExc_SystemError'), + ('c:data', 'PyExc_SystemExit'), + ('c:data', 'PyExc_TabError'), + ('c:data', 'PyExc_TimeoutError'), + ('c:data', 'PyExc_TypeError'), + ('c:data', 'PyExc_UnboundLocalError'), + ('c:data', 'PyExc_UnicodeDecodeError'), + ('c:data', 'PyExc_UnicodeEncodeError'), + ('c:data', 'PyExc_UnicodeError'), + ('c:data', 'PyExc_UnicodeTranslateError'), + ('c:data', 'PyExc_ValueError'), + ('c:data', 'PyExc_ZeroDivisionError'), + # C API: Standard Python warning classes + ('c:data', 'PyExc_BytesWarning'), + ('c:data', 'PyExc_DeprecationWarning'), + ('c:data', 'PyExc_FutureWarning'), + ('c:data', 'PyExc_ImportWarning'), + ('c:data', 'PyExc_PendingDeprecationWarning'), + ('c:data', 'PyExc_ResourceWarning'), + ('c:data', 'PyExc_RuntimeWarning'), + ('c:data', 'PyExc_SyntaxWarning'), + ('c:data', 'PyExc_UnicodeWarning'), + ('c:data', 'PyExc_UserWarning'), + ('c:data', 'PyExc_Warning'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. From webhook-mailer at python.org Thu Aug 17 06:52:52 2023 From: webhook-mailer at python.org (Yhg1s) Date: Thu, 17 Aug 2023 10:52:52 -0000 Subject: [Python-checkins] [3.12] gh-107298: Fix some references in the C API documentation (GH-108072) (#108074) Message-ID: https://github.com/python/cpython/commit/41c951b62f9c08b5965ca670e660955f95a8cdf5 commit: 41c951b62f9c08b5965ca670e660955f95a8cdf5 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-17T12:52:48+02:00 summary: [3.12] gh-107298: Fix some references in the C API documentation (GH-108072) (#108074) gh-107298: Fix some references in the C API documentation (GH-108072) (cherry picked from commit f51f0466c07eabc6177c2f64f70c952dada050e8) Co-authored-by: Serhiy Storchaka files: M Doc/c-api/typeobj.rst M Doc/extending/extending.rst M Doc/using/configure.rst M Doc/whatsnew/2.2.rst M Doc/whatsnew/2.3.rst M Doc/whatsnew/2.4.rst M Doc/whatsnew/2.6.rst M Doc/whatsnew/2.7.rst M Doc/whatsnew/3.0.rst M Doc/whatsnew/3.1.rst M Doc/whatsnew/3.2.rst diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index d394ce10504b0..cd037b4de882e 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1697,7 +1697,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) to a pointer, are valid C99 address constants. However, the unary '&' operator applied to a non-static variable - like :c:func:`PyBaseObject_Type` is not required to produce an address + like :c:data:`PyBaseObject_Type` is not required to produce an address constant. Compilers may support this (gcc does), MSVC does not. Both compilers are strictly standard conforming in this particular behavior. diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 72e30a6882807..68f8e0c667465 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -242,7 +242,7 @@ needed to ensure that it will not be discarded, causing :c:data:`!SpamError` to become a dangling pointer. Should it become a dangling pointer, C code which raises the exception could cause a core dump or other unintended side effects. -We discuss the use of ``PyMODINIT_FUNC`` as a function return type later in this +We discuss the use of :c:macro:`PyMODINIT_FUNC` as a function return type later in this sample. The :exc:`!spam.error` exception can be raised in your extension module using a @@ -363,7 +363,7 @@ only non-\ ``static`` item defined in the module file:: return PyModule_Create(&spammodule); } -Note that PyMODINIT_FUNC declares the function as ``PyObject *`` return type, +Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type, declares any special linkage declarations required by the platform, and for C++ declares the function as ``extern "C"``. diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 441d346a1a38a..f4adea82a87c3 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -746,8 +746,8 @@ Example on Linux x86-64:: At the beginning of the files, C extensions are built as built-in modules. Extensions defined after the ``*shared*`` marker are built as dynamic libraries. -The :c:macro:`PyAPI_FUNC()`, :c:macro:`PyAPI_API()` and -:c:macro:`PyMODINIT_FUNC()` macros of :file:`Include/pyport.h` are defined +The :c:macro:`PyAPI_FUNC()`, :c:macro:`PyAPI_DATA()` and +:c:macro:`PyMODINIT_FUNC` macros of :file:`Include/pyport.h` are defined differently depending if the ``Py_BUILD_CORE_MODULE`` macro is defined: * Use ``Py_EXPORTED_SYMBOL`` if the ``Py_BUILD_CORE_MODULE`` is defined diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 44e9bd8d492bf..7de48a4026303 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -1109,7 +1109,7 @@ code, none of the changes described here will affect you very much. definition tables to simplify implementation of methods with no arguments or a single untyped argument. Calling such methods is more efficient than calling a corresponding method that uses :c:macro:`METH_VARARGS`. Also, the old - :c:macro:`METH_OLDARGS` style of writing C methods is now officially deprecated. + :c:macro:`!METH_OLDARGS` style of writing C methods is now officially deprecated. * Two new wrapper functions, :c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf` were added to provide cross-platform implementations for the relatively new diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index 3dfe509e1d1c0..a9c44d501701c 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1886,7 +1886,7 @@ Changes to Python's build process and to the C API include: (:file:`libpython2.3.so`) by supplying :option:`!--enable-shared` when running Python's :program:`configure` script. (Contributed by Ondrej Palkovsky.) -* The :c:macro:`DL_EXPORT` and :c:macro:`DL_IMPORT` macros are now deprecated. +* The :c:macro:`!DL_EXPORT` and :c:macro:`!DL_IMPORT` macros are now deprecated. Initialization functions for Python extension modules should now be declared using the new macro :c:macro:`PyMODINIT_FUNC`, while the Python core will generally use the :c:macro:`PyAPI_FUNC` and :c:macro:`PyAPI_DATA` macros. diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index a00c4360b9c68..4272928fb6789 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -1476,7 +1476,7 @@ Some of the changes to Python's build process and to the C API are: :c:func:`PyArg_ParseTupleAndKeywords` but takes a :c:type:`va_list` instead of a number of arguments. (Contributed by Greg Chapman.) -* A new method flag, :c:macro:`METH_COEXISTS`, allows a function defined in slots +* A new method flag, :c:macro:`METH_COEXIST`, allows a function defined in slots to co-exist with a :c:type:`PyCFunction` having the same name. This can halve the access time for a method such as :meth:`set.__contains__`. (Contributed by Raymond Hettinger.) diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 2b8fa1546a588..563fa461d2a86 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -3060,9 +3060,9 @@ Changes to Python's build process and to the C API include: * Some macros were renamed in both 3.0 and 2.6 to make it clearer that they are macros, - not functions. :c:macro:`Py_Size()` became :c:macro:`Py_SIZE()`, - :c:macro:`Py_Type()` became :c:macro:`Py_TYPE()`, and - :c:macro:`Py_Refcnt()` became :c:macro:`Py_REFCNT()`. + not functions. :c:macro:`!Py_Size()` became :c:macro:`Py_SIZE()`, + :c:macro:`!Py_Type()` became :c:macro:`Py_TYPE()`, and + :c:macro:`!Py_Refcnt()` became :c:macro:`Py_REFCNT()`. The mixed-case macros are still available in Python 2.6 for backward compatibility. (:issue:`1629`) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index cf8b00fd30979..76beb4d746a19 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2287,10 +2287,10 @@ object, and then get the ``void *`` pointer, which will usually point to an array of pointers to the module's various API functions. There is an existing data type already used for this, -:c:type:`PyCObject`, but it doesn't provide type safety. Evil code +:c:type:`!PyCObject`, but it doesn't provide type safety. Evil code written in pure Python could cause a segmentation fault by taking a -:c:type:`PyCObject` from module A and somehow substituting it for the -:c:type:`PyCObject` in module B. Capsules know their own name, +:c:type:`!PyCObject` from module A and somehow substituting it for the +:c:type:`!PyCObject` in module B. Capsules know their own name, and getting the pointer requires providing the name: .. code-block:: c @@ -2310,10 +2310,10 @@ detect the mismatched name and return false. Refer to :ref:`using-capsules` for more information on using these objects. Python 2.7 now uses capsules internally to provide various -extension-module APIs, but the :c:func:`PyCObject_AsVoidPtr` was +extension-module APIs, but the :c:func:`!PyCObject_AsVoidPtr` was modified to handle capsules, preserving compile-time compatibility -with the :c:type:`CObject` interface. Use of -:c:func:`PyCObject_AsVoidPtr` will signal a +with the :c:type:`!PyCObject` interface. Use of +:c:func:`!PyCObject_AsVoidPtr` will signal a :exc:`PendingDeprecationWarning`, which is silent by default. Implemented in Python 3.1 and backported to 2.7 by Larry Hastings; diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 379c74981fbf0..4f8b05efd4464 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -875,7 +875,7 @@ to the C API. * Renamed the boolean conversion C-level slot and method: ``nb_nonzero`` is now ``nb_bool``. -* Removed :c:macro:`METH_OLDARGS` and :c:macro:`WITH_CYCLE_GC` from the C API. +* Removed :c:macro:`!METH_OLDARGS` and :c:macro:`!WITH_CYCLE_GC` from the C API. .. ====================================================================== diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index e4365d44928b5..ceef622f9f8da 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -510,7 +510,7 @@ Changes to Python's build process and to the C API include: (Contributed by Mark Dickinson; :issue:`5914`.) -* Added :c:type:`PyCapsule` as a replacement for the :c:type:`PyCObject` API. +* Added :c:type:`PyCapsule` as a replacement for the :c:type:`!PyCObject` API. The principal difference is that the new type has a well defined interface for passing typing safety information and a less complicated signature for calling a destructor. The old type had a problematic API and is now diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index ec4d8a0312b6b..ec554aacb096c 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -2658,7 +2658,7 @@ require changes to your code: * "t#" format has been removed: use "s#" or "s*" instead * "w" and "w#" formats has been removed: use "w*" instead -* The :c:type:`PyCObject` type, deprecated in 3.1, has been removed. To wrap +* The :c:type:`!PyCObject` type, deprecated in 3.1, has been removed. To wrap opaque C pointers in Python objects, the :c:type:`PyCapsule` API should be used instead; the new type has a well-defined interface for passing typing safety information and a less complicated signature for calling a destructor. From webhook-mailer at python.org Thu Aug 17 06:53:30 2023 From: webhook-mailer at python.org (Yhg1s) Date: Thu, 17 Aug 2023 10:53:30 -0000 Subject: [Python-checkins] [3.12] GH-92584: Remove Installing Python Modules (Distutils version) (GH-108020) (#108062) Message-ID: https://github.com/python/cpython/commit/560e8595cb34b995310157c6e7a0ca2ecf559885 commit: 560e8595cb34b995310157c6e7a0ca2ecf559885 branch: 3.12 author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: Yhg1s date: 2023-08-17T12:53:26+02:00 summary: [3.12] GH-92584: Remove Installing Python Modules (Distutils version) (GH-108020) (#108062) * [3.12] GH-92584: Remove Installing Python Modules (Distutils version) (GH-108020). (cherry picked from commit fbb7cbc0e92168077fd56de942901511e99ca60a) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: D Doc/install/index.rst M Doc/contents.rst M Doc/extending/building.rst M Doc/library/site.rst M Doc/using/cmdline.rst diff --git a/Doc/contents.rst b/Doc/contents.rst index 59cc541d4cb99..24ceacb0076b5 100644 --- a/Doc/contents.rst +++ b/Doc/contents.rst @@ -20,10 +20,3 @@ bugs.rst copyright.rst license.rst - -.. to include legacy packaging docs in build - -.. toctree:: - :hidden: - - install/index.rst diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst index 880bb33ee5671..ddde567f6f3ef 100644 --- a/Doc/extending/building.rst +++ b/Doc/extending/building.rst @@ -45,6 +45,7 @@ See the *"Multiple modules in one library"* section in :pep:`489` for details. .. highlight:: c +.. _install-index: .. _setuptools-index: Building C and C++ Extensions with setuptools diff --git a/Doc/install/index.rst b/Doc/install/index.rst deleted file mode 100644 index ffb4a202fe89f..0000000000000 --- a/Doc/install/index.rst +++ /dev/null @@ -1,1081 +0,0 @@ -.. highlight:: none - -.. _install-index: - -******************************************** - Installing Python Modules (Legacy version) -******************************************** - -:Author: Greg Ward - -.. TODO: Fill in XXX comments - -.. note:: - - The entire ``distutils`` package has been deprecated and will be - removed in Python 3.12. This documentation is retained as a - reference only, and will be removed with the package. See the - :ref:`What's New ` entry for more information. - -.. seealso:: - - :ref:`installing-index` - The up to date module installation documentation. For regular Python - usage, you almost certainly want that document rather than this one. - -.. note:: - - This document is being retained solely until the ``setuptools`` documentation - at https://setuptools.readthedocs.io/en/latest/setuptools.html - independently covers all of the relevant information currently included here. - -.. note:: - - This guide only covers the basic tools for building and distributing - extensions that are provided as part of this version of Python. Third party - tools offer easier to use and more secure alternatives. Refer to the `quick - recommendations section `__ - in the Python Packaging User Guide for more information. - - -.. _inst-intro: - - -Introduction -============ - -In Python 2.0, the ``distutils`` API was first added to the standard library. -This provided Linux distro maintainers with a standard way of converting -Python projects into Linux distro packages, and system administrators with a -standard way of installing them directly onto target systems. - -In the many years since Python 2.0 was released, tightly coupling the build -system and package installer to the language runtime release cycle has turned -out to be problematic, and it is now recommended that projects use the -``pip`` package installer and the ``setuptools`` build system, rather than -using ``distutils`` directly. - -See :ref:`installing-index` and :ref:`distributing-index` for more details. - -This legacy documentation is being retained only until we're confident that the -``setuptools`` documentation covers everything needed. - -.. _inst-new-standard: - -Distutils based source distributions ------------------------------------- - -If you download a module source distribution, you can tell pretty quickly if it -was packaged and distributed in the standard way, i.e. using the Distutils. -First, the distribution's name and version number will be featured prominently -in the name of the downloaded archive, e.g. :file:`foo-1.0.tar.gz` or -:file:`widget-0.9.7.zip`. Next, the archive will unpack into a similarly named -directory: :file:`foo-1.0` or :file:`widget-0.9.7`. Additionally, the -distribution will contain a setup script :file:`setup.py`, and a file named -:file:`README.txt` or possibly just :file:`README`, which should explain that -building and installing the module distribution is a simple matter of running -one command from a terminal:: - - python setup.py install - -For Windows, this command should be run from a command prompt window -(:menuselection:`Start --> Accessories`):: - - setup.py install - -If all these things are true, then you already know how to build and install the -modules you've just downloaded: Run the command above. Unless you need to -install things in a non-standard way or customize the build process, you don't -really need this manual. Or rather, the above command is everything you need to -get out of this manual. - - -.. _inst-standard-install: - -Standard Build and Install -========================== - -As described in section :ref:`inst-new-standard`, building and installing a module -distribution using the Distutils is usually one simple command to run from a -terminal:: - - python setup.py install - - -.. _inst-platform-variations: - -Platform variations -------------------- - -You should always run the setup command from the distribution root directory, -i.e. the top-level subdirectory that the module source distribution unpacks -into. For example, if you've just downloaded a module source distribution -:file:`foo-1.0.tar.gz` onto a Unix system, the normal thing to do is:: - - gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0 - cd foo-1.0 - python setup.py install - -On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the -archive file to :file:`C:\\Temp`, then it would unpack into -:file:`C:\\Temp\\foo-1.0`; you can use either an archive manipulator with a -graphical user interface (such as WinZip) or a command-line tool (such as -:program:`unzip` or :program:`pkunzip`) to unpack the archive. Then, open a -command prompt window and run:: - - cd c:\Temp\foo-1.0 - python setup.py install - - -.. _inst-splitting-up: - -Splitting the job up --------------------- - -Running ``setup.py install`` builds and installs all modules in one run. If you -prefer to work incrementally---especially useful if you want to customize the -build process, or if things are going wrong---you can use the setup script to do -one thing at a time. This is particularly helpful when the build and install -will be done by different users---for example, you might want to build a module -distribution and hand it off to a system administrator for installation (or do -it yourself, with super-user privileges). - -For example, you can build everything in one step, and then install everything -in a second step, by invoking the setup script twice:: - - python setup.py build - python setup.py install - -If you do this, you will notice that running the :command:`install` command -first runs the :command:`build` command, which---in this case---quickly notices -that it has nothing to do, since everything in the :file:`build` directory is -up-to-date. - -You may not need this ability to break things down often if all you do is -install modules downloaded off the 'net, but it's very handy for more advanced -tasks. If you get into distributing your own Python modules and extensions, -you'll run lots of individual Distutils commands on their own. - - -.. _inst-how-build-works: - -How building works ------------------- - -As implied above, the :command:`build` command is responsible for putting the -files to install into a *build directory*. By default, this is :file:`build` -under the distribution root; if you're excessively concerned with speed, or want -to keep the source tree pristine, you can change the build directory with the -:option:`!--build-base` option. For example:: - - python setup.py build --build-base=/path/to/pybuild/foo-1.0 - -(Or you could do this permanently with a directive in your system or personal -Distutils configuration file; see section :ref:`inst-config-files`.) Normally, this -isn't necessary. - -The default layout for the build tree is as follows:: - - --- build/ --- lib/ - or - --- build/ --- lib./ - temp./ - -where ```` expands to a brief description of the current OS/hardware -platform and Python version. The first form, with just a :file:`lib` directory, -is used for "pure module distributions"---that is, module distributions that -include only pure Python modules. If a module distribution contains any -extensions (modules written in C/C++), then the second form, with two ```` -directories, is used. In that case, the :file:`temp.{plat}` directory holds -temporary files generated by the compile/link process that don't actually get -installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory -contains all Python modules (pure Python and extensions) that will be installed. - -In the future, more directories will be added to handle Python scripts, -documentation, binary executables, and whatever else is needed to handle the job -of installing Python modules and applications. - - -.. _inst-how-install-works: - -How installation works ----------------------- - -After the :command:`build` command runs (whether you run it explicitly, or the -:command:`install` command does it for you), the work of the :command:`install` -command is relatively simple: all it has to do is copy everything under -:file:`build/lib` (or :file:`build/lib.{plat}`) to your chosen installation -directory. - -If you don't choose an installation directory---i.e., if you just run ``setup.py -install``\ ---then the :command:`install` command installs to the standard -location for third-party Python modules. This location varies by platform and -by how you built/installed Python itself. On Unix (and macOS, which is also -Unix-based), it also depends on whether the module distribution being installed -is pure Python or contains extensions ("non-pure"): - -.. tabularcolumns:: |l|l|l|l| - -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Platform | Standard installation location | Default value | Notes | -+=================+=====================================================+==================================================+=======+ -| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ - -Notes: - -(1) - Most Linux distributions include Python as a standard part of the system, so - :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on - Linux. If you build Python yourself on Linux (or any Unix-like system), the - default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`. - -(2) - The default installation directory on Windows was :file:`C:\\Program - Files\\Python` under Python 1.6a1, 1.5.2, and earlier. - -:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python -is installed to, and where it finds its libraries at run-time. They are always -the same under Windows, and very often the same under Unix and macOS. You -can find out what your Python installation uses for :file:`{prefix}` and -:file:`{exec-prefix}` by running Python in interactive mode and typing a few -simple commands. Under Unix, just type ``python`` at the shell prompt. Under -Windows, choose :menuselection:`Start --> Programs --> Python X.Y --> -Python (command line)`. Once the interpreter is started, you type Python code -at the prompt. For example, on my Linux system, I type the three Python -statements shown below, and get the output as shown, to find out my -:file:`{prefix}` and :file:`{exec-prefix}`: - -.. code-block:: pycon - - Python 2.4 (#26, Aug 7 2004, 17:19:02) - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.prefix - '/usr' - >>> sys.exec_prefix - '/usr' - -A few other placeholders are used in this document: :file:`{X.Y}` stands for the -version of Python, for example ``3.2``; :file:`{abiflags}` will be replaced by -the value of :data:`sys.abiflags` or the empty string for platforms which don't -define ABI flags; :file:`{distname}` will be replaced by the name of the module -distribution being installed. Dots and capitalization are important in the -paths; for example, a value that uses ``python3.2`` on UNIX will typically use -``Python32`` on Windows. - -If you don't want to install modules to the standard location, or if you don't -have permission to write there, then you need to read about alternate -installations in section :ref:`inst-alt-install`. If you want to customize your -installation directories more heavily, see section :ref:`inst-custom-install` on -custom installations. - - -.. _inst-alt-install: - -Alternate Installation -====================== - -Often, it is necessary or desirable to install modules to a location other than -the standard location for third-party Python modules. For example, on a Unix -system you might not have permission to write to the standard third-party module -directory. Or you might wish to try out a module before making it a standard -part of your local Python installation. This is especially true when upgrading -a distribution already present: you want to make sure your existing base of -scripts still works with the new version before actually upgrading. - -The Distutils :command:`install` command is designed to make installing module -distributions to an alternate location simple and painless. The basic idea is -that you supply a base directory for the installation, and the -:command:`install` command picks a set of directories (called an *installation -scheme*) under this base directory in which to install files. The details -differ across platforms, so read whichever of the following sections applies to -you. - -Note that the various alternate installation schemes are mutually exclusive: you -can pass ``--user``, or ``--home``, or ``--prefix`` and ``--exec-prefix``, or -``--install-base`` and ``--install-platbase``, but you can't mix from these -groups. - - -.. _inst-alt-install-user: - -Alternate installation: the user scheme ---------------------------------------- - -This scheme is designed to be the most convenient solution for users that don't -have write permission to the global site-packages directory or don't want to -install into it. It is enabled with a simple option:: - - python setup.py install --user - -Files will be installed into subdirectories of :const:`site.USER_BASE` (written -as :file:`{userbase}` hereafter). This scheme installs pure Python modules and -extension modules in the same location (also known as :const:`site.USER_SITE`). -Here are the values for UNIX, including macOS: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{userbase}/lib/python{X.Y}/site-packages` -scripts :file:`{userbase}/bin` -data :file:`{userbase}` -C headers :file:`{userbase}/include/python{X.Y}{abiflags}/{distname}` -=============== =========================================================== - -And here are the values used on Windows: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{userbase}\\Python{XY}\\site-packages` -scripts :file:`{userbase}\\Python{XY}\\Scripts` -data :file:`{userbase}` -C headers :file:`{userbase}\\Python{XY}\\Include\\{distname}` -=============== =========================================================== - -The advantage of using this scheme compared to the other ones described below is -that the user site-packages directory is under normal conditions always included -in :data:`sys.path` (see :mod:`site` for more information), which means that -there is no additional step to perform after running the :file:`setup.py` script -to finalize the installation. - -The :command:`build_ext` command also has a ``--user`` option to add -:file:`{userbase}/include` to the compiler search path for header files and -:file:`{userbase}/lib` to the compiler search path for libraries as well as to -the runtime search path for shared C libraries (rpath). - - -.. _inst-alt-install-home: - -Alternate installation: the home scheme ---------------------------------------- - -The idea behind the "home scheme" is that you build and maintain a personal -stash of Python modules. This scheme's name is derived from the idea of a -"home" directory on Unix, since it's not unusual for a Unix user to make their -home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`. -This scheme can be used by anyone, regardless of the operating system they -are installing for. - -Installing a new module distribution is as simple as :: - - python setup.py install --home= - -where you can supply any directory you like for the :option:`!--home` option. On -Unix, lazy typists can just type a tilde (``~``); the :command:`install` command -will expand this to your home directory:: - - python setup.py install --home=~ - -To make Python find the distributions installed with this scheme, you may have -to :ref:`modify Python's search path ` or edit -:mod:`!sitecustomize` (see :mod:`site`) to call :func:`site.addsitedir` or edit -:data:`sys.path`. - -The :option:`!--home` option defines the installation base directory. Files are -installed to the following directories under the installation base as follows: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{home}/lib/python` -scripts :file:`{home}/bin` -data :file:`{home}` -C headers :file:`{home}/include/python/{distname}` -=============== =========================================================== - -(Mentally replace slashes with backslashes if you're on Windows.) - - -.. _inst-alt-install-prefix-unix: - -Alternate installation: Unix (the prefix scheme) ------------------------------------------------- - -The "prefix scheme" is useful when you wish to use one Python installation to -perform the build/install (i.e., to run the setup script), but install modules -into the third-party module directory of a different Python installation (or -something that looks like a different Python installation). If this sounds a -trifle unusual, it is---that's why the user and home schemes come before. However, -there are at least two known cases where the prefix scheme will be useful. - -First, consider that many Linux distributions put Python in :file:`/usr`, rather -than the more traditional :file:`/usr/local`. This is entirely appropriate, -since in those cases Python is part of "the system" rather than a local add-on. -However, if you are installing Python modules from source, you probably want -them to go in :file:`/usr/local/lib/python2.{X}` rather than -:file:`/usr/lib/python2.{X}`. This can be done with :: - - /usr/bin/python setup.py install --prefix=/usr/local - -Another possibility is a network filesystem where the name used to write to a -remote directory is different from the name used to read it: for example, the -Python interpreter accessed as :file:`/usr/local/bin/python` might search for -modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to -be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could -be done with :: - - /usr/local/bin/python setup.py install --prefix=/mnt/@server/export - -In either case, the :option:`!--prefix` option defines the installation base, and -the :option:`!--exec-prefix` option defines the platform-specific installation -base, which is used for platform-specific files. (Currently, this just means -non-pure module distributions, but could be expanded to C libraries, binary -executables, etc.) If :option:`!--exec-prefix` is not supplied, it defaults to -:option:`!--prefix`. Files are installed as follows: - -================= ========================================================== -Type of file Installation directory -================= ========================================================== -Python modules :file:`{prefix}/lib/python{X.Y}/site-packages` -extension modules :file:`{exec-prefix}/lib/python{X.Y}/site-packages` -scripts :file:`{prefix}/bin` -data :file:`{prefix}` -C headers :file:`{prefix}/include/python{X.Y}{abiflags}/{distname}` -================= ========================================================== - -There is no requirement that :option:`!--prefix` or :option:`!--exec-prefix` -actually point to an alternate Python installation; if the directories listed -above do not already exist, they are created at installation time. - -Incidentally, the real reason the prefix scheme is important is simply that a -standard Unix installation uses the prefix scheme, but with :option:`!--prefix` -and :option:`!--exec-prefix` supplied by Python itself as ``sys.prefix`` and -``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme, -but every time you run ``python setup.py install`` without any other options, -you're using it. - -Note that installing extensions to an alternate Python installation has no -effect on how those extensions are built: in particular, the Python header files -(:file:`Python.h` and friends) installed with the Python interpreter used to run -the setup script will be used in compiling extensions. It is your -responsibility to ensure that the interpreter used to run extensions installed -in this way is compatible with the interpreter used to build them. The best way -to do this is to ensure that the two interpreters are the same version of Python -(possibly different builds, or possibly copies of the same build). (Of course, -if your :option:`!--prefix` and :option:`!--exec-prefix` don't even point to an -alternate Python installation, this is immaterial.) - - -.. _inst-alt-install-prefix-windows: - -Alternate installation: Windows (the prefix scheme) ---------------------------------------------------- - -Windows has no concept of a user's home directory, and since the standard Python -installation under Windows is simpler than under Unix, the :option:`!--prefix` -option has traditionally been used to install additional packages in separate -locations on Windows. :: - - python setup.py install --prefix="\Temp\Python" - -to install modules to the :file:`\\Temp\\Python` directory on the current drive. - -The installation base is defined by the :option:`!--prefix` option; the -:option:`!--exec-prefix` option is not supported under Windows, which means that -pure Python modules and extension modules are installed into the same location. -Files are installed as follows: - -=============== ========================================================== -Type of file Installation directory -=============== ========================================================== -modules :file:`{prefix}\\Lib\\site-packages` -scripts :file:`{prefix}\\Scripts` -data :file:`{prefix}` -C headers :file:`{prefix}\\Include\\{distname}` -=============== ========================================================== - - -.. _inst-custom-install: - -Custom Installation -=================== - -Sometimes, the alternate installation schemes described in section -:ref:`inst-alt-install` just don't do what you want. You might want to tweak just -one or two directories while keeping everything under the same base directory, -or you might want to completely redefine the installation scheme. In either -case, you're creating a *custom installation scheme*. - -To create a custom installation scheme, you start with one of the alternate -schemes and override some of the installation directories used for the various -types of files, using these options: - -====================== ======================= -Type of file Override option -====================== ======================= -Python modules ``--install-purelib`` -extension modules ``--install-platlib`` -all modules ``--install-lib`` -scripts ``--install-scripts`` -data ``--install-data`` -C headers ``--install-headers`` -====================== ======================= - -These override options can be relative, absolute, -or explicitly defined in terms of one of the installation base directories. -(There are two installation base directories, and they are normally the -same---they only differ when you use the Unix "prefix scheme" and supply -different ``--prefix`` and ``--exec-prefix`` options; using ``--install-lib`` -will override values computed or given for ``--install-purelib`` and -``--install-platlib``, and is recommended for schemes that don't make a -difference between Python and extension modules.) - -For example, say you're installing a module distribution to your home directory -under Unix---but you want scripts to go in :file:`~/scripts` rather than -:file:`~/bin`. As you might expect, you can override this directory with the -:option:`!--install-scripts` option; in this case, it makes most sense to supply -a relative path, which will be interpreted relative to the installation base -directory (your home directory, in this case):: - - python setup.py install --home=~ --install-scripts=scripts - -Another Unix example: suppose your Python installation was built and installed -with a prefix of :file:`/usr/local/python`, so under a standard installation -scripts will wind up in :file:`/usr/local/python/bin`. If you want them in -:file:`/usr/local/bin` instead, you would supply this absolute directory for the -:option:`!--install-scripts` option:: - - python setup.py install --install-scripts=/usr/local/bin - -(This performs an installation using the "prefix scheme", where the prefix is -whatever your Python interpreter was installed with--- :file:`/usr/local/python` -in this case.) - -If you maintain Python on Windows, you might want third-party modules to live in -a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}` -itself. This is almost as easy as customizing the script installation -directory---you just have to remember that there are two types of modules -to worry about, Python and extension modules, which can conveniently be both -controlled by one option:: - - python setup.py install --install-lib=Site - -The specified installation directory is relative to :file:`{prefix}`. Of -course, you also have to ensure that this directory is in Python's module -search path, such as by putting a :file:`.pth` file in a site directory (see -:mod:`site`). See section :ref:`inst-search-path` to find out how to modify -Python's search path. - -If you want to define an entire installation scheme, you just have to supply all -of the installation directory options. The recommended way to do this is to -supply relative paths; for example, if you want to maintain all Python -module-related files under :file:`python` in your home directory, and you want a -separate directory for each platform that you use your home directory from, you -might define the following installation scheme:: - - python setup.py install --home=~ \ - --install-purelib=python/lib \ - --install-platlib=python/lib.$PLAT \ - --install-scripts=python/scripts - --install-data=python/data - -or, equivalently, :: - - python setup.py install --home=~/python \ - --install-purelib=lib \ - --install-platlib='lib.$PLAT' \ - --install-scripts=scripts - --install-data=data - -``$PLAT`` is not (necessarily) an environment variable---it will be expanded by -the Distutils as it parses your command line options, just as it does when -parsing your configuration file(s). - -Obviously, specifying the entire installation scheme every time you install a -new module distribution would be very tedious. Thus, you can put these options -into your Distutils config file (see section :ref:`inst-config-files`): - -.. code-block:: ini - - [install] - install-base=$HOME - install-purelib=python/lib - install-platlib=python/lib.$PLAT - install-scripts=python/scripts - install-data=python/data - -or, equivalently, - -.. code-block:: ini - - [install] - install-base=$HOME/python - install-purelib=lib - install-platlib=lib.$PLAT - install-scripts=scripts - install-data=data - -Note that these two are *not* equivalent if you supply a different installation -base directory when you run the setup script. For example, :: - - python setup.py install --install-base=/tmp - -would install pure modules to :file:`/tmp/python/lib` in the first case, and -to :file:`/tmp/lib` in the second case. (For the second case, you probably -want to supply an installation base of :file:`/tmp/python`.) - -You probably noticed the use of ``$HOME`` and ``$PLAT`` in the sample -configuration file input. These are Distutils configuration variables, which -bear a strong resemblance to environment variables. In fact, you can use -environment variables in config files on platforms that have such a notion but -the Distutils additionally define a few extra variables that may not be in your -environment, such as ``$PLAT``. (And of course, on systems that don't have -environment variables, such as Mac OS 9, the configuration variables supplied by -the Distutils are the only ones you can use.) See section :ref:`inst-config-files` -for details. - -.. note:: When a :ref:`virtual environment ` is activated, any options - that change the installation path will be ignored from all distutils configuration - files to prevent inadvertently installing projects outside of the virtual - environment. - -.. XXX need some Windows examples---when would custom installation schemes be - needed on those platforms? - - -.. XXX Move this to Doc/using - -.. _inst-search-path: - -Modifying Python's Search Path ------------------------------- - -When the Python interpreter executes an :keyword:`import` statement, it searches -for both Python code and extension modules along a search path. A default value -for the path is configured into the Python binary when the interpreter is built. -You can determine the path by importing the :mod:`sys` module and printing the -value of ``sys.path``. :: - - $ python - Python 2.2 (#11, Oct 3 2002, 13:31:27) - [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2 - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.path - ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2', - '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload', - '/usr/local/lib/python2.3/site-packages'] - >>> - -The null string in ``sys.path`` represents the current working directory. - -The expected convention for locally installed packages is to put them in the -:file:`{...}/site-packages/` directory, but you may want to install Python -modules into some arbitrary directory. For example, your site may have a -convention of keeping all software related to the web server under :file:`/www`. -Add-on Python modules might then belong in :file:`/www/python`, and in order to -import them, this directory must be added to ``sys.path``. There are several -different ways to add the directory. - -The most convenient way is to add a path configuration file to a directory -that's already on Python's path, usually to the :file:`.../site-packages/` -directory. Path configuration files have an extension of :file:`.pth`, and each -line must contain a single path that will be appended to ``sys.path``. (Because -the new paths are appended to ``sys.path``, modules in the added directories -will not override standard modules. This means you can't use this mechanism for -installing fixed versions of standard modules.) - -Paths can be absolute or relative, in which case they're relative to the -directory containing the :file:`.pth` file. See the documentation of -the :mod:`site` module for more information. - -A slightly less convenient way is to edit the :file:`site.py` file in Python's -standard library, and modify ``sys.path``. :file:`site.py` is automatically -imported when the Python interpreter is executed, unless the :option:`-S` switch -is supplied to suppress this behaviour. So you could simply edit -:file:`site.py` and add two lines to it: - -.. code-block:: python - - import sys - sys.path.append('/www/python/') - -However, if you reinstall the same minor version of Python (perhaps when -upgrading from 2.2 to 2.2.2, for example) :file:`site.py` will be overwritten by -the stock version. You'd have to remember that it was modified and save a copy -before doing the installation. - -There are two environment variables that can modify ``sys.path``. -:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python -installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``, -the search path will be set to ``['', '/www/python/lib/pythonX.Y/', -'/www/python/lib/pythonX.Y/plat-linux2', ...]``. - -The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be -added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is -set to ``/www/python:/opt/py``, the search path will begin with -``['/www/python', '/opt/py']``. (Note that directories must exist in order to -be added to ``sys.path``; the :mod:`site` module removes paths that don't -exist.) - -Finally, ``sys.path`` is just a regular Python list, so any Python application -can modify it by adding or removing entries. - - -.. _inst-config-files: - -Distutils Configuration Files -============================= - -As mentioned above, you can use Distutils configuration files to record personal -or site preferences for any Distutils options. That is, any option to any -command can be stored in one of two or three (depending on your platform) -configuration files, which will be consulted before the command-line is parsed. -This means that configuration files will override default values, and the -command-line will in turn override configuration files. Furthermore, if -multiple configuration files apply, values from "earlier" files are overridden -by "later" files. - - -.. _inst-config-filenames: - -Location and names of config files ----------------------------------- - -The names and locations of the configuration files vary slightly across -platforms. On Unix and macOS, the three configuration files (in the order -they are processed) are: - -+--------------+----------------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+==========================================================+=======+ -| system | :file:`{prefix}/lib/python{ver}/distutils/distutils.cfg` | \(1) | -+--------------+----------------------------------------------------------+-------+ -| personal | :file:`$HOME/.pydistutils.cfg` | \(2) | -+--------------+----------------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+----------------------------------------------------------+-------+ - -And on Windows, the configuration files are: - -+--------------+-------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+=================================================+=======+ -| system | :file:`{prefix}\\Lib\\distutils\\distutils.cfg` | \(4) | -+--------------+-------------------------------------------------+-------+ -| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) | -+--------------+-------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+-------------------------------------------------+-------+ - -On all platforms, the "personal" file can be temporarily disabled by -passing the ``--no-user-cfg`` option. - -Notes: - -(1) - Strictly speaking, the system-wide configuration file lives in the directory - where the Distutils are installed; under Python 1.6 and later on Unix, this is - as shown. For Python 1.5.2, the Distutils will normally be installed to - :file:`{prefix}/lib/python1.5/site-packages/distutils`, so the system - configuration file should be put there under Python 1.5.2. - -(2) - On Unix, if the :envvar:`HOME` environment variable is not defined, the user's - home directory will be determined with the :func:`~pwd.getpwuid` function from the - standard :mod:`pwd` module. This is done by the :func:`os.path.expanduser` - function used by Distutils. - -(3) - I.e., in the current directory (usually the location of the setup script). - -(4) - (See also note (1).) Under Python 1.6 and later, Python's default "installation - prefix" is :file:`C:\\Python`, so the system configuration file is normally - :file:`C:\\Python\\Lib\\distutils\\distutils.cfg`. Under Python 1.5.2, the - default prefix was :file:`C:\\Program Files\\Python`, and the Distutils were not - part of the standard library---so the system configuration file would be - :file:`C:\\Program Files\\Python\\distutils\\distutils.cfg` in a standard Python - 1.5.2 installation under Windows. - -(5) - On Windows, if the :envvar:`HOME` environment variable is not defined, - :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will - be tried. This is done by the :func:`os.path.expanduser` function used - by Distutils. - - -.. _inst-config-syntax: - -Syntax of config files ----------------------- - -The Distutils configuration files all have the same syntax. The config files -are grouped into sections. There is one section for each Distutils command, -plus a ``global`` section for global options that affect every command. Each -section consists of one option per line, specified as ``option=value``. - -For example, the following is a complete config file that just forces all -commands to run quietly by default: - -.. code-block:: ini - - [global] - verbose=0 - -If this is installed as the system config file, it will affect all processing of -any Python module distribution by any user on the current system. If it is -installed as your personal config file (on systems that support them), it will -affect only module distributions processed by you. And if it is used as the -:file:`setup.cfg` for a particular module distribution, it affects only that -distribution. - -You could override the default "build base" directory and make the -:command:`build\*` commands always forcibly rebuild all files with the -following: - -.. code-block:: ini - - [build] - build-base=blib - force=1 - -which corresponds to the command-line arguments :: - - python setup.py build --build-base=blib --force - -except that including the :command:`build` command on the command-line means -that command will be run. Including a particular command in config files has no -such implication; it only means that if the command is run, the options in the -config file will apply. (Or if other commands that derive values from it are -run, they will use the values in the config file.) - -You can find out the complete list of options for any command using the -:option:`!--help` option, e.g.:: - - python setup.py build --help - -and you can find out the complete list of global options by using -:option:`!--help` without a command:: - - python setup.py --help - -See also the "Reference" section of the "Distributing Python Modules" manual. - - -.. _inst-building-ext: - -Building Extensions: Tips and Tricks -==================================== - -Whenever possible, the Distutils try to use the configuration information made -available by the Python interpreter used to run the :file:`setup.py` script. -For example, the same compiler and linker flags used to compile Python will also -be used for compiling extensions. Usually this will work well, but in -complicated situations this might be inappropriate. This section discusses how -to override the usual Distutils behaviour. - - -.. _inst-tweak-flags: - -Tweaking compiler/linker flags ------------------------------- - -Compiling a Python extension written in C or C++ will sometimes require -specifying custom flags for the compiler and linker in order to use a particular -library or produce a special kind of object code. This is especially true if the -extension hasn't been tested on your platform, or if you're trying to -cross-compile Python. - -In the most general case, the extension author might have foreseen that -compiling the extensions would be complicated, and provided a :file:`Setup` file -for you to edit. This will likely only be done if the module distribution -contains many separate extension modules, or if they often require elaborate -sets of compiler flags in order to work. - -A :file:`Setup` file, if present, is parsed in order to get a list of extensions -to build. Each line in a :file:`Setup` describes a single module. Lines have -the following structure:: - - module ... [sourcefile ...] [cpparg ...] [library ...] - - -Let's examine each of the fields in turn. - -* *module* is the name of the extension module to be built, and should be a - valid Python identifier. You can't just change this in order to rename a module - (edits to the source code would also be needed), so this should be left alone. - -* *sourcefile* is anything that's likely to be a source code file, at least - judging by the filename. Filenames ending in :file:`.c` are assumed to be - written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are - assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed - to be in Objective C. - -* *cpparg* is an argument for the C preprocessor, and is anything starting with - :option:`!-I`, :option:`!-D`, :option:`!-U` or :option:`!-C`. - -* *library* is anything ending in :file:`.a` or beginning with :option:`!-l` or - :option:`!-L`. - -If a particular platform requires a special library on your platform, you can -add it by editing the :file:`Setup` file and running ``python setup.py build``. -For example, if the module defined by the line :: - - foo foomodule.c - -must be linked with the math library :file:`libm.a` on your platform, simply add -:option:`!-lm` to the line:: - - foo foomodule.c -lm - -Arbitrary switches intended for the compiler or the linker can be supplied with -the :option:`!-Xcompiler` *arg* and :option:`!-Xlinker` *arg* options:: - - foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm - -The next option after :option:`!-Xcompiler` and :option:`!-Xlinker` will be -appended to the proper command line, so in the above example the compiler will -be passed the :option:`!-o32` option, and the linker will be passed -:option:`!-shared`. If a compiler option requires an argument, you'll have to -supply multiple :option:`!-Xcompiler` options; for example, to pass ``-x c++`` -the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``. - -Compiler flags can also be supplied through setting the :envvar:`CFLAGS` -environment variable. If set, the contents of :envvar:`CFLAGS` will be added to -the compiler flags specified in the :file:`Setup` file. - - -.. _inst-non-ms-compilers: - -Using non-Microsoft compilers on Windows ----------------------------------------- - -.. sectionauthor:: Rene Liebscher - - - -Borland/CodeGear C++ -^^^^^^^^^^^^^^^^^^^^ - -This subsection describes the necessary steps to use Distutils with the Borland -C++ compiler version 5.5. First you have to know that Borland's object file -format (OMF) is different from the format used by the Python version you can -download from the Python or ActiveState web site. (Python is built with -Microsoft Visual C++, which uses COFF as the object file format.) For this -reason you have to convert Python's library :file:`python25.lib` into the -Borland format. You can do this as follows: - -.. Should we mention that users have to create cfg-files for the compiler? -.. see also http://community.borland.com/article/0,1410,21205,00.html - -:: - - coff2omf python25.lib python25_bcpp.lib - -The :file:`coff2omf` program comes with the Borland compiler. The file -:file:`python25.lib` is in the :file:`Libs` directory of your Python -installation. If your extension uses other libraries (zlib, ...) you have to -convert them too. - -The converted files have to reside in the same directories as the normal -libraries. - -How does Distutils manage to use these libraries with their changed names? If -the extension needs a library (eg. :file:`foo`) Distutils checks first if it -finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then -uses this library. In the case it doesn't find such a special library it uses -the default name (:file:`foo.lib`.) [#]_ - -To let Distutils compile your extension with Borland C++ you now have to type:: - - python setup.py build --compiler=bcpp - -If you want to use the Borland C++ compiler as the default, you could specify -this in your personal or system-wide configuration file for Distutils (see -section :ref:`inst-config-files`.) - - -.. seealso:: - - `C++Builder Compiler `_ - Information about the free C++ compiler from Borland, including links to the - download pages. - - `Creating Python Extensions Using Borland's Free Compiler `_ - Document describing how to use Borland's free command-line C++ compiler to build - Python. - - -GNU C / Cygwin / MinGW -^^^^^^^^^^^^^^^^^^^^^^ - -This section describes the necessary steps to use Distutils with the GNU C/C++ -compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter -that was built with Cygwin, everything should work without any of these -following steps. - -Not all extensions can be built with MinGW or Cygwin, but many can. Extensions -most likely to not work are those that use C++ or depend on Microsoft Visual C -extensions. - -To let Distutils compile your extension with Cygwin you have to type:: - - python setup.py build --compiler=cygwin - -and for Cygwin in no-cygwin mode [#]_ or for MinGW type:: - - python setup.py build --compiler=mingw32 - -If you want to use any of these options/compilers as default, you should -consider writing it in your personal or system-wide configuration file for -Distutils (see section :ref:`inst-config-files`.) - -Older Versions of Python and MinGW -"""""""""""""""""""""""""""""""""" -The following instructions only apply if you're using a version of Python -inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with -binutils-2.13.90-20030111-1). - -These compilers require some special libraries. This task is more complex than -for Borland's C++, because there is no program to convert the library. First -you have to create a list of symbols which the Python DLL exports. (You can find -a good program for this task at -https://sourceforge.net/projects/mingw/files/MinGW/Extension/pexports/). - -.. I don't understand what the next line means. --amk -.. (inclusive the references on data structures.) - -:: - - pexports python25.dll >python25.def - -The location of an installed :file:`python25.dll` will depend on the -installation options and the version and language of Windows. In a "just for -me" installation, it will appear in the root of the installation directory. In -a shared installation, it will be located in the system directory. - -Then you can create from these information an import library for gcc. :: - - /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a - -The resulting library has to be placed in the same directory as -:file:`python25.lib`. (Should be the :file:`libs` directory under your Python -installation directory.) - -If your extension uses other libraries (zlib,...) you might have to convert -them too. The converted files have to reside in the same directories as the -normal libraries do. - - -.. seealso:: - - `Building Python modules on MS Windows platform with MinGW `_ - Information about building the required libraries for the MinGW environment. - - -.. rubric:: Footnotes - -.. [#] This also means you could replace all existing COFF-libraries with OMF-libraries - of the same name. - -.. [#] Check https://www.sourceware.org/cygwin/ for more information - -.. [#] Then you have no POSIX emulation available, but you also don't need - :file:`cygwin1.dll`. diff --git a/Doc/library/site.rst b/Doc/library/site.rst index ff9e9107f9d16..ea3b2e996574e 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -191,7 +191,7 @@ Module contents :file:`~/Library/Python/{X.Y}` for macOS framework builds, and :file:`{%APPDATA%}\\Python` for Windows. This value is used to compute the installation directories for scripts, data files, Python modules, - etc. for the :ref:`user installation scheme `. + etc. for the user installation scheme. See also :envvar:`PYTHONUSERBASE`. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 23c89400f152b..921b6a6961c7b 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -811,7 +811,7 @@ conflict. Defines the :data:`user base directory `, which is used to compute the path of the :data:`user site-packages directory ` - and :ref:`installation paths ` for + and installation paths for ``python -m pip install --user``. .. seealso:: From webhook-mailer at python.org Thu Aug 17 06:54:03 2023 From: webhook-mailer at python.org (Yhg1s) Date: Thu, 17 Aug 2023 10:54:03 -0000 Subject: [Python-checkins] [3.12] gh-107909: Test explicit `object` base in PEP695 generic classes (GH-108001) (#108022) Message-ID: https://github.com/python/cpython/commit/2166a407b29d7dc944e6d9159a4f6b8ffb6dc5e6 commit: 2166a407b29d7dc944e6d9159a4f6b8ffb6dc5e6 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-17T12:53:59+02:00 summary: [3.12] gh-107909: Test explicit `object` base in PEP695 generic classes (GH-108001) (#108022) gh-107909: Test explicit `object` base in PEP695 generic classes (GH-108001) (cherry picked from commit b61f5995aebb93496e968ca8d307375fa86d9329) Co-authored-by: Nikita Sobolev files: M Lib/test/test_type_params.py diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index bced641a9661f..0045057f181e1 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -148,6 +148,10 @@ def test_disallowed_expressions(self): check_syntax_error(self, "def f[T: [(x := 3) for _ in range(2)]](): pass") check_syntax_error(self, "type T = [(x := 3) for _ in range(2)]") + def test_incorrect_mro_explicit_object(self): + with self.assertRaisesRegex(TypeError, r"\(MRO\) for bases object, Generic"): + class My[X](object): ... + class TypeParamsNonlocalTest(unittest.TestCase): def test_nonlocal_disallowed_01(self): From webhook-mailer at python.org Thu Aug 17 09:57:52 2023 From: webhook-mailer at python.org (jaraco) Date: Thu, 17 Aug 2023 13:57:52 -0000 Subject: [Python-checkins] [3.11] gh-102215: importlib documentation cleanups (#108056) Message-ID: https://github.com/python/cpython/commit/b3f7ddd040d7ef8226cf2a23d88bbda1d0e5d242 commit: b3f7ddd040d7ef8226cf2a23d88bbda1d0e5d242 branch: 3.11 author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: jaraco date: 2023-08-17T09:57:48-04:00 summary: [3.11] gh-102215: importlib documentation cleanups (#108056) (cherry picked from commit 4cd95dce0b8d7bb8a16468ec8b5b3429555417f1) Co-authored-by: Sam Morris files: M Doc/library/importlib.metadata.rst M Doc/library/importlib.resources.rst diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index a768e934583bc..3566f7468f108 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -1,11 +1,11 @@ .. _using: -================================= - Using :mod:`!importlib.metadata` -================================= +======================================================== +:mod:`!importlib.metadata` -- Accessing package metadata +======================================================== .. module:: importlib.metadata - :synopsis: The implementation of the importlib metadata. + :synopsis: Accessing package metadata .. versionadded:: 3.8 .. versionchanged:: 3.10 @@ -13,7 +13,7 @@ **Source code:** :source:`Lib/importlib/metadata/__init__.py` -``importlib_metadata`` is a library that provides access to +``importlib.metadata`` is a library that provides access to the metadata of an installed `Distribution Package `_, such as its entry points or its top-level names (`Import Package `_\s, modules, if any). @@ -24,7 +24,7 @@ API`_ and `metadata API`_ of ``pkg_resources``. Along with this package can eliminate the need to use the older and less efficient ``pkg_resources`` package. -``importlib_metadata`` operates on third-party *distribution packages* +``importlib.metadata`` operates on third-party *distribution packages* installed into Python's ``site-packages`` directory via tools such as `pip `_. Specifically, it works with distributions with discoverable @@ -365,7 +365,7 @@ system `finders`_. To find a distribution package's metadata, ``importlib.metadata`` queries the list of :term:`meta path finders ` on :data:`sys.meta_path`. -By default ``importlib_metadata`` installs a finder for distribution packages +By default ``importlib.metadata`` installs a finder for distribution packages found on the file system. This finder doesn't actually find any *distributions*, but it can find their metadata. diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 827e7d8d5aced..437da6927ab2f 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -1,5 +1,5 @@ -:mod:`importlib.resources` -- Resources ---------------------------------------- +:mod:`importlib.resources` -- Package resource reading, opening and access +-------------------------------------------------------------------------- .. module:: importlib.resources :synopsis: Package resource reading, opening, and access @@ -79,7 +79,7 @@ for example, a package and its resources can be imported from a zip file using .. versionadded:: 3.9 Deprecated functions --------------------- +^^^^^^^^^^^^^^^^^^^^ An older, deprecated set of functions is still available, but is scheduled for removal in a future version of Python. From webhook-mailer at python.org Thu Aug 17 10:14:05 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Thu, 17 Aug 2023 14:14:05 -0000 Subject: [Python-checkins] gh-107801: Document SEEK_HOLE and SEEK_DATA (#107936) Message-ID: https://github.com/python/cpython/commit/8a19f225b948db1eebe1d9fc71a486258841f578 commit: 8a19f225b948db1eebe1d9fc71a486258841f578 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-17T16:14:01+02:00 summary: gh-107801: Document SEEK_HOLE and SEEK_DATA (#107936) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Antoine Pitrou files: M Doc/library/os.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 9735baa5bc0f3..94b76e4052b94 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1171,6 +1171,10 @@ as internal buffering of data. current position; :const:`SEEK_END` or ``2`` to set it relative to the end of the file. Return the new cursor position in bytes, starting from the beginning. + .. versionchanged:: 3.3 + + Add support for :const:`SEEK_HOLE` and :const:`SEEK_DATA`. + .. data:: SEEK_SET SEEK_CUR @@ -1179,9 +1183,30 @@ as internal buffering of data. Parameters to the :func:`lseek` function. Their values are 0, 1, and 2, respectively. + +.. data:: SEEK_HOLE + SEEK_DATA + + Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` + method on file objects, for seeking file data and holes on sparsely + allocated files. + + :data:`!SEEK_DATA` + Adjust the file offset to the next location containing data, + relative to the seek position. + + :data:`!SEEK_HOLE` + Adjust the file offset to the next location containing a hole, + relative to the seek position. + A hole is defined as a sequence of zeros. + + .. note:: + + These operations only make sense for filesystems that support them. + + .. availability:: Linux >= 3.1, macOS, Unix + .. versionadded:: 3.3 - Some operating systems could support additional values, like - :const:`os.SEEK_HOLE` or :const:`os.SEEK_DATA`. .. function:: open(path, flags, mode=0o777, *, dir_fd=None) From webhook-mailer at python.org Thu Aug 17 10:22:43 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Thu, 17 Aug 2023 14:22:43 -0000 Subject: [Python-checkins] [3.11] gh-107801: Document SEEK_HOLE and SEEK_DATA (GH-107936) (#108087) Message-ID: https://github.com/python/cpython/commit/1482b99061e1d7d4a49721d6290d7e48de8291c0 commit: 1482b99061e1d7d4a49721d6290d7e48de8291c0 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: erlend-aasland date: 2023-08-17T14:22:39Z summary: [3.11] gh-107801: Document SEEK_HOLE and SEEK_DATA (GH-107936) (#108087) (cherry picked from commit 8a19f225b948db1eebe1d9fc71a486258841f578) Co-authored-by: Erlend E. Aasland Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Antoine Pitrou files: M Doc/library/os.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 5577921478452..1d0d1049ebd7e 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1061,6 +1061,10 @@ as internal buffering of data. current position; :const:`SEEK_END` or ``2`` to set it relative to the end of the file. Return the new cursor position in bytes, starting from the beginning. + .. versionchanged:: 3.3 + + Add support for :const:`SEEK_HOLE` and :const:`SEEK_DATA`. + .. data:: SEEK_SET SEEK_CUR @@ -1069,9 +1073,30 @@ as internal buffering of data. Parameters to the :func:`lseek` function. Their values are 0, 1, and 2, respectively. + +.. data:: SEEK_HOLE + SEEK_DATA + + Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` + method on file objects, for seeking file data and holes on sparsely + allocated files. + + :data:`!SEEK_DATA` + Adjust the file offset to the next location containing data, + relative to the seek position. + + :data:`!SEEK_HOLE` + Adjust the file offset to the next location containing a hole, + relative to the seek position. + A hole is defined as a sequence of zeros. + + .. note:: + + These operations only make sense for filesystems that support them. + + .. availability:: Linux >= 3.1, macOS, Unix + .. versionadded:: 3.3 - Some operating systems could support additional values, like - :const:`os.SEEK_HOLE` or :const:`os.SEEK_DATA`. .. function:: open(path, flags, mode=0o777, *, dir_fd=None) From webhook-mailer at python.org Thu Aug 17 11:37:11 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Thu, 17 Aug 2023 15:37:11 -0000 Subject: [Python-checkins] Add workflow for automatic issue headers (#108054) Message-ID: https://github.com/python/cpython/commit/4cb08188e8f0faaec303e268e12aa1d6f54017f7 commit: 4cb08188e8f0faaec303e268e12aa1d6f54017f7 branch: main author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: AlexWaygood date: 2023-08-17T16:37:07+01:00 summary: Add workflow for automatic issue headers (#108054) We don't get the "Bug report" and "Feature or enhancement" titles anymore, with the new issue forms. This brings them back! files: A .github/workflows/add-issue-header.yml diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml new file mode 100644 index 0000000000000..1ef9178b95e5f --- /dev/null +++ b/.github/workflows/add-issue-header.yml @@ -0,0 +1,53 @@ +name: Add issue header +# Automatically edits an issue's descriptions with a header, +# one of: +# +# - Bug report +# - Crash report +# - Feature or enhancement + +on: + issues: + types: + # Only ever run once + - opened + + +jobs: + add-header: + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - uses: actions/github-script at v6 + with: + # language=JavaScript + script: | + // https://devguide.python.org/triage/labels/#type-labels + const HEADERS = new Map([ + ['type-bug', 'Bug report'], + ['type-crash', 'Crash report'], + ['type-feature', 'Feature or enhancement'], + ]); + let issue_data = await github.rest.issues.get({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo + }).then(issue => issue.data); + let header = ''; + for (const label_data of issue_data.labels) { + const label_name = (typeof label_data === 'string') ? label_data : label_data.name; + if (HEADERS.has(label_name)) { + header = HEADERS.get(label_name); + break; + } + } + if (header !== '') { + console.log(`Setting new header: ${header}`); + await github.rest.issues.update({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `# ${header}\n\n${issue_data.body.replaceAll('\r', '')}` + }); + } From webhook-mailer at python.org Thu Aug 17 12:08:02 2023 From: webhook-mailer at python.org (iritkatriel) Date: Thu, 17 Aug 2023 16:08:02 -0000 Subject: [Python-checkins] gh-105481: opcode.h is no longer generated during the build (#108080) Message-ID: https://github.com/python/cpython/commit/0b243c2f665e6784b74ac4d602d4ee429a2ad7f0 commit: 0b243c2f665e6784b74ac4d602d4ee429a2ad7f0 branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-17T17:07:58+01:00 summary: gh-105481: opcode.h is no longer generated during the build (#108080) files: M Include/opcode.h M Lib/opcode.py M Makefile.pre.in M Modules/_opcode.c M Modules/clinic/_opcode.c.h M PCbuild/regen.targets M Tools/build/generate_opcode_h.py diff --git a/Include/opcode.h b/Include/opcode.h index e5c42d5a71828..2619b690019ac 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -1,5 +1,3 @@ -// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py - #ifndef Py_OPCODE_H #define Py_OPCODE_H #ifdef __cplusplus @@ -36,6 +34,7 @@ extern "C" { #define NB_INPLACE_TRUE_DIVIDE 24 #define NB_INPLACE_XOR 25 +#define NB_OPARG_LAST 25 #ifdef __cplusplus } diff --git a/Lib/opcode.py b/Lib/opcode.py index 6b9d9ce811a61..f8487522bfdc6 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -44,39 +44,10 @@ _intrinsic_1_descs = _opcode.get_intrinsic1_descs() _intrinsic_2_descs = _opcode.get_intrinsic2_descs() + _nb_ops = _opcode.get_nb_ops() hascompare = [opmap["COMPARE_OP"]] -_nb_ops = [ - ("NB_ADD", "+"), - ("NB_AND", "&"), - ("NB_FLOOR_DIVIDE", "//"), - ("NB_LSHIFT", "<<"), - ("NB_MATRIX_MULTIPLY", "@"), - ("NB_MULTIPLY", "*"), - ("NB_REMAINDER", "%"), - ("NB_OR", "|"), - ("NB_POWER", "**"), - ("NB_RSHIFT", ">>"), - ("NB_SUBTRACT", "-"), - ("NB_TRUE_DIVIDE", "/"), - ("NB_XOR", "^"), - ("NB_INPLACE_ADD", "+="), - ("NB_INPLACE_AND", "&="), - ("NB_INPLACE_FLOOR_DIVIDE", "//="), - ("NB_INPLACE_LSHIFT", "<<="), - ("NB_INPLACE_MATRIX_MULTIPLY", "@="), - ("NB_INPLACE_MULTIPLY", "*="), - ("NB_INPLACE_REMAINDER", "%="), - ("NB_INPLACE_OR", "|="), - ("NB_INPLACE_POWER", "**="), - ("NB_INPLACE_RSHIFT", ">>="), - ("NB_INPLACE_SUBTRACT", "-="), - ("NB_INPLACE_TRUE_DIVIDE", "/="), - ("NB_INPLACE_XOR", "^="), -] - - _cache_format = { "LOAD_GLOBAL": { "counter": 1, diff --git a/Makefile.pre.in b/Makefile.pre.in index bcec0782f6e95..9be5c3b50eb9e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1427,13 +1427,11 @@ regen-ast: .PHONY: regen-opcode regen-opcode: - # Regenerate Include/opcode.h from Lib/opcode.py + # Regenerate Include/internal/pycore_opcode.h from Lib/opcode.py # using Tools/build/generate_opcode_h.py $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \ $(srcdir)/Lib/opcode.py \ - $(srcdir)/Include/opcode.h.new \ $(srcdir)/Include/internal/pycore_opcode.h.new - $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new .PHONY: regen-token diff --git a/Modules/_opcode.c b/Modules/_opcode.c index ad0fa736f8276..4f85e7ee26de2 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -223,6 +223,75 @@ _opcode_get_specialization_stats_impl(PyObject *module) /*[clinic input] +_opcode.get_nb_ops + +Return array of symbols of binary ops. + +Indexed by the BINARY_OP oparg value. +[clinic start generated code]*/ + +static PyObject * +_opcode_get_nb_ops_impl(PyObject *module) +/*[clinic end generated code: output=d997d306cc15426f input=9462fc544c823176]*/ +{ + PyObject *list = PyList_New(NB_OPARG_LAST + 1); + if (list == NULL) { + return NULL; + } +#define ADD_NB_OP(NUM, STR) \ + do { \ + PyObject *pair = Py_BuildValue( \ + "NN", PyUnicode_FromString(#NUM), PyUnicode_FromString(STR)); \ + if (pair == NULL) { \ + Py_DECREF(list); \ + return NULL; \ + } \ + PyList_SET_ITEM(list, (NUM), pair); \ + } while(0); + + ADD_NB_OP(NB_ADD, "+"); + ADD_NB_OP(NB_AND, "&"); + ADD_NB_OP(NB_FLOOR_DIVIDE, "//"); + ADD_NB_OP(NB_LSHIFT, "<<"); + ADD_NB_OP(NB_MATRIX_MULTIPLY, "@"); + ADD_NB_OP(NB_MULTIPLY, "*"); + ADD_NB_OP(NB_REMAINDER, "%"); + ADD_NB_OP(NB_OR, "|"); + ADD_NB_OP(NB_POWER, "**"); + ADD_NB_OP(NB_RSHIFT, ">>"); + ADD_NB_OP(NB_SUBTRACT, "-"); + ADD_NB_OP(NB_TRUE_DIVIDE, "/"); + ADD_NB_OP(NB_XOR, "^"); + ADD_NB_OP(NB_INPLACE_ADD, "+="); + ADD_NB_OP(NB_INPLACE_AND, "&="); + ADD_NB_OP(NB_INPLACE_FLOOR_DIVIDE, "//="); + ADD_NB_OP(NB_INPLACE_LSHIFT, "<<="); + ADD_NB_OP(NB_INPLACE_MATRIX_MULTIPLY, "@="); + ADD_NB_OP(NB_INPLACE_MULTIPLY, "*="); + ADD_NB_OP(NB_INPLACE_REMAINDER, "%="); + ADD_NB_OP(NB_INPLACE_OR, "|="); + ADD_NB_OP(NB_INPLACE_POWER, "**="); + ADD_NB_OP(NB_INPLACE_RSHIFT, ">>="); + ADD_NB_OP(NB_INPLACE_SUBTRACT, "-="); + ADD_NB_OP(NB_INPLACE_TRUE_DIVIDE, "/="); + ADD_NB_OP(NB_INPLACE_XOR, "^="); + +#undef ADD_NB_OP + + for(int i = 0; i <= NB_OPARG_LAST; i++) { + if (PyList_GET_ITEM(list, i) == NULL) { + Py_DECREF(list); + PyErr_Format(PyExc_ValueError, + "Missing initialization for NB_OP %d", + i); + return NULL; + } + } + return list; +} + +/*[clinic input] + _opcode.get_intrinsic1_descs Return a list of names of the unary intrinsics. @@ -287,6 +356,7 @@ opcode_functions[] = { _OPCODE_HAS_LOCAL_METHODDEF _OPCODE_HAS_EXC_METHODDEF _OPCODE_GET_SPECIALIZATION_STATS_METHODDEF + _OPCODE_GET_NB_OPS_METHODDEF _OPCODE_GET_INTRINSIC1_DESCS_METHODDEF _OPCODE_GET_INTRINSIC2_DESCS_METHODDEF {NULL, NULL, 0, NULL} diff --git a/Modules/clinic/_opcode.c.h b/Modules/clinic/_opcode.c.h index e1fc5ba17f707..52fbdcbd2a652 100644 --- a/Modules/clinic/_opcode.c.h +++ b/Modules/clinic/_opcode.c.h @@ -613,6 +613,26 @@ _opcode_get_specialization_stats(PyObject *module, PyObject *Py_UNUSED(ignored)) return _opcode_get_specialization_stats_impl(module); } +PyDoc_STRVAR(_opcode_get_nb_ops__doc__, +"get_nb_ops($module, /)\n" +"--\n" +"\n" +"Return array of symbols of binary ops.\n" +"\n" +"Indexed by the BINARY_OP oparg value."); + +#define _OPCODE_GET_NB_OPS_METHODDEF \ + {"get_nb_ops", (PyCFunction)_opcode_get_nb_ops, METH_NOARGS, _opcode_get_nb_ops__doc__}, + +static PyObject * +_opcode_get_nb_ops_impl(PyObject *module); + +static PyObject * +_opcode_get_nb_ops(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _opcode_get_nb_ops_impl(module); +} + PyDoc_STRVAR(_opcode_get_intrinsic1_descs__doc__, "get_intrinsic1_descs($module, /)\n" "--\n" @@ -648,4 +668,4 @@ _opcode_get_intrinsic2_descs(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _opcode_get_intrinsic2_descs_impl(module); } -/*[clinic end generated code: output=d85de5f2887b3661 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=31a1a11c2f81dca4 input=a9049054013a1b77]*/ diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index ed02f9163549b..2ff18a8966f55 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -14,7 +14,7 @@ -C <_OpcodeSources Include="$(PySourcePath)Tools\build\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" /> - <_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Include\internal\pycore_opcode.h" /> + <_OpcodeOutputs Include="$(PySourcePath)Include\internal\pycore_opcode.h" /> <_TokenSources Include="$(PySourcePath)Grammar\Tokens" /> <_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc"> rst @@ -59,7 +59,7 @@ Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)" DependsOnTargets="FindPythonForBuild"> - diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 344709a05184c..643918c1bc2ce 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -1,4 +1,4 @@ -# This script generates the opcode.h header file. +# This script generates the pycore_opcode.h header file. import sys import tokenize @@ -6,27 +6,6 @@ SCRIPT_NAME = "Tools/build/generate_opcode_h.py" PYTHON_OPCODE = "Lib/opcode.py" -opcode_h_header = f""" -// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} - -#ifndef Py_OPCODE_H -#define Py_OPCODE_H -#ifdef __cplusplus -extern "C" {{ -#endif - -#include "opcode_ids.h" - -""".lstrip() - -opcode_h_footer = """ - -#ifdef __cplusplus -} -#endif -#endif /* !Py_OPCODE_H */ -""" - internal_header = f""" // Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} @@ -62,20 +41,10 @@ def get_python_module_dict(filename): return mod def main(opcode_py, - opcode_h='Include/opcode.h', internal_opcode_h='Include/internal/pycore_opcode.h'): opcode = get_python_module_dict(opcode_py) - with open(opcode_h, 'w') as fobj: - fobj.write(opcode_h_header) - - fobj.write("\n") - for i, (op, _) in enumerate(opcode["_nb_ops"]): - fobj.write(DEFINE.format(op, i)) - - fobj.write(opcode_h_footer) - with open(internal_opcode_h, 'w') as iobj: iobj.write(internal_header) @@ -91,8 +60,8 @@ def main(opcode_py, iobj.write(internal_footer) - print(f"{opcode_h} regenerated from {opcode_py}") + print(f"{internal_opcode_h} regenerated from {opcode_py}") if __name__ == '__main__': - main(sys.argv[1], sys.argv[2], sys.argv[3]) + main(sys.argv[1], sys.argv[2]) From webhook-mailer at python.org Thu Aug 17 12:19:11 2023 From: webhook-mailer at python.org (gpshead) Date: Thu, 17 Aug 2023 16:19:11 -0000 Subject: [Python-checkins] gh-102029: Deprecate passing arguments to `_PyRLock` in `threading` (#102071) Message-ID: https://github.com/python/cpython/commit/80f30cf51bd89411ef1220d714b33667d6a39901 commit: 80f30cf51bd89411ef1220d714b33667d6a39901 branch: main author: Nikita Sobolev committer: gpshead date: 2023-08-17T09:19:07-07:00 summary: gh-102029: Deprecate passing arguments to `_PyRLock` in `threading` (#102071) files: A Misc/NEWS.d/next/Library/2023-02-20-15-41-59.gh-issue-102029.9ZPG99.rst M Doc/whatsnew/3.13.rst M Lib/test/test_threading.py M Lib/threading.py diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 519dee5eb7d6c..13ae6e595dc99 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -357,6 +357,12 @@ Pending Removal in Python 3.15 They will be removed in Python 3.15. (Contributed by Victor Stinner in :gh:`105096`.) +* Passing any arguments to :func:`threading.RLock` is now deprecated. + C version allows any numbers of args and kwargs, + but they are just ignored. Python version does not allow any arguments. + All arguments will be removed from :func:`threading.RLock` in Python 3.15. + (Contributed by Nikita Sobolev in :gh:`102029`.) + Pending Removal in Python 3.16 ------------------------------ diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 4a91eef31c4b8..6465a44656584 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1748,6 +1748,30 @@ class PyRLockTests(lock_tests.RLockTests): class CRLockTests(lock_tests.RLockTests): locktype = staticmethod(threading._CRLock) + def test_signature(self): # gh-102029 + with warnings.catch_warnings(record=True) as warnings_log: + threading.RLock() + self.assertEqual(warnings_log, []) + + arg_types = [ + ((1,), {}), + ((), {'a': 1}), + ((1, 2), {'a': 1}), + ] + for args, kwargs in arg_types: + with self.subTest(args=args, kwargs=kwargs): + with self.assertWarns(DeprecationWarning): + threading.RLock(*args, **kwargs) + + # Subtypes with custom `__init__` are allowed (but, not recommended): + class CustomRLock(self.locktype): + def __init__(self, a, *, b) -> None: + super().__init__() + + with warnings.catch_warnings(record=True) as warnings_log: + CustomRLock(1, b=2) + self.assertEqual(warnings_log, []) + class EventTests(lock_tests.EventTests): eventtype = staticmethod(threading.Event) diff --git a/Lib/threading.py b/Lib/threading.py index e036cb891e196..f6bbdb0d0c80e 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -4,6 +4,7 @@ import sys as _sys import _thread import functools +import warnings from time import monotonic as _time from _weakrefset import WeakSet @@ -116,6 +117,12 @@ def RLock(*args, **kwargs): acquired it. """ + if args or kwargs: + warnings.warn( + 'Passing arguments to RLock is deprecated and will be removed in 3.15', + DeprecationWarning, + stacklevel=2, + ) if _CRLock is None: return _PyRLock(*args, **kwargs) return _CRLock(*args, **kwargs) diff --git a/Misc/NEWS.d/next/Library/2023-02-20-15-41-59.gh-issue-102029.9ZPG99.rst b/Misc/NEWS.d/next/Library/2023-02-20-15-41-59.gh-issue-102029.9ZPG99.rst new file mode 100644 index 0000000000000..4d6f05f252452 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-20-15-41-59.gh-issue-102029.9ZPG99.rst @@ -0,0 +1 @@ +Deprecate passing any arguments to :func:`threading.RLock`. From webhook-mailer at python.org Thu Aug 17 14:16:12 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Thu, 17 Aug 2023 18:16:12 -0000 Subject: [Python-checkins] gh-104683: Argument Clinic: Remove unreachable code from _module_and_class() (#108092) Message-ID: https://github.com/python/cpython/commit/292a22bdc22f2aa70c96e9e53ca6d6b0c5f8d5bf commit: 292a22bdc22f2aa70c96e9e53ca6d6b0c5f8d5bf branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-17T18:16:08Z summary: gh-104683: Argument Clinic: Remove unreachable code from _module_and_class() (#108092) 'not hasattr(parent, "classes")' is always false, since 'parent' is an instance of either the Module, Class, or Clinic classes, and all of them has a "classes" attribute. files: M Tools/clinic/clinic.py diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 9f7c47430772f..1593dc49e07e1 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2427,8 +2427,6 @@ def _module_and_class( if child: parent = module = child continue - if not hasattr(parent, 'classes'): - return module, cls child = parent.classes.get(field) if not child: fullname = ".".join(so_far) From webhook-mailer at python.org Thu Aug 17 14:30:02 2023 From: webhook-mailer at python.org (gvanrossum) Date: Thu, 17 Aug 2023 18:30:02 -0000 Subject: [Python-checkins] gh-106581: Project through calls (#108067) Message-ID: https://github.com/python/cpython/commit/61c7249759ce88465ea655d5c19d17d03ff3f74b commit: 61c7249759ce88465ea655d5c19d17d03ff3f74b branch: main author: Guido van Rossum committer: gvanrossum date: 2023-08-17T11:29:58-07:00 summary: gh-106581: Project through calls (#108067) This finishes the work begun in gh-107760. When, while projecting a superblock, we encounter a call to a short, simple function, the superblock will now enter the function using `_PUSH_FRAME`, continue through it, and leave it using `_POP_FRAME`, and then continue through the original code. Multiple frame pushes and pops are even possible. It is also possible to stop appending to the superblock in the middle of a called function, when running out of space or encountering an unsupported bytecode. files: M Include/internal/pycore_ceval.h M Include/internal/pycore_function.h M Include/internal/pycore_opcode_metadata.h M Lib/test/test_capi/test_misc.py M Lib/test/test_code.py M Objects/codeobject.c M Objects/funcobject.c M Python/abstract_interp_cases.c.h M Python/bytecodes.c M Python/ceval.c M Python/ceval_macros.h M Python/executor_cases.c.h M Python/generated_cases.c.h M Python/optimizer.c M Tools/cases_generator/analysis.py M Tools/cases_generator/stacking.py diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 05b7380597812..0e3a99be8c36a 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -171,6 +171,7 @@ void _PyEval_FormatKwargsError(PyThreadState *tstate, PyObject *func, PyObject * PyObject *_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyObject *_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); int _PyEval_UnpackIterable(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, PyObject **sp); +void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); #ifdef __cplusplus diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index e844d323ec792..3f3da8a44b77e 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -16,13 +16,22 @@ extern PyObject* _PyFunction_Vectorcall( #define FUNC_MAX_WATCHERS 8 +#define FUNC_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ struct _py_func_state { uint32_t next_version; + // Borrowed references to function objects whose + // func_version % FUNC_VERSION_CACHE_SIZE + // once was equal to the index in the table. + // They are cleared when the function is deallocated. + PyFunctionObject *func_version_cache[FUNC_VERSION_CACHE_SIZE]; }; extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr); extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); +extern void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); +PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version); + extern PyObject *_Py_set_function_type_params( PyThreadState* unused, PyObject *func, PyObject *type_params); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index afe8aa172b703..396d194ed2734 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -33,35 +33,36 @@ #define _BINARY_OP_SUBTRACT_FLOAT 309 #define _GUARD_BOTH_UNICODE 310 #define _BINARY_OP_ADD_UNICODE 311 -#define _LOAD_LOCALS 312 -#define _LOAD_FROM_DICT_OR_GLOBALS 313 -#define _GUARD_GLOBALS_VERSION 314 -#define _GUARD_BUILTINS_VERSION 315 -#define _LOAD_GLOBAL_MODULE 316 -#define _LOAD_GLOBAL_BUILTINS 317 -#define _GUARD_TYPE_VERSION 318 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 319 -#define _LOAD_ATTR_INSTANCE_VALUE 320 -#define IS_NONE 321 -#define _ITER_CHECK_LIST 322 -#define _IS_ITER_EXHAUSTED_LIST 323 -#define _ITER_NEXT_LIST 324 -#define _ITER_CHECK_TUPLE 325 -#define _IS_ITER_EXHAUSTED_TUPLE 326 -#define _ITER_NEXT_TUPLE 327 -#define _ITER_CHECK_RANGE 328 -#define _IS_ITER_EXHAUSTED_RANGE 329 -#define _ITER_NEXT_RANGE 330 -#define _CHECK_PEP_523 331 -#define _CHECK_FUNCTION_EXACT_ARGS 332 -#define _CHECK_STACK_SPACE 333 -#define _INIT_CALL_PY_EXACT_ARGS 334 -#define _PUSH_FRAME 335 -#define _POP_JUMP_IF_FALSE 336 -#define _POP_JUMP_IF_TRUE 337 -#define JUMP_TO_TOP 338 -#define SAVE_CURRENT_IP 339 -#define INSERT 340 +#define _POP_FRAME 312 +#define _LOAD_LOCALS 313 +#define _LOAD_FROM_DICT_OR_GLOBALS 314 +#define _GUARD_GLOBALS_VERSION 315 +#define _GUARD_BUILTINS_VERSION 316 +#define _LOAD_GLOBAL_MODULE 317 +#define _LOAD_GLOBAL_BUILTINS 318 +#define _GUARD_TYPE_VERSION 319 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 320 +#define _LOAD_ATTR_INSTANCE_VALUE 321 +#define IS_NONE 322 +#define _ITER_CHECK_LIST 323 +#define _IS_ITER_EXHAUSTED_LIST 324 +#define _ITER_NEXT_LIST 325 +#define _ITER_CHECK_TUPLE 326 +#define _IS_ITER_EXHAUSTED_TUPLE 327 +#define _ITER_NEXT_TUPLE 328 +#define _ITER_CHECK_RANGE 329 +#define _IS_ITER_EXHAUSTED_RANGE 330 +#define _ITER_NEXT_RANGE 331 +#define _CHECK_PEP_523 332 +#define _CHECK_FUNCTION_EXACT_ARGS 333 +#define _CHECK_STACK_SPACE 334 +#define _INIT_CALL_PY_EXACT_ARGS 335 +#define _PUSH_FRAME 336 +#define _POP_JUMP_IF_FALSE 337 +#define _POP_JUMP_IF_TRUE 338 +#define JUMP_TO_TOP 339 +#define SAVE_CURRENT_IP 340 +#define INSERT 341 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -197,6 +198,8 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return oparg; case INTERPRETER_EXIT: return 1; + case _POP_FRAME: + return 1; case RETURN_VALUE: return 1; case INSTRUMENTED_RETURN_VALUE: @@ -723,6 +726,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case INTERPRETER_EXIT: return 0; + case _POP_FRAME: + return 0; case RETURN_VALUE: return 0; case INSTRUMENTED_RETURN_VALUE: @@ -1191,7 +1196,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [STORE_FAST_STORE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [POP_TOP] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, 0 }, - [END_FOR] = { true, INSTR_FMT_IB, 0 }, + [END_FOR] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, 0 }, [END_SEND] = { true, INSTR_FMT_IX, 0 }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, 0 }, @@ -1205,14 +1210,14 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, 0 }, [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, 0 }, [UNARY_INVERT] = { true, INSTR_FMT_IX, 0 }, - [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IBC, 0 }, - [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IB, HAS_LOCAL_FLAG }, + [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, 0 }, + [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IX, HAS_LOCAL_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, 0 }, [BINARY_SLICE] = { true, INSTR_FMT_IX, 0 }, [STORE_SLICE] = { true, INSTR_FMT_IX, 0 }, @@ -1259,7 +1264,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, - [LOAD_LOCALS] = { true, INSTR_FMT_IB, 0 }, + [LOAD_LOCALS] = { true, INSTR_FMT_IX, 0 }, [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG }, @@ -1400,6 +1405,7 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACR #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE] = { [NOP] = { .nuops = 1, .uops = { { NOP, 0, 0 } } }, + [RESUME] = { .nuops = 1, .uops = { { RESUME, 0, 0 } } }, [LOAD_FAST_CHECK] = { .nuops = 1, .uops = { { LOAD_FAST_CHECK, 0, 0 } } }, [LOAD_FAST] = { .nuops = 1, .uops = { { LOAD_FAST, 0, 0 } } }, [LOAD_FAST_AND_CLEAR] = { .nuops = 1, .uops = { { LOAD_FAST_AND_CLEAR, 0, 0 } } }, @@ -1444,6 +1450,8 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [DELETE_SUBSCR] = { .nuops = 1, .uops = { { DELETE_SUBSCR, 0, 0 } } }, [CALL_INTRINSIC_1] = { .nuops = 1, .uops = { { CALL_INTRINSIC_1, 0, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 1, .uops = { { CALL_INTRINSIC_2, 0, 0 } } }, + [RETURN_VALUE] = { .nuops = 3, .uops = { { SAVE_IP, 7, 0 }, { SAVE_CURRENT_IP, 0, 0 }, { _POP_FRAME, 0, 0 } } }, + [RETURN_CONST] = { .nuops = 4, .uops = { { LOAD_CONST, 0, 0 }, { SAVE_IP, 7, 0 }, { SAVE_CURRENT_IP, 0, 0 }, { _POP_FRAME, 0, 0 } } }, [GET_AITER] = { .nuops = 1, .uops = { { GET_AITER, 0, 0 } } }, [GET_ANEXT] = { .nuops = 1, .uops = { { GET_ANEXT, 0, 0 } } }, [GET_AWAITABLE] = { .nuops = 1, .uops = { { GET_AWAITABLE, 0, 0 } } }, @@ -1545,6 +1553,7 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", + [_POP_FRAME] = "_POP_FRAME", [_LOAD_LOCALS] = "_LOAD_LOCALS", [_LOAD_FROM_DICT_OR_GLOBALS] = "_LOAD_FROM_DICT_OR_GLOBALS", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 3dfbfdc26e741..18a0476122dab 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2633,6 +2633,7 @@ def dummy(x): self.assertIsNotNone(ex) uops = {opname for opname, _, _ in ex} self.assertIn("_PUSH_FRAME", uops) + self.assertIn("_BINARY_OP_ADD_INT", uops) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index ca06a39f5df14..e056c16466e8c 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -264,7 +264,7 @@ def func2(): ("co_posonlyargcount", 0), ("co_kwonlyargcount", 0), ("co_nlocals", 1), - ("co_stacksize", 0), + ("co_stacksize", 1), ("co_flags", code.co_flags | inspect.CO_COROUTINE), ("co_firstlineno", 100), ("co_code", code2.co_code), diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 2c9c8cec77ff9..4d6efe938f45d 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -396,6 +396,9 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) int nlocals, ncellvars, nfreevars; get_localsplus_counts(con->localsplusnames, con->localspluskinds, &nlocals, &ncellvars, &nfreevars); + if (con->stacksize == 0) { + con->stacksize = 1; + } co->co_filename = Py_NewRef(con->filename); co->co_name = Py_NewRef(con->name); diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 8c0bface3ac71..33191d23f1823 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -223,7 +223,73 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname return NULL; } -uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func) +/* +Function versions +----------------- + +Function versions are used to detect when a function object has been +updated, invalidating inline cache data used by the `CALL` bytecode +(notably `CALL_PY_EXACT_ARGS` and a few other `CALL` specializations). + +They are also used by the Tier 2 superblock creation code to find +the function being called (and from there the code object). + +How does a function's `func_version` field get initialized? + +- `PyFunction_New` and friends initialize it to 0. +- The `MAKE_FUNCTION` instruction sets it from the code's `co_version`. +- It is reset to 0 when various attributes like `__code__` are set. +- A new version is allocated by `_PyFunction_GetVersionForCurrentState` + when the specializer needs a version and the version is 0. + +The latter allocates versions using a counter in the interpreter state; +when the counter wraps around to 0, no more versions are allocated. +There is one other special case: functions with a non-standard +`vectorcall` field are not given a version. + +When the function version is 0, the `CALL` bytecode is not specialized. + +Code object versions +-------------------- + +So where to code objects get their `co_version`? There is a single +static global counter, `_Py_next_func_version`. This is initialized in +the generated (!) file `Python/deepfreeze/deepfreeze.c`, to 1 plus the +number of deep-frozen function objects in that file. +(In `_bootstrap_python.c` and `freeze_module.c` it is initialized to 1.) + +Code objects get a new `co_version` allocated from this counter upon +creation. Since code objects are nominally immutable, `co_version` can +not be invalidated. The only way it can be 0 is when 2**32 or more +code objects have been created during the process's lifetime. +(The counter isn't reset by `fork()`, extending the lifetime.) +*/ + +void +_PyFunction_SetVersion(PyFunctionObject *func, uint32_t version) +{ + func->func_version = version; + if (version != 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + interp->func_state.func_version_cache[ + version % FUNC_VERSION_CACHE_SIZE] = func; + } +} + +PyFunctionObject * +_PyFunction_LookupByVersion(uint32_t version) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyFunctionObject *func = interp->func_state.func_version_cache[ + version % FUNC_VERSION_CACHE_SIZE]; + if (func != NULL && func->func_version == version) { + return (PyFunctionObject *)Py_NewRef(func); + } + return NULL; +} + +uint32_t +_PyFunction_GetVersionForCurrentState(PyFunctionObject *func) { if (func->func_version != 0) { return func->func_version; @@ -236,7 +302,7 @@ uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func) return 0; } uint32_t v = interp->func_state.next_version++; - func->func_version = v; + _PyFunction_SetVersion(func, v); return v; } @@ -851,6 +917,15 @@ func_dealloc(PyFunctionObject *op) if (op->func_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) op); } + if (op->func_version != 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyFunctionObject **slot = + interp->func_state.func_version_cache + + (op->func_version % FUNC_VERSION_CACHE_SIZE); + if (*slot == op) { + *slot = NULL; + } + } (void)func_clear(op); // These aren't cleared by func_clear(). Py_DECREF(op->func_code); diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index eef071119bcd8..1b99b929fa801 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -7,6 +7,10 @@ break; } + case RESUME: { + break; + } + case POP_TOP: { STACK_SHRINK(1); break; @@ -191,6 +195,11 @@ break; } + case _POP_FRAME: { + STACK_SHRINK(1); + break; + } + case GET_AITER: { PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); break; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6f17472e04e5e..ae459cabaddc0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -133,6 +133,7 @@ dummy_func( } inst(RESUME, (--)) { + #if TIER_ONE assert(frame == tstate->current_frame); /* Possibly combine this with eval breaker */ if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { @@ -140,7 +141,9 @@ dummy_func( ERROR_IF(err, error); next_instr--; } - else if (oparg < 2) { + else + #endif + if (oparg < 2) { CHECK_EVAL_BREAKER(); } } @@ -757,21 +760,37 @@ dummy_func( return retval; } - inst(RETURN_VALUE, (retval --)) { - STACK_SHRINK(1); + // The stack effect here is ambiguous. + // We definitely pop the return value off the stack on entry. + // We also push it onto the stack on exit, but that's a + // different frame, and it's accounted for by _PUSH_FRAME. + op(_POP_FRAME, (retval --)) { assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; + #if TIER_ONE + assert(frame != &entry_frame); + #endif frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); + _PyEval_FrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); + #if TIER_ONE goto resume_frame; + #endif + #if TIER_TWO + stack_pointer = _PyFrame_GetStackPointer(frame); + ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + #endif } + macro(RETURN_VALUE) = + SAVE_IP + // Tier 2 only; special-cased oparg + SAVE_CURRENT_IP + // Sets frame->prev_instr + _POP_FRAME; + inst(INSTRUMENTED_RETURN_VALUE, (retval --)) { int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, @@ -785,27 +804,17 @@ dummy_func( // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); + _PyEval_FrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; } - inst(RETURN_CONST, (--)) { - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; - } + macro(RETURN_CONST) = + LOAD_CONST + + SAVE_IP + // Tier 2 only; special-cased oparg + SAVE_CURRENT_IP + // Sets frame->prev_instr + _POP_FRAME; inst(INSTRUMENTED_RETURN_CONST, (--)) { PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); @@ -821,7 +830,7 @@ dummy_func( // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); + _PyEval_FrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; @@ -3545,7 +3554,8 @@ dummy_func( goto error; } - func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); func = (PyObject *)func_obj; } diff --git a/Python/ceval.c b/Python/ceval.c index 1e2262c1f18c3..329a1a17cf09d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -222,8 +222,6 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, static _PyInterpreterFrame * _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs); -static void -_PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); #ifdef HAVE_ERRNO_H #include @@ -603,10 +601,6 @@ int _Py_CheckRecursiveCallPy( } -static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { - tstate->py_recursion_remaining++; -} - static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { /* Put a NOP at the start, so that the IP points into * the code, rather than before it */ @@ -731,7 +725,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // When tracing executed uops, also trace bytecode char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); if (uop_debug != NULL && *uop_debug >= '0') { - lltrace = (*uop_debug - '0') >= 4; // TODO: Parse an int and all that + lltrace = (*uop_debug - '0') >= 5; // TODO: Parse an int and all that } } } @@ -918,7 +912,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); + _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; if (frame == &entry_frame) { /* Restore previous frame and exit */ @@ -1487,8 +1481,8 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) frame->previous = NULL; } -static void -_PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) +void +_PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) { if (frame->owner == FRAME_OWNED_BY_THREAD) { clear_thread_frame(tstate, frame); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 08f19cd9a397f..635b8e501e523 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -369,3 +369,7 @@ static inline int _Py_EnterRecursivePy(PyThreadState *tstate) { return (tstate->py_recursion_remaining-- <= 0) && _Py_CheckRecursiveCallPy(tstate); } + +static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { + tstate->py_recursion_remaining++; +} diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9fbf026f164a6..89a5bbfecded0 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7,6 +7,23 @@ break; } + case RESUME: { + #if TIER_ONE + assert(frame == tstate->current_frame); + /* Possibly combine this with eval breaker */ + if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) goto error; + next_instr--; + } + else + #endif + if (oparg < 2) { + CHECK_EVAL_BREAKER(); + } + break; + } + case LOAD_FAST_CHECK: { PyObject *value; value = GETLOCAL(oparg); @@ -666,6 +683,32 @@ break; } + case _POP_FRAME: { + PyObject *retval; + retval = stack_pointer[-1]; + STACK_SHRINK(1); + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + #if TIER_ONE + assert(frame != &entry_frame); + #endif + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->prev_instr += frame->return_offset; + _PyFrame_StackPush(frame, retval); + #if TIER_ONE + goto resume_frame; + #endif + #if TIER_TWO + stack_pointer = _PyFrame_GetStackPointer(frame); + ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + #endif + break; + } + case GET_AITER: { PyObject *obj; PyObject *iter; @@ -2607,7 +2650,8 @@ goto error; } - func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); func = (PyObject *)func_obj; stack_pointer[-1] = func; break; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 80af8a7bcd56d..f6322df566c65 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8,6 +8,7 @@ } TARGET(RESUME) { + #if TIER_ONE assert(frame == tstate->current_frame); /* Possibly combine this with eval breaker */ if (_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version) { @@ -15,7 +16,9 @@ if (err) goto error; next_instr--; } - else if (oparg < 2) { + else + #endif + if (oparg < 2) { CHECK_EVAL_BREAKER(); } DISPATCH(); @@ -970,20 +973,40 @@ TARGET(RETURN_VALUE) { PyObject *retval; + // SAVE_CURRENT_IP + { + #if TIER_ONE + frame->prev_instr = next_instr - 1; + #endif + #if TIER_TWO + // Relies on a preceding SAVE_IP + frame->prev_instr--; + #endif + } + // _POP_FRAME retval = stack_pointer[-1]; STACK_SHRINK(1); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; - STACK_SHRINK(1); + { + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + #if TIER_ONE + assert(frame != &entry_frame); + #endif + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->prev_instr += frame->return_offset; + _PyFrame_StackPush(frame, retval); + #if TIER_ONE + goto resume_frame; + #endif + #if TIER_TWO + stack_pointer = _PyFrame_GetStackPointer(frame); + ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + #endif + } } TARGET(INSTRUMENTED_RETURN_VALUE) { @@ -1001,7 +1024,7 @@ // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); + _PyEval_FrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; @@ -1009,19 +1032,46 @@ } TARGET(RETURN_CONST) { - PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg); - Py_INCREF(retval); - assert(EMPTY()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_LeaveRecursiveCallPy(tstate); - assert(frame != &entry_frame); - // GH-99729: We need to unlink the frame *before* clearing it: - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); - frame->prev_instr += frame->return_offset; - _PyFrame_StackPush(frame, retval); - goto resume_frame; + PyObject *value; + PyObject *retval; + // LOAD_CONST + { + value = GETITEM(FRAME_CO_CONSTS, oparg); + Py_INCREF(value); + } + // SAVE_CURRENT_IP + { + #if TIER_ONE + frame->prev_instr = next_instr - 1; + #endif + #if TIER_TWO + // Relies on a preceding SAVE_IP + frame->prev_instr--; + #endif + } + // _POP_FRAME + retval = value; + { + assert(EMPTY()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_LeaveRecursiveCallPy(tstate); + // GH-99729: We need to unlink the frame *before* clearing it: + _PyInterpreterFrame *dying = frame; + #if TIER_ONE + assert(frame != &entry_frame); + #endif + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + frame->prev_instr += frame->return_offset; + _PyFrame_StackPush(frame, retval); + #if TIER_ONE + goto resume_frame; + #endif + #if TIER_TWO + stack_pointer = _PyFrame_GetStackPointer(frame); + ip_offset = (_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive; + #endif + } } TARGET(INSTRUMENTED_RETURN_CONST) { @@ -1038,7 +1088,7 @@ // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; - _PyEvalFrameClearAndPop(tstate, dying); + _PyEval_FrameClearAndPop(tstate, dying); frame->prev_instr += frame->return_offset; _PyFrame_StackPush(frame, retval); goto resume_frame; @@ -4575,7 +4625,8 @@ goto error; } - func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); func = (PyObject *)func_obj; stack_pointer[-1] = func; DISPATCH(); diff --git a/Python/optimizer.c b/Python/optimizer.c index 559c4ae987263..57518404c3f19 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -373,6 +373,8 @@ static PyTypeObject UOpExecutor_Type = { .tp_as_sequence = &uop_as_sequence, }; +#define TRACE_STACK_SIZE 5 + static int translate_bytecode_to_trace( PyCodeObject *code, @@ -380,10 +382,16 @@ translate_bytecode_to_trace( _PyUOpInstruction *trace, int buffer_size) { + PyCodeObject *initial_code = code; _Py_CODEUNIT *initial_instr = instr; int trace_length = 0; int max_length = buffer_size; int reserved = 0; + struct { + PyCodeObject *code; + _Py_CODEUNIT *instr; + } trace_stack[TRACE_STACK_SIZE]; + int trace_stack_depth = 0; #ifdef Py_DEBUG char *uop_debug = Py_GETENV("PYTHONUOPSDEBUG"); @@ -441,6 +449,24 @@ translate_bytecode_to_trace( // Reserve space for main+stub uops, plus 2 for SAVE_IP and EXIT_TRACE #define RESERVE(main, stub) RESERVE_RAW((main) + (stub) + 2, uop_name(opcode)) +// Trace stack operations (used by _PUSH_FRAME, _POP_FRAME) +#define TRACE_STACK_PUSH() \ + if (trace_stack_depth >= TRACE_STACK_SIZE) { \ + DPRINTF(2, "Trace stack overflow\n"); \ + ADD_TO_TRACE(SAVE_IP, 0, 0); \ + goto done; \ + } \ + trace_stack[trace_stack_depth].code = code; \ + trace_stack[trace_stack_depth].instr = instr; \ + trace_stack_depth++; +#define TRACE_STACK_POP() \ + if (trace_stack_depth <= 0) { \ + Py_FatalError("Trace stack underflow\n"); \ + } \ + trace_stack_depth--; \ + code = trace_stack[trace_stack_depth].code; \ + instr = trace_stack[trace_stack_depth].instr; + DPRINTF(4, "Optimizing %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), @@ -448,6 +474,7 @@ translate_bytecode_to_trace( code->co_firstlineno, 2 * INSTR_IP(initial_instr, code)); +top: // Jump here after _PUSH_FRAME for (;;) { RESERVE_RAW(2, "epilogue"); // Always need space for SAVE_IP and EXIT_TRACE ADD_TO_TRACE(SAVE_IP, INSTR_IP(instr, code), 0); @@ -508,7 +535,7 @@ translate_bytecode_to_trace( case JUMP_BACKWARD: { - if (instr + 2 - oparg == initial_instr) { + if (instr + 2 - oparg == initial_instr && code == initial_code) { RESERVE(1, 0); ADD_TO_TRACE(JUMP_TO_TOP, 0, 0); } @@ -573,6 +600,14 @@ translate_bytecode_to_trace( // Reserve space for nuops (+ SAVE_IP + EXIT_TRACE) int nuops = expansion->nuops; RESERVE(nuops, 0); + if (expansion->uops[nuops-1].uop == _POP_FRAME) { + // Check for trace stack underflow now: + // We can't bail e.g. in the middle of + // LOAD_CONST + _POP_FRAME. + if (trace_stack_depth == 0) { + DPRINTF(2, "Trace stack underflow\n"); + goto done;} + } uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM for (int i = 0; i < nuops; i++) { oparg = orig_oparg; @@ -619,8 +654,57 @@ translate_bytecode_to_trace( Py_FatalError("garbled expansion"); } ADD_TO_TRACE(expansion->uops[i].uop, oparg, operand); + if (expansion->uops[i].uop == _POP_FRAME) { + TRACE_STACK_POP(); + DPRINTF(2, + "Returning to %s (%s:%d) at byte offset %d\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + 2 * INSTR_IP(instr, code)); + goto top; + } if (expansion->uops[i].uop == _PUSH_FRAME) { assert(i + 1 == nuops); + int func_version_offset = + offsetof(_PyCallCache, func_version)/sizeof(_Py_CODEUNIT) + // Add one to account for the actual opcode/oparg pair: + + 1; + uint32_t func_version = read_u32(&instr[func_version_offset].cache); + PyFunctionObject *func = _PyFunction_LookupByVersion(func_version); + DPRINTF(3, "Function object: %p\n", func); + if (func != NULL) { + PyCodeObject *new_code = (PyCodeObject *)PyFunction_GET_CODE(func); + if (new_code == code) { + // Recursive call, bail (we could be here forever). + DPRINTF(2, "Bailing on recursive call to %s (%s:%d)\n", + PyUnicode_AsUTF8(new_code->co_qualname), + PyUnicode_AsUTF8(new_code->co_filename), + new_code->co_firstlineno); + ADD_TO_TRACE(SAVE_IP, 0, 0); + goto done; + } + if (new_code->co_version != func_version) { + // func.__code__ was updated. + // Perhaps it may happen again, so don't bother tracing. + // TODO: Reason about this -- is it better to bail or not? + DPRINTF(2, "Bailing because co_version != func_version\n"); + ADD_TO_TRACE(SAVE_IP, 0, 0); + goto done; + } + // Increment IP to the return address + instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; + TRACE_STACK_PUSH(); + code = new_code; + instr = _PyCode_CODE(code); + DPRINTF(2, + "Continuing in %s (%s:%d) at byte offset %d\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + 2 * INSTR_IP(instr, code)); + goto top; + } ADD_TO_TRACE(SAVE_IP, 0, 0); goto done; } @@ -639,6 +723,10 @@ translate_bytecode_to_trace( } // End for (;;) done: + while (trace_stack_depth > 0) { + TRACE_STACK_POP(); + } + assert(code == initial_code); // Skip short traces like SAVE_IP, LOAD_FAST, SAVE_IP, EXIT_TRACE if (trace_length > 3) { ADD_TO_TRACE(EXIT_TRACE, 0, 0); diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 2db1cd01c19ae..48f2db981c95b 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -364,10 +364,12 @@ def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: case Instruction() as instr: part, offset = self.analyze_instruction(instr, offset) parts.append(part) - flags.add(instr.instr_flags) + if instr.name != "SAVE_IP": + # SAVE_IP in a macro is a no-op in Tier 1 + flags.add(instr.instr_flags) case _: typing.assert_never(component) - format = "IB" + format = "IB" if flags.HAS_ARG_FLAG else "IX" if offset: format += "C" + "0" * (offset - 1) return MacroInstruction(macro.name, format, flags, macro, parts, offset) diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 8361eb99f88a7..632298a567dd4 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -380,7 +380,7 @@ def write_components( poke.as_stack_effect(lax=True), ) - if mgr.instr.name == "_PUSH_FRAME": + if mgr.instr.name in ("_PUSH_FRAME", "_POP_FRAME"): # Adjust stack to min_offset (input effects materialized) out.stack_adjust(mgr.min_offset.deep, mgr.min_offset.high) # Use clone() since adjust_inverse() mutates final_offset From webhook-mailer at python.org Thu Aug 17 14:39:46 2023 From: webhook-mailer at python.org (pablogsal) Date: Thu, 17 Aug 2023 18:39:46 -0000 Subject: [Python-checkins] gh-107944: Improve error message for function calls with bad keyword arguments (#107969) Message-ID: https://github.com/python/cpython/commit/75b3db8445188c2ad38cabc0021af694df0829b8 commit: 75b3db8445188c2ad38cabc0021af694df0829b8 branch: main author: Pablo Galindo Salgado committer: pablogsal date: 2023-08-17T19:39:42+01:00 summary: gh-107944: Improve error message for function calls with bad keyword arguments (#107969) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-15-11-09-50.gh-issue-107944.zQLp3j.rst M Include/internal/pycore_pyerrors.h M Lib/test/test_call.py M Python/ceval.c M Python/suggestions.c diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 45929f40a0549..91fd68984523d 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -150,7 +150,7 @@ extern PyObject* _PyExc_PrepReraiseStar( extern int _PyErr_CheckSignalsTstate(PyThreadState *tstate); extern void _Py_DumpExtensionModules(int fd, PyInterpreterState *interp); - +extern PyObject* _Py_CalculateSuggestions(PyObject *dir, PyObject *name); extern PyObject* _Py_Offer_Suggestions(PyObject* exception); // Export for '_testinternalcapi' shared extension PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b, diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index c3c3b1853b573..008a8c1f0cb87 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -915,6 +915,74 @@ def test_multiple_values(self): with self.check_raises_type_error(msg): A().method_two_args("x", "y", x="oops") + at cpython_only +class TestErrorMessagesSuggestions(unittest.TestCase): + @contextlib.contextmanager + def check_suggestion_includes(self, message): + with self.assertRaises(TypeError) as cm: + yield + self.assertIn(f"Did you mean '{message}'?", str(cm.exception)) + + @contextlib.contextmanager + def check_suggestion_not_pressent(self): + with self.assertRaises(TypeError) as cm: + yield + self.assertNotIn("Did you mean", str(cm.exception)) + + def test_unexpected_keyword_suggestion_valid_positions(self): + def foo(blech=None, /, aaa=None, *args, late1=None): + pass + + cases = [ + ("blach", None), + ("aa", "aaa"), + ("orgs", None), + ("late11", "late1"), + ] + + for keyword, suggestion in cases: + with self.subTest(keyword): + ctx = self.check_suggestion_includes(suggestion) if suggestion else self.check_suggestion_not_pressent() + with ctx: + foo(**{keyword:None}) + + def test_unexpected_keyword_suggestion_kinds(self): + + def substitution(noise=None, more_noise=None, a = None, blech = None): + pass + + def elimination(noise = None, more_noise = None, a = None, blch = None): + pass + + def addition(noise = None, more_noise = None, a = None, bluchin = None): + pass + + def substitution_over_elimination(blach = None, bluc = None): + pass + + def substitution_over_addition(blach = None, bluchi = None): + pass + + def elimination_over_addition(bluc = None, blucha = None): + pass + + def case_change_over_substitution(BLuch=None, Luch = None, fluch = None): + pass + + for func, suggestion in [ + (addition, "bluchin"), + (substitution, "blech"), + (elimination, "blch"), + (addition, "bluchin"), + (substitution_over_elimination, "blach"), + (substitution_over_addition, "blach"), + (elimination_over_addition, "bluc"), + (case_change_over_substitution, "BLuch"), + ]: + with self.subTest(suggestion): + with self.check_suggestion_includes(suggestion): + func(bluch=None) + @cpython_only class TestRecursion(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-15-11-09-50.gh-issue-107944.zQLp3j.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-15-11-09-50.gh-issue-107944.zQLp3j.rst new file mode 100644 index 0000000000000..9a53332c70261 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-15-11-09-50.gh-issue-107944.zQLp3j.rst @@ -0,0 +1,2 @@ +Improve error message for function calls with bad keyword arguments. Patch +by Pablo Galindo diff --git a/Python/ceval.c b/Python/ceval.c index 329a1a17cf09d..f7dfaebcdd8f8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -26,6 +26,7 @@ #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_typeobject.h" // _PySuper_Lookup() #include "pycore_uops.h" // _PyUOpExecutorObject +#include "pycore_pyerrors.h" #include "pycore_dict.h" #include "dictobject.h" @@ -1337,9 +1338,33 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, goto kw_fail; } - _PyErr_Format(tstate, PyExc_TypeError, - "%U() got an unexpected keyword argument '%S'", - func->func_qualname, keyword); + PyObject* suggestion_keyword = NULL; + if (total_args > co->co_posonlyargcount) { + PyObject* possible_keywords = PyList_New(total_args - co->co_posonlyargcount); + + if (!possible_keywords) { + PyErr_Clear(); + } else { + for (Py_ssize_t k = co->co_posonlyargcount; k < total_args; k++) { + PyList_SET_ITEM(possible_keywords, k - co->co_posonlyargcount, co_varnames[k]); + } + + suggestion_keyword = _Py_CalculateSuggestions(possible_keywords, keyword); + Py_DECREF(possible_keywords); + } + } + + if (suggestion_keyword) { + _PyErr_Format(tstate, PyExc_TypeError, + "%U() got an unexpected keyword argument '%S'. Did you mean '%S'?", + func->func_qualname, keyword, suggestion_keyword); + Py_DECREF(suggestion_keyword); + } else { + _PyErr_Format(tstate, PyExc_TypeError, + "%U() got an unexpected keyword argument '%S'", + func->func_qualname, keyword); + } + goto kw_fail; } diff --git a/Python/suggestions.c b/Python/suggestions.c index 47aeb08180f6b..12097f793e357 100644 --- a/Python/suggestions.c +++ b/Python/suggestions.c @@ -126,8 +126,8 @@ levenshtein_distance(const char *a, size_t a_size, return result; } -static inline PyObject * -calculate_suggestions(PyObject *dir, +PyObject * +_Py_CalculateSuggestions(PyObject *dir, PyObject *name) { assert(!PyErr_Occurred()); @@ -195,7 +195,7 @@ get_suggestions_for_attribute_error(PyAttributeErrorObject *exc) return NULL; } - PyObject *suggestions = calculate_suggestions(dir, name); + PyObject *suggestions = _Py_CalculateSuggestions(dir, name); Py_DECREF(dir); return suggestions; } @@ -259,7 +259,7 @@ get_suggestions_for_name_error(PyObject* name, PyFrameObject* frame) } } - PyObject *suggestions = calculate_suggestions(dir, name); + PyObject *suggestions = _Py_CalculateSuggestions(dir, name); Py_DECREF(dir); if (suggestions != NULL || PyErr_Occurred()) { return suggestions; @@ -269,7 +269,7 @@ get_suggestions_for_name_error(PyObject* name, PyFrameObject* frame) if (dir == NULL) { return NULL; } - suggestions = calculate_suggestions(dir, name); + suggestions = _Py_CalculateSuggestions(dir, name); Py_DECREF(dir); if (suggestions != NULL || PyErr_Occurred()) { return suggestions; @@ -279,7 +279,7 @@ get_suggestions_for_name_error(PyObject* name, PyFrameObject* frame) if (dir == NULL) { return NULL; } - suggestions = calculate_suggestions(dir, name); + suggestions = _Py_CalculateSuggestions(dir, name); Py_DECREF(dir); return suggestions; @@ -371,7 +371,7 @@ offer_suggestions_for_import_error(PyImportErrorObject *exc) return NULL; } - PyObject *suggestion = calculate_suggestions(dir, name); + PyObject *suggestion = _Py_CalculateSuggestions(dir, name); Py_DECREF(dir); if (!suggestion) { return NULL; From webhook-mailer at python.org Thu Aug 17 15:19:05 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Thu, 17 Aug 2023 19:19:05 -0000 Subject: [Python-checkins] Docs: Fix Sphinx warnings in io.rst (#107903) Message-ID: https://github.com/python/cpython/commit/5c76899dadf3bdcfdedf6f30b3ab9742cb87af04 commit: 5c76899dadf3bdcfdedf6f30b3ab9742cb87af04 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-17T19:19:01Z summary: Docs: Fix Sphinx warnings in io.rst (#107903) - Mark up parameter and argument names properly - If possible, link to docs for methods like `seek`, `tell`, `write`, `read`, etc. Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/io.rst M Doc/tools/.nitignore diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 7eec1f87583b8..66273d9ed1ff0 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -38,9 +38,9 @@ location), or only sequential access (for example in the case of a socket or pipe). All streams are careful about the type of data you give to them. For example -giving a :class:`str` object to the ``write()`` method of a binary stream +giving a :class:`str` object to the :meth:`!write` method of a binary stream will raise a :exc:`TypeError`. So will giving a :class:`bytes` object to the -``write()`` method of a text stream. +:meth:`!write` method of a text stream. .. versionchanged:: 3.3 Operations that used to raise :exc:`IOError` now raise :exc:`OSError`, since @@ -146,7 +146,7 @@ Opt-in EncodingWarning See :pep:`597` for more details. To find where the default locale encoding is used, you can enable -the ``-X warn_default_encoding`` command line option or set the +the :option:`-X warn_default_encoding <-X>` command line option or set the :envvar:`PYTHONWARNDEFAULTENCODING` environment variable, which will emit an :exc:`EncodingWarning` when the default encoding is used. @@ -175,7 +175,7 @@ High-level Module Interface .. audit-event:: open path,mode,flags io.open This function raises an :ref:`auditing event ` ``open`` with - arguments ``path``, ``mode`` and ``flags``. The ``mode`` and ``flags`` + arguments *path*, *mode* and *flags*. The *mode* and *flags* arguments may have been modified or inferred from the original call. @@ -184,10 +184,10 @@ High-level Module Interface Opens the provided file with mode ``'rb'``. This function should be used when the intent is to treat the contents as executable code. - ``path`` should be a :class:`str` and an absolute path. + *path* should be a :class:`str` and an absolute path. The behavior of this function may be overridden by an earlier call to the - :c:func:`PyFile_SetOpenCodeHook`. However, assuming that ``path`` is a + :c:func:`PyFile_SetOpenCodeHook`. However, assuming that *path* is a :class:`str` and an absolute path, ``open_code(path)`` should always behave the same as ``open(path, 'rb')``. Overriding the behavior is intended for additional validation or preprocessing of the file. @@ -258,7 +258,7 @@ standard stream implementations. The abstract base classes also provide default implementations of some methods in order to help implementation of concrete stream classes. For example, :class:`BufferedIOBase` provides unoptimized implementations of - :meth:`~IOBase.readinto` and :meth:`~IOBase.readline`. + :meth:`!readinto` and :meth:`!readline`. At the top of the I/O hierarchy is the abstract base class :class:`IOBase`. It defines the basic interface to a stream. Note, however, that there is no @@ -320,8 +320,8 @@ I/O Base Classes implementations represent a file that cannot be read, written or seeked. - Even though :class:`IOBase` does not declare :meth:`read` - or :meth:`write` because their signatures will vary, implementations and + Even though :class:`IOBase` does not declare :meth:`!read` + or :meth:`!write` because their signatures will vary, implementations and clients should consider those methods part of the interface. Also, implementations may raise a :exc:`ValueError` (or :exc:`UnsupportedOperation`) when operations they do not support are called. @@ -379,8 +379,8 @@ I/O Base Classes .. method:: readable() - Return ``True`` if the stream can be read from. If ``False``, :meth:`read` - will raise :exc:`OSError`. + Return ``True`` if the stream can be read from. + If ``False``, :meth:`!read` will raise :exc:`OSError`. .. method:: readline(size=-1, /) @@ -401,25 +401,25 @@ I/O Base Classes hint. Note that it's already possible to iterate on file objects using ``for - line in file: ...`` without calling ``file.readlines()``. + line in file: ...`` without calling :meth:`!file.readlines`. .. method:: seek(offset, whence=SEEK_SET, /) Change the stream position to the given byte *offset*. *offset* is interpreted relative to the position indicated by *whence*. The default - value for *whence* is :data:`SEEK_SET`. Values for *whence* are: + value for *whence* is :data:`!SEEK_SET`. Values for *whence* are: - * :data:`SEEK_SET` or ``0`` -- start of the stream (the default); + * :data:`!SEEK_SET` or ``0`` -- start of the stream (the default); *offset* should be zero or positive - * :data:`SEEK_CUR` or ``1`` -- current stream position; *offset* may + * :data:`!SEEK_CUR` or ``1`` -- current stream position; *offset* may be negative - * :data:`SEEK_END` or ``2`` -- end of the stream; *offset* is usually + * :data:`!SEEK_END` or ``2`` -- end of the stream; *offset* is usually negative Return the new absolute position. .. versionadded:: 3.1 - The ``SEEK_*`` constants. + The :data:`!SEEK_*` constants. .. versionadded:: 3.3 Some operating systems could support additional values, like @@ -450,7 +450,7 @@ I/O Base Classes .. method:: writable() Return ``True`` if the stream supports writing. If ``False``, - :meth:`write` and :meth:`truncate` will raise :exc:`OSError`. + :meth:`!write` and :meth:`truncate` will raise :exc:`OSError`. .. method:: writelines(lines, /) @@ -654,8 +654,9 @@ Raw File I/O implies writing, so this mode behaves in a similar way to ``'w'``. Add a ``'+'`` to the mode to allow simultaneous reading and writing. - The :meth:`read` (when called with a positive argument), :meth:`readinto` - and :meth:`write` methods on this class will only make one system call. + The :meth:`~RawIOBase.read` (when called with a positive argument), + :meth:`~RawIOBase.readinto` and :meth:`~RawIOBase.write` methods on this + class will only make one system call. A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by calling *opener* with @@ -791,8 +792,8 @@ than raw I/O does. object under various conditions, including: * when the buffer gets too small for all pending data; - * when :meth:`flush()` is called; - * when a :meth:`seek()` is requested (for :class:`BufferedRandom` objects); + * when :meth:`flush` is called; + * when a :meth:`~IOBase.seek` is requested (for :class:`BufferedRandom` objects); * when the :class:`BufferedWriter` object is closed or destroyed. The constructor creates a :class:`BufferedWriter` for the given writeable @@ -826,8 +827,8 @@ than raw I/O does. :data:`DEFAULT_BUFFER_SIZE`. :class:`BufferedRandom` is capable of anything :class:`BufferedReader` or - :class:`BufferedWriter` can do. In addition, :meth:`seek` and :meth:`tell` - are guaranteed to be implemented. + :class:`BufferedWriter` can do. In addition, :meth:`~IOBase.seek` and + :meth:`~IOBase.tell` are guaranteed to be implemented. .. class:: BufferedRWPair(reader, writer, buffer_size=DEFAULT_BUFFER_SIZE, /) @@ -904,7 +905,7 @@ Text I/O .. method:: readline(size=-1, /) - Read until newline or EOF and return a single ``str``. If the stream is + Read until newline or EOF and return a single :class:`str`. If the stream is already at EOF, an empty string is returned. If *size* is specified, at most *size* characters will be read. @@ -913,22 +914,22 @@ Text I/O Change the stream position to the given *offset*. Behaviour depends on the *whence* parameter. The default value for *whence* is - :data:`SEEK_SET`. + :data:`!SEEK_SET`. - * :data:`SEEK_SET` or ``0``: seek from the start of the stream + * :data:`!SEEK_SET` or ``0``: seek from the start of the stream (the default); *offset* must either be a number returned by :meth:`TextIOBase.tell`, or zero. Any other *offset* value produces undefined behaviour. - * :data:`SEEK_CUR` or ``1``: "seek" to the current position; + * :data:`!SEEK_CUR` or ``1``: "seek" to the current position; *offset* must be zero, which is a no-operation (all other values are unsupported). - * :data:`SEEK_END` or ``2``: seek to the end of the stream; + * :data:`!SEEK_END` or ``2``: seek to the end of the stream; *offset* must be zero (all other values are unsupported). Return the new absolute position as an opaque number. .. versionadded:: 3.1 - The ``SEEK_*`` constants. + The :data:`!SEEK_*` constants. .. method:: tell() @@ -988,10 +989,10 @@ Text I/O takes place. If *newline* is any of the other legal values, any ``'\n'`` characters written are translated to the given string. - If *line_buffering* is ``True``, :meth:`flush` is implied when a call to + If *line_buffering* is ``True``, :meth:`~IOBase.flush` is implied when a call to write contains a newline character or a carriage return. - If *write_through* is ``True``, calls to :meth:`write` are guaranteed + If *write_through* is ``True``, calls to :meth:`~BufferedIOBase.write` are guaranteed not to be buffered: any data written on the :class:`TextIOWrapper` object is immediately handled to its underlying binary *buffer*. @@ -1070,7 +1071,7 @@ Text I/O .. method:: getvalue() - Return a ``str`` containing the entire contents of the buffer. + Return a :class:`str` containing the entire contents of the buffer. Newlines are decoded as if by :meth:`~TextIOBase.read`, although the stream position is not changed. @@ -1125,7 +1126,7 @@ Text I/O over a binary storage (such as a file) is significantly slower than binary I/O over the same storage, because it requires conversions between unicode and binary data using a character codec. This can become noticeable handling huge amounts of text data like large log files. Also, -:meth:`TextIOWrapper.tell` and :meth:`TextIOWrapper.seek` are both quite slow +:meth:`~TextIOBase.tell` and :meth:`~TextIOBase.seek` are both quite slow due to the reconstruction algorithm used. :class:`StringIO`, however, is a native in-memory unicode container and will @@ -1135,7 +1136,7 @@ Multi-threading ^^^^^^^^^^^^^^^ :class:`FileIO` objects are thread-safe to the extent that the operating system -calls (such as ``read(2)`` under Unix) they wrap are thread-safe too. +calls (such as :manpage:`read(2)` under Unix) they wrap are thread-safe too. Binary buffered objects (instances of :class:`BufferedReader`, :class:`BufferedWriter`, :class:`BufferedRandom` and :class:`BufferedRWPair`) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 0f68cefc92b97..654287cc50b55 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -89,7 +89,6 @@ Doc/library/http.server.rst Doc/library/importlib.resources.rst Doc/library/importlib.rst Doc/library/inspect.rst -Doc/library/io.rst Doc/library/locale.rst Doc/library/logging.config.rst Doc/library/logging.handlers.rst From webhook-mailer at python.org Thu Aug 17 15:29:14 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Thu, 17 Aug 2023 19:29:14 -0000 Subject: [Python-checkins] [3.11] Docs: Fix Sphinx warnings in io.rst (GH-107903) (#108094) Message-ID: https://github.com/python/cpython/commit/b389939be96f21162d7e8c187dceccc62e8a1c82 commit: b389939be96f21162d7e8c187dceccc62e8a1c82 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: erlend-aasland date: 2023-08-17T19:29:10Z summary: [3.11] Docs: Fix Sphinx warnings in io.rst (GH-107903) (#108094) - Mark up parameter and argument names properly - If possible, link to docs for methods like `seek`, `tell`, `write`, `read`, etc. (cherry picked from commit 5c76899dadf3bdcfdedf6f30b3ab9742cb87af04) Co-authored-by: Erlend E. Aasland Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/io.rst M Doc/tools/.nitignore diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 7eec1f87583b8..66273d9ed1ff0 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -38,9 +38,9 @@ location), or only sequential access (for example in the case of a socket or pipe). All streams are careful about the type of data you give to them. For example -giving a :class:`str` object to the ``write()`` method of a binary stream +giving a :class:`str` object to the :meth:`!write` method of a binary stream will raise a :exc:`TypeError`. So will giving a :class:`bytes` object to the -``write()`` method of a text stream. +:meth:`!write` method of a text stream. .. versionchanged:: 3.3 Operations that used to raise :exc:`IOError` now raise :exc:`OSError`, since @@ -146,7 +146,7 @@ Opt-in EncodingWarning See :pep:`597` for more details. To find where the default locale encoding is used, you can enable -the ``-X warn_default_encoding`` command line option or set the +the :option:`-X warn_default_encoding <-X>` command line option or set the :envvar:`PYTHONWARNDEFAULTENCODING` environment variable, which will emit an :exc:`EncodingWarning` when the default encoding is used. @@ -175,7 +175,7 @@ High-level Module Interface .. audit-event:: open path,mode,flags io.open This function raises an :ref:`auditing event ` ``open`` with - arguments ``path``, ``mode`` and ``flags``. The ``mode`` and ``flags`` + arguments *path*, *mode* and *flags*. The *mode* and *flags* arguments may have been modified or inferred from the original call. @@ -184,10 +184,10 @@ High-level Module Interface Opens the provided file with mode ``'rb'``. This function should be used when the intent is to treat the contents as executable code. - ``path`` should be a :class:`str` and an absolute path. + *path* should be a :class:`str` and an absolute path. The behavior of this function may be overridden by an earlier call to the - :c:func:`PyFile_SetOpenCodeHook`. However, assuming that ``path`` is a + :c:func:`PyFile_SetOpenCodeHook`. However, assuming that *path* is a :class:`str` and an absolute path, ``open_code(path)`` should always behave the same as ``open(path, 'rb')``. Overriding the behavior is intended for additional validation or preprocessing of the file. @@ -258,7 +258,7 @@ standard stream implementations. The abstract base classes also provide default implementations of some methods in order to help implementation of concrete stream classes. For example, :class:`BufferedIOBase` provides unoptimized implementations of - :meth:`~IOBase.readinto` and :meth:`~IOBase.readline`. + :meth:`!readinto` and :meth:`!readline`. At the top of the I/O hierarchy is the abstract base class :class:`IOBase`. It defines the basic interface to a stream. Note, however, that there is no @@ -320,8 +320,8 @@ I/O Base Classes implementations represent a file that cannot be read, written or seeked. - Even though :class:`IOBase` does not declare :meth:`read` - or :meth:`write` because their signatures will vary, implementations and + Even though :class:`IOBase` does not declare :meth:`!read` + or :meth:`!write` because their signatures will vary, implementations and clients should consider those methods part of the interface. Also, implementations may raise a :exc:`ValueError` (or :exc:`UnsupportedOperation`) when operations they do not support are called. @@ -379,8 +379,8 @@ I/O Base Classes .. method:: readable() - Return ``True`` if the stream can be read from. If ``False``, :meth:`read` - will raise :exc:`OSError`. + Return ``True`` if the stream can be read from. + If ``False``, :meth:`!read` will raise :exc:`OSError`. .. method:: readline(size=-1, /) @@ -401,25 +401,25 @@ I/O Base Classes hint. Note that it's already possible to iterate on file objects using ``for - line in file: ...`` without calling ``file.readlines()``. + line in file: ...`` without calling :meth:`!file.readlines`. .. method:: seek(offset, whence=SEEK_SET, /) Change the stream position to the given byte *offset*. *offset* is interpreted relative to the position indicated by *whence*. The default - value for *whence* is :data:`SEEK_SET`. Values for *whence* are: + value for *whence* is :data:`!SEEK_SET`. Values for *whence* are: - * :data:`SEEK_SET` or ``0`` -- start of the stream (the default); + * :data:`!SEEK_SET` or ``0`` -- start of the stream (the default); *offset* should be zero or positive - * :data:`SEEK_CUR` or ``1`` -- current stream position; *offset* may + * :data:`!SEEK_CUR` or ``1`` -- current stream position; *offset* may be negative - * :data:`SEEK_END` or ``2`` -- end of the stream; *offset* is usually + * :data:`!SEEK_END` or ``2`` -- end of the stream; *offset* is usually negative Return the new absolute position. .. versionadded:: 3.1 - The ``SEEK_*`` constants. + The :data:`!SEEK_*` constants. .. versionadded:: 3.3 Some operating systems could support additional values, like @@ -450,7 +450,7 @@ I/O Base Classes .. method:: writable() Return ``True`` if the stream supports writing. If ``False``, - :meth:`write` and :meth:`truncate` will raise :exc:`OSError`. + :meth:`!write` and :meth:`truncate` will raise :exc:`OSError`. .. method:: writelines(lines, /) @@ -654,8 +654,9 @@ Raw File I/O implies writing, so this mode behaves in a similar way to ``'w'``. Add a ``'+'`` to the mode to allow simultaneous reading and writing. - The :meth:`read` (when called with a positive argument), :meth:`readinto` - and :meth:`write` methods on this class will only make one system call. + The :meth:`~RawIOBase.read` (when called with a positive argument), + :meth:`~RawIOBase.readinto` and :meth:`~RawIOBase.write` methods on this + class will only make one system call. A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by calling *opener* with @@ -791,8 +792,8 @@ than raw I/O does. object under various conditions, including: * when the buffer gets too small for all pending data; - * when :meth:`flush()` is called; - * when a :meth:`seek()` is requested (for :class:`BufferedRandom` objects); + * when :meth:`flush` is called; + * when a :meth:`~IOBase.seek` is requested (for :class:`BufferedRandom` objects); * when the :class:`BufferedWriter` object is closed or destroyed. The constructor creates a :class:`BufferedWriter` for the given writeable @@ -826,8 +827,8 @@ than raw I/O does. :data:`DEFAULT_BUFFER_SIZE`. :class:`BufferedRandom` is capable of anything :class:`BufferedReader` or - :class:`BufferedWriter` can do. In addition, :meth:`seek` and :meth:`tell` - are guaranteed to be implemented. + :class:`BufferedWriter` can do. In addition, :meth:`~IOBase.seek` and + :meth:`~IOBase.tell` are guaranteed to be implemented. .. class:: BufferedRWPair(reader, writer, buffer_size=DEFAULT_BUFFER_SIZE, /) @@ -904,7 +905,7 @@ Text I/O .. method:: readline(size=-1, /) - Read until newline or EOF and return a single ``str``. If the stream is + Read until newline or EOF and return a single :class:`str`. If the stream is already at EOF, an empty string is returned. If *size* is specified, at most *size* characters will be read. @@ -913,22 +914,22 @@ Text I/O Change the stream position to the given *offset*. Behaviour depends on the *whence* parameter. The default value for *whence* is - :data:`SEEK_SET`. + :data:`!SEEK_SET`. - * :data:`SEEK_SET` or ``0``: seek from the start of the stream + * :data:`!SEEK_SET` or ``0``: seek from the start of the stream (the default); *offset* must either be a number returned by :meth:`TextIOBase.tell`, or zero. Any other *offset* value produces undefined behaviour. - * :data:`SEEK_CUR` or ``1``: "seek" to the current position; + * :data:`!SEEK_CUR` or ``1``: "seek" to the current position; *offset* must be zero, which is a no-operation (all other values are unsupported). - * :data:`SEEK_END` or ``2``: seek to the end of the stream; + * :data:`!SEEK_END` or ``2``: seek to the end of the stream; *offset* must be zero (all other values are unsupported). Return the new absolute position as an opaque number. .. versionadded:: 3.1 - The ``SEEK_*`` constants. + The :data:`!SEEK_*` constants. .. method:: tell() @@ -988,10 +989,10 @@ Text I/O takes place. If *newline* is any of the other legal values, any ``'\n'`` characters written are translated to the given string. - If *line_buffering* is ``True``, :meth:`flush` is implied when a call to + If *line_buffering* is ``True``, :meth:`~IOBase.flush` is implied when a call to write contains a newline character or a carriage return. - If *write_through* is ``True``, calls to :meth:`write` are guaranteed + If *write_through* is ``True``, calls to :meth:`~BufferedIOBase.write` are guaranteed not to be buffered: any data written on the :class:`TextIOWrapper` object is immediately handled to its underlying binary *buffer*. @@ -1070,7 +1071,7 @@ Text I/O .. method:: getvalue() - Return a ``str`` containing the entire contents of the buffer. + Return a :class:`str` containing the entire contents of the buffer. Newlines are decoded as if by :meth:`~TextIOBase.read`, although the stream position is not changed. @@ -1125,7 +1126,7 @@ Text I/O over a binary storage (such as a file) is significantly slower than binary I/O over the same storage, because it requires conversions between unicode and binary data using a character codec. This can become noticeable handling huge amounts of text data like large log files. Also, -:meth:`TextIOWrapper.tell` and :meth:`TextIOWrapper.seek` are both quite slow +:meth:`~TextIOBase.tell` and :meth:`~TextIOBase.seek` are both quite slow due to the reconstruction algorithm used. :class:`StringIO`, however, is a native in-memory unicode container and will @@ -1135,7 +1136,7 @@ Multi-threading ^^^^^^^^^^^^^^^ :class:`FileIO` objects are thread-safe to the extent that the operating system -calls (such as ``read(2)`` under Unix) they wrap are thread-safe too. +calls (such as :manpage:`read(2)` under Unix) they wrap are thread-safe too. Binary buffered objects (instances of :class:`BufferedReader`, :class:`BufferedWriter`, :class:`BufferedRandom` and :class:`BufferedRWPair`) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 1859d53e15284..407a4bd837c54 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -90,7 +90,6 @@ Doc/library/http.server.rst Doc/library/importlib.resources.rst Doc/library/importlib.rst Doc/library/inspect.rst -Doc/library/io.rst Doc/library/locale.rst Doc/library/logging.config.rst Doc/library/logging.handlers.rst From webhook-mailer at python.org Thu Aug 17 16:41:38 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Thu, 17 Aug 2023 20:41:38 -0000 Subject: [Python-checkins] gh-107801: Improve the docs of the SEEK_* constants (#108099) Message-ID: https://github.com/python/cpython/commit/02079b010c39a89b284e8f0bb6d5f378e554260e commit: 02079b010c39a89b284e8f0bb6d5f378e554260e branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-17T20:41:35Z summary: gh-107801: Improve the docs of the SEEK_* constants (#108099) files: M Doc/library/os.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 94b76e4052b94..4668984b10c69 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1180,16 +1180,26 @@ as internal buffering of data. SEEK_CUR SEEK_END - Parameters to the :func:`lseek` function. Their values are 0, 1, and 2, - respectively. + Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` + method on :term:`file-like objects `, + for whence to adjust the file position indicator. + + :const:`SEEK_SET` + Adjust the file position relative to the beginning of the file. + :const:`SEEK_CUR` + Adjust the file position relative to the current file position. + :const:`SEEK_END` + Adjust the file position relative to the end of the file. + + Their values are 0, 1, and 2, respectively. .. data:: SEEK_HOLE SEEK_DATA Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` - method on file objects, for seeking file data and holes on sparsely - allocated files. + method on :term:`file-like objects `, + for seeking file data and holes on sparsely allocated files. :data:`!SEEK_DATA` Adjust the file offset to the next location containing data, From webhook-mailer at python.org Thu Aug 17 16:52:46 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Thu, 17 Aug 2023 20:52:46 -0000 Subject: [Python-checkins] [3.11] gh-107801: Improve the docs of the SEEK_* constants (GH-108099) (#108100) Message-ID: https://github.com/python/cpython/commit/d6555abfa7384b5a40435a11bdd2aa6bbf8f5cfc commit: d6555abfa7384b5a40435a11bdd2aa6bbf8f5cfc branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: erlend-aasland date: 2023-08-17T20:52:43Z summary: [3.11] gh-107801: Improve the docs of the SEEK_* constants (GH-108099) (#108100) (cherry picked from commit 02079b010c39a89b284e8f0bb6d5f378e554260e) Co-authored-by: Erlend E. Aasland files: M Doc/library/os.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 1d0d1049ebd7e..993236c521a87 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1070,16 +1070,26 @@ as internal buffering of data. SEEK_CUR SEEK_END - Parameters to the :func:`lseek` function. Their values are 0, 1, and 2, - respectively. + Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` + method on :term:`file-like objects `, + for whence to adjust the file position indicator. + + :const:`SEEK_SET` + Adjust the file position relative to the beginning of the file. + :const:`SEEK_CUR` + Adjust the file position relative to the current file position. + :const:`SEEK_END` + Adjust the file position relative to the end of the file. + + Their values are 0, 1, and 2, respectively. .. data:: SEEK_HOLE SEEK_DATA Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` - method on file objects, for seeking file data and holes on sparsely - allocated files. + method on :term:`file-like objects `, + for seeking file data and holes on sparsely allocated files. :data:`!SEEK_DATA` Adjust the file offset to the next location containing data, From webhook-mailer at python.org Thu Aug 17 18:40:12 2023 From: webhook-mailer at python.org (Yhg1s) Date: Thu, 17 Aug 2023 22:40:12 -0000 Subject: [Python-checkins] [3.12] gh-107801: Document SEEK_HOLE and SEEK_DATA (GH-107936) (#108086) Message-ID: https://github.com/python/cpython/commit/9342ac314985face0ae117c8f10296904f30f71a commit: 9342ac314985face0ae117c8f10296904f30f71a branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-18T00:40:09+02:00 summary: [3.12] gh-107801: Document SEEK_HOLE and SEEK_DATA (GH-107936) (#108086) gh-107801: Document SEEK_HOLE and SEEK_DATA (GH-107936) (cherry picked from commit 8a19f225b948db1eebe1d9fc71a486258841f578) Co-authored-by: Erlend E. Aasland Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Antoine Pitrou files: M Doc/library/os.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 9735baa5bc0f3..94b76e4052b94 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1171,6 +1171,10 @@ as internal buffering of data. current position; :const:`SEEK_END` or ``2`` to set it relative to the end of the file. Return the new cursor position in bytes, starting from the beginning. + .. versionchanged:: 3.3 + + Add support for :const:`SEEK_HOLE` and :const:`SEEK_DATA`. + .. data:: SEEK_SET SEEK_CUR @@ -1179,9 +1183,30 @@ as internal buffering of data. Parameters to the :func:`lseek` function. Their values are 0, 1, and 2, respectively. + +.. data:: SEEK_HOLE + SEEK_DATA + + Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` + method on file objects, for seeking file data and holes on sparsely + allocated files. + + :data:`!SEEK_DATA` + Adjust the file offset to the next location containing data, + relative to the seek position. + + :data:`!SEEK_HOLE` + Adjust the file offset to the next location containing a hole, + relative to the seek position. + A hole is defined as a sequence of zeros. + + .. note:: + + These operations only make sense for filesystems that support them. + + .. availability:: Linux >= 3.1, macOS, Unix + .. versionadded:: 3.3 - Some operating systems could support additional values, like - :const:`os.SEEK_HOLE` or :const:`os.SEEK_DATA`. .. function:: open(path, flags, mode=0o777, *, dir_fd=None) From webhook-mailer at python.org Fri Aug 18 03:12:11 2023 From: webhook-mailer at python.org (hugovk) Date: Fri, 18 Aug 2023 07:12:11 -0000 Subject: [Python-checkins] [3.11] GH-107987: Remove the Distributing Python Modules guide (GH-108016) (#108091) Message-ID: https://github.com/python/cpython/commit/6dd1729a888c50ddb312fae758cc5a92aaf5055d commit: 6dd1729a888c50ddb312fae758cc5a92aaf5055d branch: 3.11 author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: hugovk date: 2023-08-18T10:12:07+03:00 summary: [3.11] GH-107987: Remove the Distributing Python Modules guide (GH-108016) (#108091) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/conf.py M Doc/contents.rst M Doc/distributing/index.rst M Doc/installing/index.rst M Doc/tutorial/venv.rst diff --git a/Doc/conf.py b/Doc/conf.py index cfe34244fee39..161a8dec69b5d 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -346,8 +346,6 @@ latex_documents = [ ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), - ('distributing/index', 'distributing.tex', - 'Distributing Python Modules', _stdauthor, 'manual'), ('extending/index', 'extending.tex', 'Extending and Embedding Python', _stdauthor, 'manual'), ('installing/index', 'installing.tex', diff --git a/Doc/contents.rst b/Doc/contents.rst index 8690de77bf3d8..3eedede7ba49a 100644 --- a/Doc/contents.rst +++ b/Doc/contents.rst @@ -11,7 +11,6 @@ library/index.rst extending/index.rst c-api/index.rst - distributing/index.rst installing/index.rst howto/index.rst faq/index.rst diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 8e70c24e70266..8622ec60c0f2f 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -1,175 +1,20 @@ +:orphan: + +.. This page is retained solely for existing links to /distributing/index.html. + Direct readers to the PPUG instead. + .. _distributing-index: +.. _publishing-python-packages: ############################### Distributing Python Modules ############################### -:Email: distutils-sig at python.org - - -As a popular open source development project, Python has an active -supporting community of contributors and users that also make their software -available for other Python developers to use under open source license terms. - -This allows Python users to share and collaborate effectively, benefiting -from the solutions others have already created to common (and sometimes -even rare!) problems, as well as potentially contributing their own -solutions to the common pool. - -This guide covers the distribution part of the process. For a guide to -installing other Python projects, refer to the -:ref:`installation guide `. - .. note:: - For corporate and other institutional users, be aware that many - organisations have their own policies around using and contributing to - open source software. Please take such policies into account when making - use of the distribution and installation tools provided with Python. - - -Key terms -========= - -* the `Python Package Index `__ is a public - repository of open source licensed packages made available for use by - other Python users -* the `Python Packaging Authority - `__ are the group of - developers and documentation authors responsible for the maintenance and - evolution of the standard packaging tools and the associated metadata and - file format standards. They maintain a variety of tools, documentation - and issue trackers on both `GitHub `__ and - `Bitbucket `__. -* :mod:`distutils` is the original build and distribution system first added - to the Python standard library in 1998. While direct use of :mod:`distutils` - is being phased out, it still laid the foundation for the current packaging - and distribution infrastructure, and it not only remains part of the - standard library, but its name lives on in other ways (such as the name - of the mailing list used to coordinate Python packaging standards - development). -* `setuptools`_ is a (largely) drop-in replacement for :mod:`distutils` first - published in 2004. Its most notable addition over the unmodified - :mod:`distutils` tools was the ability to declare dependencies on other - packages. It is currently recommended as a more regularly updated - alternative to :mod:`distutils` that offers consistent support for more - recent packaging standards across a wide range of Python versions. -* `wheel`_ (in this context) is a project that adds the ``bdist_wheel`` - command to :mod:`distutils`/`setuptools`_. This produces a cross platform - binary packaging format (called "wheels" or "wheel files" and defined in - :pep:`427`) that allows Python libraries, even those including binary - extensions, to be installed on a system without needing to be built - locally. - -.. _setuptools: https://setuptools.readthedocs.io/en/latest/ -.. _wheel: https://wheel.readthedocs.io/ - -Open source licensing and collaboration -======================================= - -In most parts of the world, software is automatically covered by copyright. -This means that other developers require explicit permission to copy, use, -modify and redistribute the software. - -Open source licensing is a way of explicitly granting such permission in a -relatively consistent way, allowing developers to share and collaborate -efficiently by making common solutions to various problems freely available. -This leaves many developers free to spend more time focusing on the problems -that are relatively unique to their specific situation. - -The distribution tools provided with Python are designed to make it -reasonably straightforward for developers to make their own contributions -back to that common pool of software if they choose to do so. - -The same distribution tools can also be used to distribute software within -an organisation, regardless of whether that software is published as open -source software or not. - - -Installing the tools -==================== - -The standard library does not include build tools that support modern -Python packaging standards, as the core development team has found that it -is important to have standard tools that work consistently, even on older -versions of Python. - -The currently recommended build and distribution tools can be installed -by invoking the ``pip`` module at the command line:: - - python -m pip install setuptools wheel twine - -.. note:: - - For POSIX users (including macOS and Linux users), these instructions - assume the use of a :term:`virtual environment`. - - For Windows users, these instructions assume that the option to - adjust the system PATH environment variable was selected when installing - Python. - -The Python Packaging User Guide includes more details on the `currently -recommended tools`_. - -.. _currently recommended tools: https://packaging.python.org/guides/tool-recommendations/#packaging-tool-recommendations - -.. index:: - single: Python Package Index (PyPI) - single: PyPI; (see Python Package Index (PyPI)) - -.. _publishing-python-packages: - -Reading the Python Packaging User Guide -======================================= - -The Python Packaging User Guide covers the various key steps and elements -involved in creating and publishing a project: - -* `Project structure`_ -* `Building and packaging the project`_ -* `Uploading the project to the Python Package Index`_ -* `The .pypirc file`_ - -.. _Project structure: https://packaging.python.org/tutorials/packaging-projects/#packaging-python-projects -.. _Building and packaging the project: https://packaging.python.org/tutorials/packaging-projects/#creating-the-package-files -.. _Uploading the project to the Python Package Index: https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives -.. _The .pypirc file: https://packaging.python.org/specifications/pypirc/ - - -How do I...? -============ - -These are quick answers or links for some common tasks. - -... choose a name for my project? ---------------------------------- - -This isn't an easy topic, but here are a few tips: - -* check the Python Package Index to see if the name is already in use -* check popular hosting sites like GitHub, Bitbucket, etc to see if there - is already a project with that name -* check what comes up in a web search for the name you're considering -* avoid particularly common words, especially ones with multiple meanings, - as they can make it difficult for users to find your software when - searching for it - - -... create and distribute binary extensions? --------------------------------------------- - -This is actually quite a complex topic, with a variety of alternatives -available depending on exactly what you're aiming to achieve. See the -Python Packaging User Guide for more information and recommendations. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions - `__ - -.. other topics: + Information and guidance on distributing Python modules and packages + has been moved to the `Python Packaging User Guide`_, + and the tutorial on `packaging Python projects`_. - Once the Development & Deployment part of PPUG is fleshed out, some of - those sections should be linked from new questions here (most notably, - we should have a question about avoiding depending on PyPI that links to - https://packaging.python.org/en/latest/mirrors/) + .. _Python Packaging User Guide: https://packaging.python.org/ + .. _packaging Python projects: https://packaging.python.org/en/latest/tutorials/packaging-projects/ diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index e158bf1c4c0c7..da4a2b3c186a7 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -19,7 +19,9 @@ solutions to the common pool. This guide covers the installation part of the process. For a guide to creating and sharing your own Python projects, refer to the -:ref:`distribution guide `. +`Python packaging user guide`_. + +.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/packaging-projects/ .. note:: diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index 9a3f49d1380d8..d91fc3de4f92d 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -207,4 +207,6 @@ necessary packages with ``install -r``: ``pip`` has many more options. Consult the :ref:`installing-index` guide for complete documentation for ``pip``. When you've written a package and want to make it available on the Python Package Index, -consult the :ref:`distributing-index` guide. +consult the `Python packaging user guide`_. + +.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/packaging-projects/ From webhook-mailer at python.org Fri Aug 18 04:55:37 2023 From: webhook-mailer at python.org (Yhg1s) Date: Fri, 18 Aug 2023 08:55:37 -0000 Subject: [Python-checkins] [3.12] Docs: Fix Sphinx warnings in io.rst (GH-107903) (#108093) Message-ID: https://github.com/python/cpython/commit/60edc70a9374f1cc6ecff5974e438d58fec29985 commit: 60edc70a9374f1cc6ecff5974e438d58fec29985 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-18T10:55:33+02:00 summary: [3.12] Docs: Fix Sphinx warnings in io.rst (GH-107903) (#108093) Docs: Fix Sphinx warnings in io.rst (GH-107903) - Mark up parameter and argument names properly - If possible, link to docs for methods like `seek`, `tell`, `write`, `read`, etc. (cherry picked from commit 5c76899dadf3bdcfdedf6f30b3ab9742cb87af04) Co-authored-by: Erlend E. Aasland Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: T. Wouters files: M Doc/library/io.rst M Doc/tools/.nitignore diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 7eec1f87583b8..66273d9ed1ff0 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -38,9 +38,9 @@ location), or only sequential access (for example in the case of a socket or pipe). All streams are careful about the type of data you give to them. For example -giving a :class:`str` object to the ``write()`` method of a binary stream +giving a :class:`str` object to the :meth:`!write` method of a binary stream will raise a :exc:`TypeError`. So will giving a :class:`bytes` object to the -``write()`` method of a text stream. +:meth:`!write` method of a text stream. .. versionchanged:: 3.3 Operations that used to raise :exc:`IOError` now raise :exc:`OSError`, since @@ -146,7 +146,7 @@ Opt-in EncodingWarning See :pep:`597` for more details. To find where the default locale encoding is used, you can enable -the ``-X warn_default_encoding`` command line option or set the +the :option:`-X warn_default_encoding <-X>` command line option or set the :envvar:`PYTHONWARNDEFAULTENCODING` environment variable, which will emit an :exc:`EncodingWarning` when the default encoding is used. @@ -175,7 +175,7 @@ High-level Module Interface .. audit-event:: open path,mode,flags io.open This function raises an :ref:`auditing event ` ``open`` with - arguments ``path``, ``mode`` and ``flags``. The ``mode`` and ``flags`` + arguments *path*, *mode* and *flags*. The *mode* and *flags* arguments may have been modified or inferred from the original call. @@ -184,10 +184,10 @@ High-level Module Interface Opens the provided file with mode ``'rb'``. This function should be used when the intent is to treat the contents as executable code. - ``path`` should be a :class:`str` and an absolute path. + *path* should be a :class:`str` and an absolute path. The behavior of this function may be overridden by an earlier call to the - :c:func:`PyFile_SetOpenCodeHook`. However, assuming that ``path`` is a + :c:func:`PyFile_SetOpenCodeHook`. However, assuming that *path* is a :class:`str` and an absolute path, ``open_code(path)`` should always behave the same as ``open(path, 'rb')``. Overriding the behavior is intended for additional validation or preprocessing of the file. @@ -258,7 +258,7 @@ standard stream implementations. The abstract base classes also provide default implementations of some methods in order to help implementation of concrete stream classes. For example, :class:`BufferedIOBase` provides unoptimized implementations of - :meth:`~IOBase.readinto` and :meth:`~IOBase.readline`. + :meth:`!readinto` and :meth:`!readline`. At the top of the I/O hierarchy is the abstract base class :class:`IOBase`. It defines the basic interface to a stream. Note, however, that there is no @@ -320,8 +320,8 @@ I/O Base Classes implementations represent a file that cannot be read, written or seeked. - Even though :class:`IOBase` does not declare :meth:`read` - or :meth:`write` because their signatures will vary, implementations and + Even though :class:`IOBase` does not declare :meth:`!read` + or :meth:`!write` because their signatures will vary, implementations and clients should consider those methods part of the interface. Also, implementations may raise a :exc:`ValueError` (or :exc:`UnsupportedOperation`) when operations they do not support are called. @@ -379,8 +379,8 @@ I/O Base Classes .. method:: readable() - Return ``True`` if the stream can be read from. If ``False``, :meth:`read` - will raise :exc:`OSError`. + Return ``True`` if the stream can be read from. + If ``False``, :meth:`!read` will raise :exc:`OSError`. .. method:: readline(size=-1, /) @@ -401,25 +401,25 @@ I/O Base Classes hint. Note that it's already possible to iterate on file objects using ``for - line in file: ...`` without calling ``file.readlines()``. + line in file: ...`` without calling :meth:`!file.readlines`. .. method:: seek(offset, whence=SEEK_SET, /) Change the stream position to the given byte *offset*. *offset* is interpreted relative to the position indicated by *whence*. The default - value for *whence* is :data:`SEEK_SET`. Values for *whence* are: + value for *whence* is :data:`!SEEK_SET`. Values for *whence* are: - * :data:`SEEK_SET` or ``0`` -- start of the stream (the default); + * :data:`!SEEK_SET` or ``0`` -- start of the stream (the default); *offset* should be zero or positive - * :data:`SEEK_CUR` or ``1`` -- current stream position; *offset* may + * :data:`!SEEK_CUR` or ``1`` -- current stream position; *offset* may be negative - * :data:`SEEK_END` or ``2`` -- end of the stream; *offset* is usually + * :data:`!SEEK_END` or ``2`` -- end of the stream; *offset* is usually negative Return the new absolute position. .. versionadded:: 3.1 - The ``SEEK_*`` constants. + The :data:`!SEEK_*` constants. .. versionadded:: 3.3 Some operating systems could support additional values, like @@ -450,7 +450,7 @@ I/O Base Classes .. method:: writable() Return ``True`` if the stream supports writing. If ``False``, - :meth:`write` and :meth:`truncate` will raise :exc:`OSError`. + :meth:`!write` and :meth:`truncate` will raise :exc:`OSError`. .. method:: writelines(lines, /) @@ -654,8 +654,9 @@ Raw File I/O implies writing, so this mode behaves in a similar way to ``'w'``. Add a ``'+'`` to the mode to allow simultaneous reading and writing. - The :meth:`read` (when called with a positive argument), :meth:`readinto` - and :meth:`write` methods on this class will only make one system call. + The :meth:`~RawIOBase.read` (when called with a positive argument), + :meth:`~RawIOBase.readinto` and :meth:`~RawIOBase.write` methods on this + class will only make one system call. A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by calling *opener* with @@ -791,8 +792,8 @@ than raw I/O does. object under various conditions, including: * when the buffer gets too small for all pending data; - * when :meth:`flush()` is called; - * when a :meth:`seek()` is requested (for :class:`BufferedRandom` objects); + * when :meth:`flush` is called; + * when a :meth:`~IOBase.seek` is requested (for :class:`BufferedRandom` objects); * when the :class:`BufferedWriter` object is closed or destroyed. The constructor creates a :class:`BufferedWriter` for the given writeable @@ -826,8 +827,8 @@ than raw I/O does. :data:`DEFAULT_BUFFER_SIZE`. :class:`BufferedRandom` is capable of anything :class:`BufferedReader` or - :class:`BufferedWriter` can do. In addition, :meth:`seek` and :meth:`tell` - are guaranteed to be implemented. + :class:`BufferedWriter` can do. In addition, :meth:`~IOBase.seek` and + :meth:`~IOBase.tell` are guaranteed to be implemented. .. class:: BufferedRWPair(reader, writer, buffer_size=DEFAULT_BUFFER_SIZE, /) @@ -904,7 +905,7 @@ Text I/O .. method:: readline(size=-1, /) - Read until newline or EOF and return a single ``str``. If the stream is + Read until newline or EOF and return a single :class:`str`. If the stream is already at EOF, an empty string is returned. If *size* is specified, at most *size* characters will be read. @@ -913,22 +914,22 @@ Text I/O Change the stream position to the given *offset*. Behaviour depends on the *whence* parameter. The default value for *whence* is - :data:`SEEK_SET`. + :data:`!SEEK_SET`. - * :data:`SEEK_SET` or ``0``: seek from the start of the stream + * :data:`!SEEK_SET` or ``0``: seek from the start of the stream (the default); *offset* must either be a number returned by :meth:`TextIOBase.tell`, or zero. Any other *offset* value produces undefined behaviour. - * :data:`SEEK_CUR` or ``1``: "seek" to the current position; + * :data:`!SEEK_CUR` or ``1``: "seek" to the current position; *offset* must be zero, which is a no-operation (all other values are unsupported). - * :data:`SEEK_END` or ``2``: seek to the end of the stream; + * :data:`!SEEK_END` or ``2``: seek to the end of the stream; *offset* must be zero (all other values are unsupported). Return the new absolute position as an opaque number. .. versionadded:: 3.1 - The ``SEEK_*`` constants. + The :data:`!SEEK_*` constants. .. method:: tell() @@ -988,10 +989,10 @@ Text I/O takes place. If *newline* is any of the other legal values, any ``'\n'`` characters written are translated to the given string. - If *line_buffering* is ``True``, :meth:`flush` is implied when a call to + If *line_buffering* is ``True``, :meth:`~IOBase.flush` is implied when a call to write contains a newline character or a carriage return. - If *write_through* is ``True``, calls to :meth:`write` are guaranteed + If *write_through* is ``True``, calls to :meth:`~BufferedIOBase.write` are guaranteed not to be buffered: any data written on the :class:`TextIOWrapper` object is immediately handled to its underlying binary *buffer*. @@ -1070,7 +1071,7 @@ Text I/O .. method:: getvalue() - Return a ``str`` containing the entire contents of the buffer. + Return a :class:`str` containing the entire contents of the buffer. Newlines are decoded as if by :meth:`~TextIOBase.read`, although the stream position is not changed. @@ -1125,7 +1126,7 @@ Text I/O over a binary storage (such as a file) is significantly slower than binary I/O over the same storage, because it requires conversions between unicode and binary data using a character codec. This can become noticeable handling huge amounts of text data like large log files. Also, -:meth:`TextIOWrapper.tell` and :meth:`TextIOWrapper.seek` are both quite slow +:meth:`~TextIOBase.tell` and :meth:`~TextIOBase.seek` are both quite slow due to the reconstruction algorithm used. :class:`StringIO`, however, is a native in-memory unicode container and will @@ -1135,7 +1136,7 @@ Multi-threading ^^^^^^^^^^^^^^^ :class:`FileIO` objects are thread-safe to the extent that the operating system -calls (such as ``read(2)`` under Unix) they wrap are thread-safe too. +calls (such as :manpage:`read(2)` under Unix) they wrap are thread-safe too. Binary buffered objects (instances of :class:`BufferedReader`, :class:`BufferedWriter`, :class:`BufferedRandom` and :class:`BufferedRWPair`) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 9a0a6c2561d12..f7f2d6d8f3289 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -100,7 +100,6 @@ Doc/library/http.server.rst Doc/library/importlib.resources.rst Doc/library/importlib.rst Doc/library/inspect.rst -Doc/library/io.rst Doc/library/locale.rst Doc/library/logging.config.rst Doc/library/logging.handlers.rst From webhook-mailer at python.org Fri Aug 18 06:16:26 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Fri, 18 Aug 2023 10:16:26 -0000 Subject: [Python-checkins] Docs: emphasise warning and add accurate markups for sys.unraisablehook (#108105) Message-ID: https://github.com/python/cpython/commit/cc58ec9724772a8d5c4a5c9a6525f9f96e994227 commit: cc58ec9724772a8d5c4a5c9a6525f9f96e994227 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-18T12:16:22+02:00 summary: Docs: emphasise warning and add accurate markups for sys.unraisablehook (#108105) files: M Doc/library/sys.rst diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 33391d11ab392..b6346bafbf625 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1811,35 +1811,39 @@ always available. The *unraisable* argument has the following attributes: - * *exc_type*: Exception type. - * *exc_value*: Exception value, can be ``None``. - * *exc_traceback*: Exception traceback, can be ``None``. - * *err_msg*: Error message, can be ``None``. - * *object*: Object causing the exception, can be ``None``. + * :attr:`!exc_type`: Exception type. + * :attr:`!exc_value`: Exception value, can be ``None``. + * :attr:`!exc_traceback`: Exception traceback, can be ``None``. + * :attr:`!err_msg`: Error message, can be ``None``. + * :attr:`!object`: Object causing the exception, can be ``None``. - The default hook formats *err_msg* and *object* as: + The default hook formats :attr:`!err_msg` and :attr:`!object` as: ``f'{err_msg}: {object!r}'``; use "Exception ignored in" error message - if *err_msg* is ``None``. + if :attr:`!err_msg` is ``None``. :func:`sys.unraisablehook` can be overridden to control how unraisable exceptions are handled. - Storing *exc_value* using a custom hook can create a reference cycle. It - should be cleared explicitly to break the reference cycle when the - exception is no longer needed. + .. seealso:: + + :func:`excepthook` which handles uncaught exceptions. + + .. warning:: - Storing *object* using a custom hook can resurrect it if it is set to an - object which is being finalized. Avoid storing *object* after the custom - hook completes to avoid resurrecting objects. + Storing :attr:`!exc_value` using a custom hook can create a reference cycle. + It should be cleared explicitly to break the reference cycle when the + exception is no longer needed. - See also :func:`excepthook` which handles uncaught exceptions. + Storing :attr:`!object` using a custom hook can resurrect it if it is set to an + object which is being finalized. Avoid storing :attr:`!object` after the custom + hook completes to avoid resurrecting objects. .. audit-event:: sys.unraisablehook hook,unraisable sys.unraisablehook Raise an auditing event ``sys.unraisablehook`` with arguments - ``hook``, ``unraisable`` when an exception that cannot be handled occurs. - The ``unraisable`` object is the same as what will be passed to the hook. - If no hook has been set, ``hook`` may be ``None``. + *hook*, *unraisable* when an exception that cannot be handled occurs. + The *unraisable* object is the same as what will be passed to the hook. + If no hook has been set, *hook* may be ``None``. .. versionadded:: 3.8 From webhook-mailer at python.org Fri Aug 18 06:25:12 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Fri, 18 Aug 2023 10:25:12 -0000 Subject: [Python-checkins] [3.11] Docs: emphasise warning and add accurate markups for sys.unraisablehook (GH-108105) (#108110) Message-ID: https://github.com/python/cpython/commit/9a70989a6f5f3f9d6fd36de002dab0ab1e13a9e3 commit: 9a70989a6f5f3f9d6fd36de002dab0ab1e13a9e3 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: erlend-aasland date: 2023-08-18T10:25:08Z summary: [3.11] Docs: emphasise warning and add accurate markups for sys.unraisablehook (GH-108105) (#108110) (cherry picked from commit cc58ec9724772a8d5c4a5c9a6525f9f96e994227) Co-authored-by: Erlend E. Aasland files: M Doc/library/sys.rst diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 641be05a16fb3..b5ad0b6860b95 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1747,35 +1747,39 @@ always available. The *unraisable* argument has the following attributes: - * *exc_type*: Exception type. - * *exc_value*: Exception value, can be ``None``. - * *exc_traceback*: Exception traceback, can be ``None``. - * *err_msg*: Error message, can be ``None``. - * *object*: Object causing the exception, can be ``None``. + * :attr:`!exc_type`: Exception type. + * :attr:`!exc_value`: Exception value, can be ``None``. + * :attr:`!exc_traceback`: Exception traceback, can be ``None``. + * :attr:`!err_msg`: Error message, can be ``None``. + * :attr:`!object`: Object causing the exception, can be ``None``. - The default hook formats *err_msg* and *object* as: + The default hook formats :attr:`!err_msg` and :attr:`!object` as: ``f'{err_msg}: {object!r}'``; use "Exception ignored in" error message - if *err_msg* is ``None``. + if :attr:`!err_msg` is ``None``. :func:`sys.unraisablehook` can be overridden to control how unraisable exceptions are handled. - Storing *exc_value* using a custom hook can create a reference cycle. It - should be cleared explicitly to break the reference cycle when the - exception is no longer needed. + .. seealso:: + + :func:`excepthook` which handles uncaught exceptions. + + .. warning:: - Storing *object* using a custom hook can resurrect it if it is set to an - object which is being finalized. Avoid storing *object* after the custom - hook completes to avoid resurrecting objects. + Storing :attr:`!exc_value` using a custom hook can create a reference cycle. + It should be cleared explicitly to break the reference cycle when the + exception is no longer needed. - See also :func:`excepthook` which handles uncaught exceptions. + Storing :attr:`!object` using a custom hook can resurrect it if it is set to an + object which is being finalized. Avoid storing :attr:`!object` after the custom + hook completes to avoid resurrecting objects. .. audit-event:: sys.unraisablehook hook,unraisable sys.unraisablehook Raise an auditing event ``sys.unraisablehook`` with arguments - ``hook``, ``unraisable`` when an exception that cannot be handled occurs. - The ``unraisable`` object is the same as what will be passed to the hook. - If no hook has been set, ``hook`` may be ``None``. + *hook*, *unraisable* when an exception that cannot be handled occurs. + The *unraisable* object is the same as what will be passed to the hook. + If no hook has been set, *hook* may be ``None``. .. versionadded:: 3.8 From webhook-mailer at python.org Fri Aug 18 06:34:45 2023 From: webhook-mailer at python.org (vstinner) Date: Fri, 18 Aug 2023 10:34:45 -0000 Subject: [Python-checkins] gh-108014: Add Py_IsFinalizing() function (#108032) Message-ID: https://github.com/python/cpython/commit/3ff5ef2ad3d89c3ccf4e07ac8fdd798267ae6c61 commit: 3ff5ef2ad3d89c3ccf4e07ac8fdd798267ae6c61 branch: main author: Victor Stinner committer: vstinner date: 2023-08-18T12:34:41+02:00 summary: gh-108014: Add Py_IsFinalizing() function (#108032) Co-authored-by: Serhiy Storchaka files: A Misc/NEWS.d/next/C API/2023-08-16-17-16-19.gh-issue-108014.wXN3CF.rst M Doc/c-api/init.rst M Doc/library/sys.rst M Doc/whatsnew/3.13.rst M Include/cpython/pylifecycle.h M Include/internal/pycore_pylifecycle.h M Modules/_asynciomodule.c M Python/pylifecycle.c M Python/sysmodule.c diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index dd53fe2bc9569..60f5c81cff572 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -373,6 +373,14 @@ Initializing and finalizing the interpreter :c:func:`Py_Initialize` is called again. +.. c:function:: int Py_IsFinalizing() + + Return true (non-zero) if the main Python interpreter is + :term:`shutting down `. Return false (zero) otherwise. + + .. versionadded:: 3.13 + + .. c:function:: int Py_FinalizeEx() Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of @@ -852,7 +860,7 @@ code, or when embedding the Python interpreter: .. note:: Calling this function from a thread when the runtime is finalizing will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing` to + You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to check if the interpreter is in process of being finalized before calling this function to avoid unwanted termination. @@ -898,7 +906,7 @@ with sub-interpreters: .. note:: Calling this function from a thread when the runtime is finalizing will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing` to + You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to check if the interpreter is in process of being finalized before calling this function to avoid unwanted termination. @@ -1180,7 +1188,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. note:: Calling this function from a thread when the runtime is finalizing will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing` to + You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to check if the interpreter is in process of being finalized before calling this function to avoid unwanted termination. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index b6346bafbf625..da2435aa4bc23 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1121,8 +1121,8 @@ always available. .. function:: is_finalizing() - Return :const:`True` if the Python interpreter is - :term:`shutting down `, :const:`False` otherwise. + Return :const:`True` if the main Python interpreter is + :term:`shutting down `. Return :const:`False` otherwise. .. versionadded:: 3.5 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 13ae6e595dc99..85aa81b4a98d2 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -832,6 +832,10 @@ New Features not needed. (Contributed by Victor Stinner in :gh:`106004`.) +* Add :c:func:`Py_IsFinalizing` function: check if the main Python interpreter is + :term:`shutting down `. + (Contributed by Victor Stinner in :gh:`108014`.) + Porting to Python 3.13 ---------------------- diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index d425a233f7100..11b280afa8435 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -81,3 +81,5 @@ PyAPI_FUNC(PyStatus) Py_NewInterpreterFromConfig( typedef void (*atexit_datacallbackfunc)(void *); PyAPI_FUNC(int) PyUnstable_AtExit( PyInterpreterState *, atexit_datacallbackfunc, void *); + +PyAPI_FUNC(int) Py_IsFinalizing(void); diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index b4d5b1f1239e1..56abd57d2bd5c 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -98,7 +98,6 @@ extern int _Py_FdIsInteractive(FILE *fp, PyObject *filename); extern const char* _Py_gitidentifier(void); extern const char* _Py_gitversion(void); -extern int _Py_IsFinalizing(void); PyAPI_FUNC(int) _Py_IsInterpreterFinalizing(PyInterpreterState *interp); /* Random */ diff --git a/Misc/NEWS.d/next/C API/2023-08-16-17-16-19.gh-issue-108014.wXN3CF.rst b/Misc/NEWS.d/next/C API/2023-08-16-17-16-19.gh-issue-108014.wXN3CF.rst new file mode 100644 index 0000000000000..fee3d5b941dc8 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-16-17-16-19.gh-issue-108014.wXN3CF.rst @@ -0,0 +1,2 @@ +Add :c:func:`Py_IsFinalizing` function: check if the main Python interpreter is +:term:`shutting down `. Patch by Victor Stinner. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 39c803355ba95..6266dc8e3555f 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -529,7 +529,7 @@ future_init(FutureObj *fut, PyObject *loop) } if (is_true && !_Py_IsInterpreterFinalizing(_PyInterpreterState_GET())) { /* Only try to capture the traceback if the interpreter is not being - finalized. The original motivation to add a `_Py_IsFinalizing()` + finalized. The original motivation to add a `Py_IsFinalizing()` call was to prevent SIGSEGV when a Future is created in a __del__ method, which is called during the interpreter shutdown and the traceback module is already unloaded. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 90633960cb00e..263ead39fa472 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -129,7 +129,7 @@ _PyRuntime_Finalize(void) } int -_Py_IsFinalizing(void) +Py_IsFinalizing(void) { return _PyRuntimeState_GetFinalizing(&_PyRuntime) != NULL; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index f82901181f886..54533603f1a7f 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2095,7 +2095,7 @@ static PyObject * sys_is_finalizing_impl(PyObject *module) /*[clinic end generated code: output=735b5ff7962ab281 input=f0df747a039948a5]*/ { - return PyBool_FromLong(_Py_IsFinalizing()); + return PyBool_FromLong(Py_IsFinalizing()); } #ifdef Py_STATS From webhook-mailer at python.org Fri Aug 18 07:39:16 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Fri, 18 Aug 2023 11:39:16 -0000 Subject: [Python-checkins] gh-108083: Don't ignore exceptions in sqlite3.Connection.__init__() and .close() (#108084) Message-ID: https://github.com/python/cpython/commit/fd195092204aa7fc9f13c5c6d423bc723d0b3520 commit: fd195092204aa7fc9f13c5c6d423bc723d0b3520 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-18T11:39:12Z summary: gh-108083: Don't ignore exceptions in sqlite3.Connection.__init__() and .close() (#108084) - Add explanatory comments - Add return value to connection_close() for propagating errors - Always check the return value of connection_exec_stmt() - Assert pre/post state in remove_callbacks() - Don't log unraisable exceptions in case of interpreter shutdown - Make sure we're not initialized if reinit fails - Try to close the database even if ROLLBACK fails Co-authored-by: Victor Stinner Co-authored-by: Serhiy Storchaka files: A Misc/NEWS.d/next/Library/2023-08-17-12-59-35.gh-issue-108083.9J7UcT.rst M Modules/_sqlite/connection.c diff --git a/Misc/NEWS.d/next/Library/2023-08-17-12-59-35.gh-issue-108083.9J7UcT.rst b/Misc/NEWS.d/next/Library/2023-08-17-12-59-35.gh-issue-108083.9J7UcT.rst new file mode 100644 index 0000000000000..ff499ced73a30 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-17-12-59-35.gh-issue-108083.9J7UcT.rst @@ -0,0 +1,3 @@ +Fix bugs in the constructor of :mod:`sqlite3.Connection` and +:meth:`sqlite3.Connection.close` where exceptions could be leaked. Patch by +Erlend E. Aasland. diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 0819acd094096..282855f886594 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -149,7 +149,7 @@ static void _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self); static void free_callback_context(callback_context *ctx); static void set_callback_context(callback_context **ctx_pp, callback_context *ctx); -static void connection_close(pysqlite_Connection *self); +static int connection_close(pysqlite_Connection *self); PyObject *_pysqlite_query_execute(pysqlite_Cursor *, int, PyObject *, PyObject *); static PyObject * @@ -247,10 +247,13 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, } if (self->initialized) { + self->initialized = 0; + PyTypeObject *tp = Py_TYPE(self); tp->tp_clear((PyObject *)self); - connection_close(self); - self->initialized = 0; + if (connection_close(self) < 0) { + return -1; + } } // Create and configure SQLite database object. @@ -334,7 +337,9 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, self->initialized = 1; if (autocommit == AUTOCOMMIT_DISABLED) { - (void)connection_exec_stmt(self, "BEGIN"); + if (connection_exec_stmt(self, "BEGIN") < 0) { + return -1; + } } return 0; @@ -432,48 +437,83 @@ free_callback_contexts(pysqlite_Connection *self) static void remove_callbacks(sqlite3 *db) { - sqlite3_trace_v2(db, SQLITE_TRACE_STMT, 0, 0); + /* None of these APIs can fail, as long as they are given a valid + * database pointer. */ + assert(db != NULL); + int rc = sqlite3_trace_v2(db, SQLITE_TRACE_STMT, 0, 0); + assert(rc == SQLITE_OK), (void)rc; + sqlite3_progress_handler(db, 0, 0, (void *)0); - (void)sqlite3_set_authorizer(db, NULL, NULL); + + rc = sqlite3_set_authorizer(db, NULL, NULL); + assert(rc == SQLITE_OK), (void)rc; } -static void +static int connection_close(pysqlite_Connection *self) { - if (self->db) { - if (self->autocommit == AUTOCOMMIT_DISABLED && - !sqlite3_get_autocommit(self->db)) - { - /* If close is implicitly called as a result of interpreter - * tear-down, we must not call back into Python. */ - if (_Py_IsInterpreterFinalizing(PyInterpreterState_Get())) { - remove_callbacks(self->db); - } - (void)connection_exec_stmt(self, "ROLLBACK"); + if (self->db == NULL) { + return 0; + } + + int rc = 0; + if (self->autocommit == AUTOCOMMIT_DISABLED && + !sqlite3_get_autocommit(self->db)) + { + if (connection_exec_stmt(self, "ROLLBACK") < 0) { + rc = -1; } + } - free_callback_contexts(self); + sqlite3 *db = self->db; + self->db = NULL; - sqlite3 *db = self->db; - self->db = NULL; + Py_BEGIN_ALLOW_THREADS + /* The v2 close call always returns SQLITE_OK if given a valid database + * pointer (which we do), so we can safely ignore the return value */ + (void)sqlite3_close_v2(db); + Py_END_ALLOW_THREADS - Py_BEGIN_ALLOW_THREADS - int rc = sqlite3_close_v2(db); - assert(rc == SQLITE_OK), (void)rc; - Py_END_ALLOW_THREADS - } + free_callback_contexts(self); + return rc; } static void -connection_dealloc(pysqlite_Connection *self) +connection_finalize(PyObject *self) { - PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); - tp->tp_clear((PyObject *)self); + pysqlite_Connection *con = (pysqlite_Connection *)self; + PyObject *exc = PyErr_GetRaisedException(); + + /* If close is implicitly called as a result of interpreter + * tear-down, we must not call back into Python. */ + PyInterpreterState *interp = PyInterpreterState_Get(); + int teardown = _Py_IsInterpreterFinalizing(interp); + if (teardown && con->db) { + remove_callbacks(con->db); + } /* Clean up if user has not called .close() explicitly. */ - connection_close(self); + if (connection_close(con) < 0) { + if (teardown) { + PyErr_Clear(); + } + else { + PyErr_WriteUnraisable((PyObject *)self); + } + } + + PyErr_SetRaisedException(exc); +} +static void +connection_dealloc(PyObject *self) +{ + if (PyObject_CallFinalizerFromDealloc(self) < 0) { + return; + } + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + tp->tp_clear(self); tp->tp_free(self); Py_DECREF(tp); } @@ -621,7 +661,9 @@ pysqlite_connection_close_impl(pysqlite_Connection *self) pysqlite_close_all_blobs(self); Py_CLEAR(self->statement_cache); - connection_close(self); + if (connection_close(self) < 0) { + return NULL; + } Py_RETURN_NONE; } @@ -2555,6 +2597,7 @@ static struct PyMemberDef connection_members[] = }; static PyType_Slot connection_slots[] = { + {Py_tp_finalize, connection_finalize}, {Py_tp_dealloc, connection_dealloc}, {Py_tp_doc, (void *)connection_doc}, {Py_tp_methods, connection_methods}, From webhook-mailer at python.org Fri Aug 18 08:16:45 2023 From: webhook-mailer at python.org (Yhg1s) Date: Fri, 18 Aug 2023 12:16:45 -0000 Subject: [Python-checkins] [3.12] gh-107801: Improve the docs of the SEEK_* constants (#108099) (#108108) Message-ID: https://github.com/python/cpython/commit/2b8e0207cb91c5377ea82638d038e737070c64bb commit: 2b8e0207cb91c5377ea82638d038e737070c64bb branch: 3.12 author: Erlend E. Aasland committer: Yhg1s date: 2023-08-18T14:16:42+02:00 summary: [3.12] gh-107801: Improve the docs of the SEEK_* constants (#108099) (#108108) (cherry picked from commit 02079b010c39a89b284e8f0bb6d5f378e554260e) files: M Doc/library/os.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 94b76e4052b94..4668984b10c69 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1180,16 +1180,26 @@ as internal buffering of data. SEEK_CUR SEEK_END - Parameters to the :func:`lseek` function. Their values are 0, 1, and 2, - respectively. + Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` + method on :term:`file-like objects `, + for whence to adjust the file position indicator. + + :const:`SEEK_SET` + Adjust the file position relative to the beginning of the file. + :const:`SEEK_CUR` + Adjust the file position relative to the current file position. + :const:`SEEK_END` + Adjust the file position relative to the end of the file. + + Their values are 0, 1, and 2, respectively. .. data:: SEEK_HOLE SEEK_DATA Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` - method on file objects, for seeking file data and holes on sparsely - allocated files. + method on :term:`file-like objects `, + for seeking file data and holes on sparsely allocated files. :data:`!SEEK_DATA` Adjust the file offset to the next location containing data, From webhook-mailer at python.org Fri Aug 18 08:17:16 2023 From: webhook-mailer at python.org (Yhg1s) Date: Fri, 18 Aug 2023 12:17:16 -0000 Subject: [Python-checkins] [3.12] Docs: emphasise warning and add accurate markups for sys.unraisablehook (GH-108105) (#108109) Message-ID: https://github.com/python/cpython/commit/af6e5fa718535b4ccbfa47f382584878aaa69bc2 commit: af6e5fa718535b4ccbfa47f382584878aaa69bc2 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-18T14:17:12+02:00 summary: [3.12] Docs: emphasise warning and add accurate markups for sys.unraisablehook (GH-108105) (#108109) Docs: emphasise warning and add accurate markups for sys.unraisablehook (GH-108105) (cherry picked from commit cc58ec9724772a8d5c4a5c9a6525f9f96e994227) Co-authored-by: Erlend E. Aasland files: M Doc/library/sys.rst diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 33391d11ab392..b6346bafbf625 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1811,35 +1811,39 @@ always available. The *unraisable* argument has the following attributes: - * *exc_type*: Exception type. - * *exc_value*: Exception value, can be ``None``. - * *exc_traceback*: Exception traceback, can be ``None``. - * *err_msg*: Error message, can be ``None``. - * *object*: Object causing the exception, can be ``None``. + * :attr:`!exc_type`: Exception type. + * :attr:`!exc_value`: Exception value, can be ``None``. + * :attr:`!exc_traceback`: Exception traceback, can be ``None``. + * :attr:`!err_msg`: Error message, can be ``None``. + * :attr:`!object`: Object causing the exception, can be ``None``. - The default hook formats *err_msg* and *object* as: + The default hook formats :attr:`!err_msg` and :attr:`!object` as: ``f'{err_msg}: {object!r}'``; use "Exception ignored in" error message - if *err_msg* is ``None``. + if :attr:`!err_msg` is ``None``. :func:`sys.unraisablehook` can be overridden to control how unraisable exceptions are handled. - Storing *exc_value* using a custom hook can create a reference cycle. It - should be cleared explicitly to break the reference cycle when the - exception is no longer needed. + .. seealso:: + + :func:`excepthook` which handles uncaught exceptions. + + .. warning:: - Storing *object* using a custom hook can resurrect it if it is set to an - object which is being finalized. Avoid storing *object* after the custom - hook completes to avoid resurrecting objects. + Storing :attr:`!exc_value` using a custom hook can create a reference cycle. + It should be cleared explicitly to break the reference cycle when the + exception is no longer needed. - See also :func:`excepthook` which handles uncaught exceptions. + Storing :attr:`!object` using a custom hook can resurrect it if it is set to an + object which is being finalized. Avoid storing :attr:`!object` after the custom + hook completes to avoid resurrecting objects. .. audit-event:: sys.unraisablehook hook,unraisable sys.unraisablehook Raise an auditing event ``sys.unraisablehook`` with arguments - ``hook``, ``unraisable`` when an exception that cannot be handled occurs. - The ``unraisable`` object is the same as what will be passed to the hook. - If no hook has been set, ``hook`` may be ``None``. + *hook*, *unraisable* when an exception that cannot be handled occurs. + The *unraisable* object is the same as what will be passed to the hook. + If no hook has been set, *hook* may be ``None``. .. versionadded:: 3.8 From webhook-mailer at python.org Fri Aug 18 09:42:49 2023 From: webhook-mailer at python.org (corona10) Date: Fri, 18 Aug 2023 13:42:49 -0000 Subject: [Python-checkins] gh-104504: Run mypy on cases_generator in CI (and blacken the code) (gh-108090) Message-ID: https://github.com/python/cpython/commit/28cab71f954f3a14de9f474ce9c4abbd23c97862 commit: 28cab71f954f3a14de9f474ce9c4abbd23c97862 branch: main author: Dong-hee Na committer: corona10 date: 2023-08-18T22:42:45+09:00 summary: gh-104504: Run mypy on cases_generator in CI (and blacken the code) (gh-108090) Co-authored-by: Alex Waygood files: A Tools/cases_generator/mypy.ini A Tools/requirements-dev.txt D Tools/clinic/requirements-dev.txt M .github/dependabot.yml M .github/workflows/mypy.yml M Tools/cases_generator/flags.py M Tools/cases_generator/formatting.py M Tools/cases_generator/generate_cases.py M Tools/cases_generator/lexer.py M Tools/cases_generator/parsing.py M Tools/cases_generator/stacking.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f026b0f5f9454..c8a3165d69036 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,7 +13,7 @@ updates: - "version-update:semver-minor" - "version-update:semver-patch" - package-ecosystem: "pip" - directory: "/Tools/clinic/" + directory: "/Tools/" schedule: interval: "monthly" labels: diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 1315bb5a966f0..a83a90c69529f 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -8,6 +8,7 @@ on: pull_request: paths: - "Tools/clinic/**" + - "Tools/cases_generator/**" - ".github/workflows/mypy.yml" workflow_dispatch: @@ -25,15 +26,18 @@ concurrency: jobs: mypy: - name: Run mypy on Tools/clinic/ + strategy: + matrix: + target: ["Tools/cases_generator", "Tools/clinic"] + name: Run mypy on ${{ matrix.target }} runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout at v3 - uses: actions/setup-python at v4 with: - python-version: "3.x" + python-version: "3.11" cache: pip - cache-dependency-path: Tools/clinic/requirements-dev.txt - - run: pip install -r Tools/clinic/requirements-dev.txt - - run: mypy --config-file Tools/clinic/mypy.ini + cache-dependency-path: Tools/requirements-dev.txt + - run: pip install -r Tools/requirements-dev.txt + - run: mypy --config-file ${{ matrix.target }}/mypy.ini diff --git a/Tools/cases_generator/flags.py b/Tools/cases_generator/flags.py index 962f003b194db..536f093c7d6ed 100644 --- a/Tools/cases_generator/flags.py +++ b/Tools/cases_generator/flags.py @@ -16,12 +16,11 @@ class InstructionFlags: HAS_FREE_FLAG: bool HAS_LOCAL_FLAG: bool - def __post_init__(self): + def __post_init__(self) -> None: self.bitmask = {name: (1 << i) for i, name in enumerate(self.names())} @staticmethod - def fromInstruction(instr: parsing.Node): - + def fromInstruction(instr: parsing.Node) -> "InstructionFlags": has_free = ( variable_used(instr, "PyCell_New") or variable_used(instr, "PyCell_GET") @@ -41,7 +40,7 @@ def fromInstruction(instr: parsing.Node): ) @staticmethod - def newEmpty(): + def newEmpty() -> "InstructionFlags": return InstructionFlags(False, False, False, False, False, False) def add(self, other: "InstructionFlags") -> None: @@ -49,7 +48,7 @@ def add(self, other: "InstructionFlags") -> None: if value: setattr(self, name, value) - def names(self, value=None) -> list[str]: + def names(self, value: bool | None = None) -> list[str]: if value is None: return list(dataclasses.asdict(self).keys()) return [n for n, v in dataclasses.asdict(self).items() if v == value] @@ -62,7 +61,7 @@ def bitmap(self) -> int: return flags @classmethod - def emit_macros(cls, out: Formatter): + def emit_macros(cls, out: Formatter) -> None: flags = cls.newEmpty() for name, value in flags.bitmask.items(): out.emit(f"#define {name} ({value})") @@ -90,9 +89,9 @@ def variable_used_unspecialized(node: parsing.Node, name: str) -> bool: text = "".join(token.text.split()) # TODO: Handle nested #if if text == "#if": - if ( - i + 1 < len(node.tokens) - and node.tokens[i + 1].text in ("ENABLE_SPECIALIZATION", "TIER_ONE") + if i + 1 < len(node.tokens) and node.tokens[i + 1].text in ( + "ENABLE_SPECIALIZATION", + "TIER_ONE", ): skipping = True elif text in ("#else", "#endif"): diff --git a/Tools/cases_generator/formatting.py b/Tools/cases_generator/formatting.py index 5894751bd9635..4fd9172d20c27 100644 --- a/Tools/cases_generator/formatting.py +++ b/Tools/cases_generator/formatting.py @@ -1,6 +1,7 @@ import contextlib import re import typing +from collections.abc import Iterator from parsing import StackEffect, Family @@ -58,13 +59,13 @@ def reset_lineno(self) -> None: self.set_lineno(self.lineno + 1, self.filename) @contextlib.contextmanager - def indent(self): + def indent(self) -> Iterator[None]: self.prefix += " " yield self.prefix = self.prefix[:-4] @contextlib.contextmanager - def block(self, head: str, tail: str = ""): + def block(self, head: str, tail: str = "") -> Iterator[None]: if head: self.emit(head + " {") else: @@ -77,7 +78,7 @@ def stack_adjust( self, input_effects: list[StackEffect], output_effects: list[StackEffect], - ): + ) -> None: shrink, isym = list_effect_size(input_effects) grow, osym = list_effect_size(output_effects) diff = grow - shrink @@ -90,7 +91,7 @@ def stack_adjust( if osym and osym != isym: self.emit(f"STACK_GROW({osym});") - def declare(self, dst: StackEffect, src: StackEffect | None): + def declare(self, dst: StackEffect, src: StackEffect | None) -> None: if dst.name == UNUSED or dst.cond == "0": return typ = f"{dst.type}" if dst.type else "PyObject *" @@ -107,7 +108,7 @@ def declare(self, dst: StackEffect, src: StackEffect | None): sepa = "" if typ.endswith("*") else " " self.emit(f"{typ}{sepa}{dst.name}{init};") - def assign(self, dst: StackEffect, src: StackEffect): + def assign(self, dst: StackEffect, src: StackEffect) -> None: if src.name == UNUSED or dst.name == UNUSED: return cast = self.cast(dst, src) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index f31b6658d6599..de31129ac0548 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -10,6 +10,7 @@ import posixpath import sys import typing +from collections.abc import Iterator import stacking # Early import to avoid circular import from analysis import Analyzer @@ -23,7 +24,6 @@ MacroInstruction, MacroParts, PseudoInstruction, - StackEffect, OverriddenInstructionPlaceHolder, TIER_ONE, TIER_TWO, @@ -80,12 +80,10 @@ "STORE_FAST", "STORE_FAST_MAYBE_NULL", "COPY", - # Arithmetic "_BINARY_OP_MULTIPLY_INT", "_BINARY_OP_ADD_INT", "_BINARY_OP_SUBTRACT_INT", - } arg_parser = argparse.ArgumentParser( @@ -144,6 +142,7 @@ default=DEFAULT_ABSTRACT_INTERPRETER_OUTPUT, ) + class Generator(Analyzer): def get_stack_effect_info( self, thing: parsing.InstDef | parsing.Macro | parsing.Pseudo @@ -183,7 +182,8 @@ def effect_str(effects: list[StackEffect]) -> str: assert target_instr target_popped = effect_str(target_instr.input_effects) target_pushed = effect_str(target_instr.output_effects) - if popped is None and pushed is None: + if pushed is None: + assert popped is None popped, pushed = target_popped, target_pushed else: assert popped == target_popped @@ -193,7 +193,7 @@ def effect_str(effects: list[StackEffect]) -> str: return instr, popped, pushed @contextlib.contextmanager - def metadata_item(self, signature, open, close): + def metadata_item(self, signature: str, open: str, close: str) -> Iterator[None]: self.out.emit("") self.out.emit(f"extern {signature};") self.out.emit("#ifdef NEED_OPCODE_METADATA") @@ -216,9 +216,10 @@ def write_stack_effect_functions(self) -> None: def write_function( direction: str, data: list[tuple[AnyInstruction, str]] ) -> None: - with self.metadata_item( - f"int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump)", "", "" + f"int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump)", + "", + "", ): with self.out.block("switch(opcode)"): for instr, effect in data: @@ -243,23 +244,24 @@ def from_source_files(self) -> str: paths = f"\n{self.out.comment} ".join(filenames) return f"{self.out.comment} from:\n{self.out.comment} {paths}\n" - def write_provenance_header(self): + def write_provenance_header(self) -> None: self.out.write_raw(f"{self.out.comment} This file is generated by {THIS}\n") self.out.write_raw(self.from_source_files()) self.out.write_raw(f"{self.out.comment} Do not edit!\n") - def assign_opcode_ids(self): + def assign_opcode_ids(self) -> None: """Assign IDs to opcodes""" - ops: list[(bool, str)] = [] # (has_arg, name) for each opcode + ops: list[tuple[bool, str]] = [] # (has_arg, name) for each opcode instrumented_ops: list[str] = [] for instr in itertools.chain( [instr for instr in self.instrs.values() if instr.kind != "op"], - self.macro_instrs.values()): - + self.macro_instrs.values(), + ): + assert isinstance(instr, (Instruction, MacroInstruction, PseudoInstruction)) name = instr.name - if name.startswith('INSTRUMENTED_'): + if name.startswith("INSTRUMENTED_"): instrumented_ops.append(name) else: ops.append((instr.instr_flags.HAS_ARG_FLAG, name)) @@ -268,33 +270,32 @@ def assign_opcode_ids(self): # rather than bytecodes.c, so we need to add it explicitly # here (at least until we add something to bytecodes.c to # declare external instructions). - instrumented_ops.append('INSTRUMENTED_LINE') + instrumented_ops.append("INSTRUMENTED_LINE") # assert lists are unique assert len(set(ops)) == len(ops) assert len(set(instrumented_ops)) == len(instrumented_ops) - opname: list[str or None] = [None] * 512 - opmap: dict = {} - markers: dict = {} + opname: list[str | None] = [None] * 512 + opmap: dict[str, int] = {} + markers: dict[str, int] = {} - def map_op(op, name): + def map_op(op: int, name: str) -> None: assert op < len(opname) assert opname[op] is None assert name not in opmap opname[op] = name opmap[name] = op - # 0 is reserved for cache entries. This helps debugging. - map_op(0, 'CACHE') + map_op(0, "CACHE") # 17 is reserved as it is the initial value for the specializing counter. # This helps catch cases where we attempt to execute a cache. - map_op(17, 'RESERVED') + map_op(17, "RESERVED") # 166 is RESUME - it is hard coded as such in Tools/build/deepfreeze.py - map_op(166, 'RESUME') + map_op(166, "RESUME") next_opcode = 1 @@ -306,13 +307,13 @@ def map_op(op, name): assert next_opcode < 255 map_op(next_opcode, name) - if has_arg and 'HAVE_ARGUMENT' not in markers: - markers['HAVE_ARGUMENT'] = next_opcode + if has_arg and "HAVE_ARGUMENT" not in markers: + markers["HAVE_ARGUMENT"] = next_opcode # Instrumented opcodes are at the end of the valid range min_instrumented = 254 - (len(instrumented_ops) - 1) assert next_opcode <= min_instrumented - markers['MIN_INSTRUMENTED_OPCODE'] = min_instrumented + markers["MIN_INSTRUMENTED_OPCODE"] = min_instrumented for i, op in enumerate(instrumented_ops): map_op(min_instrumented + i, op) @@ -320,11 +321,13 @@ def map_op(op, name): for i, op in enumerate(sorted(self.pseudos)): map_op(256 + i, op) - assert 255 not in opmap # 255 is reserved + assert 255 not in opmap.values() # 255 is reserved self.opmap = opmap self.markers = markers - def write_opcode_ids(self, opcode_ids_h_filename, opcode_targets_filename): + def write_opcode_ids( + self, opcode_ids_h_filename: str, opcode_targets_filename: str + ) -> None: """Write header file that defined the opcode IDs""" with open(opcode_ids_h_filename, "w") as f: @@ -337,15 +340,15 @@ def write_opcode_ids(self, opcode_ids_h_filename, opcode_targets_filename): self.out.emit("#ifndef Py_OPCODE_IDS_H") self.out.emit("#define Py_OPCODE_IDS_H") self.out.emit("#ifdef __cplusplus") - self.out.emit("extern \"C\" {") + self.out.emit('extern "C" {') self.out.emit("#endif") self.out.emit("") self.out.emit("/* Instruction opcodes for compiled code */") - def define(name, opcode): + def define(name: str, opcode: int) -> None: self.out.emit(f"#define {name:<38} {opcode:>3}") - all_pairs = [] + all_pairs: list[tuple[int, int, str]] = [] # the second item in the tuple sorts the markers before the ops all_pairs.extend((i, 1, name) for (name, i) in self.markers.items()) all_pairs.extend((i, 2, name) for (name, i) in self.opmap.items()) @@ -370,7 +373,6 @@ def define(name, opcode): targets[op] = f"TARGET_{name}" f.write(",\n".join([f" &&{s}" for s in targets])) - def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> None: """Write instruction metadata to output file.""" @@ -469,7 +471,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No "const struct opcode_metadata " "_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE]", "=", - ";" + ";", ): # Write metadata for each instruction for thing in self.everything: @@ -482,7 +484,9 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No case parsing.Macro(): self.write_metadata_for_macro(self.macro_instrs[thing.name]) case parsing.Pseudo(): - self.write_metadata_for_pseudo(self.pseudo_instrs[thing.name]) + self.write_metadata_for_pseudo( + self.pseudo_instrs[thing.name] + ) case _: typing.assert_never(thing) @@ -490,7 +494,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No "const struct opcode_macro_expansion " "_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]", "=", - ";" + ";", ): # Write macro expansion for each non-pseudo instruction for thing in self.everything: @@ -529,7 +533,9 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.write_uop_items(lambda name, counter: f'[{name}] = "{name}",') with self.metadata_item( - f"const char *const _PyOpcode_OpName[{1 + max(self.opmap.values())}]", "=", ";" + f"const char *const _PyOpcode_OpName[{1 + max(self.opmap.values())}]", + "=", + ";", ): for name in self.opmap: self.out.emit(f'[{name}] = "{name}",') @@ -542,11 +548,9 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No for m in family.members: deoptcodes[m] = name # special case: - deoptcodes['BINARY_OP_INPLACE_ADD_UNICODE'] = 'BINARY_OP' + deoptcodes["BINARY_OP_INPLACE_ADD_UNICODE"] = "BINARY_OP" - with self.metadata_item( - f"const uint8_t _PyOpcode_Deopt[256]", "=", ";" - ): + with self.metadata_item(f"const uint8_t _PyOpcode_Deopt[256]", "=", ";"): for opt, deopt in sorted(deoptcodes.items()): self.out.emit(f"[{opt}] = {deopt},") @@ -604,10 +608,9 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No if name not in specialized_ops: self.out.emit(f"'{name}': {op},") - for name in ['MIN_INSTRUMENTED_OPCODE', 'HAVE_ARGUMENT']: + for name in ["MIN_INSTRUMENTED_OPCODE", "HAVE_ARGUMENT"]: self.out.emit(f"{name} = {self.markers[name]}") - def write_pseudo_instrs(self) -> None: """Write the IS_PSEUDO_INSTR macro""" self.out.emit("\n\n#define IS_PSEUDO_INSTR(OP) ( \\") @@ -834,7 +837,10 @@ def write_abstract_interpreter_instructions( pass case parsing.InstDef(): instr = AbstractInstruction(self.instrs[thing.name].inst) - if instr.is_viable_uop() and instr.name not in SPECIALLY_HANDLED_ABSTRACT_INSTR: + if ( + instr.is_viable_uop() + and instr.name not in SPECIALLY_HANDLED_ABSTRACT_INSTR + ): self.out.emit("") with self.out.block(f"case {thing.name}:"): instr.write(self.out, tier=TIER_TWO) @@ -878,7 +884,7 @@ def write_instr(self, instr: Instruction) -> None: self.out.emit(f"DISPATCH();") -def main(): +def main() -> None: """Parse command line, parse input, analyze, write output.""" args = arg_parser.parse_args() # Prints message and sys.exit(2) on error if len(args.input) == 0: @@ -899,8 +905,9 @@ def main(): a.write_opcode_ids(args.opcode_ids_h, args.opcode_targets_h) a.write_metadata(args.metadata, args.pymetadata) a.write_executor_instructions(args.executor_cases, args.emit_line_directives) - a.write_abstract_interpreter_instructions(args.abstract_interpreter_cases, - args.emit_line_directives) + a.write_abstract_interpreter_instructions( + args.abstract_interpreter_cases, args.emit_line_directives + ) if __name__ == "__main__": diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index fe9c05ede5aa4..a60f6c11a4c46 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -4,132 +4,221 @@ import re from dataclasses import dataclass +from collections.abc import Iterator -def choice(*opts): + +def choice(*opts: str) -> str: return "|".join("(%s)" % opt for opt in opts) + # Regexes # Longer operators must go before shorter ones. -PLUSPLUS = r'\+\+' -MINUSMINUS = r'--' +PLUSPLUS = r"\+\+" +MINUSMINUS = r"--" # -> -ARROW = r'->' -ELLIPSIS = r'\.\.\.' +ARROW = r"->" +ELLIPSIS = r"\.\.\." # Assignment operators -TIMESEQUAL = r'\*=' -DIVEQUAL = r'/=' -MODEQUAL = r'%=' -PLUSEQUAL = r'\+=' -MINUSEQUAL = r'-=' -LSHIFTEQUAL = r'<<=' -RSHIFTEQUAL = r'>>=' -ANDEQUAL = r'&=' -OREQUAL = r'\|=' -XOREQUAL = r'\^=' +TIMESEQUAL = r"\*=" +DIVEQUAL = r"/=" +MODEQUAL = r"%=" +PLUSEQUAL = r"\+=" +MINUSEQUAL = r"-=" +LSHIFTEQUAL = r"<<=" +RSHIFTEQUAL = r">>=" +ANDEQUAL = r"&=" +OREQUAL = r"\|=" +XOREQUAL = r"\^=" # Operators -PLUS = r'\+' -MINUS = r'-' -TIMES = r'\*' -DIVIDE = r'/' -MOD = r'%' -NOT = r'~' -XOR = r'\^' -LOR = r'\|\|' -LAND = r'&&' -LSHIFT = r'<<' -RSHIFT = r'>>' -LE = r'<=' -GE = r'>=' -EQ = r'==' -NE = r'!=' -LT = r'<' -GT = r'>' -LNOT = r'!' -OR = r'\|' -AND = r'&' -EQUALS = r'=' +PLUS = r"\+" +MINUS = r"-" +TIMES = r"\*" +DIVIDE = r"/" +MOD = r"%" +NOT = r"~" +XOR = r"\^" +LOR = r"\|\|" +LAND = r"&&" +LSHIFT = r"<<" +RSHIFT = r">>" +LE = r"<=" +GE = r">=" +EQ = r"==" +NE = r"!=" +LT = r"<" +GT = r">" +LNOT = r"!" +OR = r"\|" +AND = r"&" +EQUALS = r"=" # ? -CONDOP = r'\?' +CONDOP = r"\?" # Delimiters -LPAREN = r'\(' -RPAREN = r'\)' -LBRACKET = r'\[' -RBRACKET = r'\]' -LBRACE = r'\{' -RBRACE = r'\}' -COMMA = r',' -PERIOD = r'\.' -SEMI = r';' -COLON = r':' -BACKSLASH = r'\\' - -operators = { op: pattern for op, pattern in globals().items() if op == op.upper() } +LPAREN = r"\(" +RPAREN = r"\)" +LBRACKET = r"\[" +RBRACKET = r"\]" +LBRACE = r"\{" +RBRACE = r"\}" +COMMA = r"," +PERIOD = r"\." +SEMI = r";" +COLON = r":" +BACKSLASH = r"\\" + +operators = {op: pattern for op, pattern in globals().items() if op == op.upper()} for op in operators: globals()[op] = op -opmap = { pattern.replace("\\", "") or '\\' : op for op, pattern in operators.items() } +opmap = {pattern.replace("\\", "") or "\\": op for op, pattern in operators.items()} # Macros -macro = r'# *(ifdef|ifndef|undef|define|error|endif|if|else|include|#)' -MACRO = 'MACRO' +macro = r"# *(ifdef|ifndef|undef|define|error|endif|if|else|include|#)" +MACRO = "MACRO" -id_re = r'[a-zA-Z_][0-9a-zA-Z_]*' -IDENTIFIER = 'IDENTIFIER' +id_re = r"[a-zA-Z_][0-9a-zA-Z_]*" +IDENTIFIER = "IDENTIFIER" -suffix = r'([uU]?[lL]?[lL]?)' -octal = r'0[0-7]+' + suffix -hex = r'0[xX][0-9a-fA-F]+' -decimal_digits = r'(0|[1-9][0-9]*)' +suffix = r"([uU]?[lL]?[lL]?)" +octal = r"0[0-7]+" + suffix +hex = r"0[xX][0-9a-fA-F]+" +decimal_digits = r"(0|[1-9][0-9]*)" decimal = decimal_digits + suffix exponent = r"""([eE][-+]?[0-9]+)""" fraction = r"""([0-9]*\.[0-9]+)|([0-9]+\.)""" -float = '(((('+fraction+')'+exponent+'?)|([0-9]+'+exponent+'))[FfLl]?)' +float = "((((" + fraction + ")" + exponent + "?)|([0-9]+" + exponent + "))[FfLl]?)" number_re = choice(octal, hex, float, decimal) -NUMBER = 'NUMBER' +NUMBER = "NUMBER" simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])""" decimal_escape = r"""(\d+)""" hex_escape = r"""(x[0-9a-fA-F]+)""" -escape_sequence = r"""(\\("""+simple_escape+'|'+decimal_escape+'|'+hex_escape+'))' -string_char = r"""([^"\\\n]|"""+escape_sequence+')' -str_re = '"'+string_char+'*"' -STRING = 'STRING' -char = r'\'.\'' # TODO: escape sequence -CHARACTER = 'CHARACTER' +escape_sequence = ( + r"""(\\(""" + simple_escape + "|" + decimal_escape + "|" + hex_escape + "))" +) +string_char = r"""([^"\\\n]|""" + escape_sequence + ")" +str_re = '"' + string_char + '*"' +STRING = "STRING" +char = r"\'.\'" # TODO: escape sequence +CHARACTER = "CHARACTER" -comment_re = r'//.*|/\*([^*]|\*[^/])*\*/' -COMMENT = 'COMMENT' +comment_re = r"//.*|/\*([^*]|\*[^/])*\*/" +COMMENT = "COMMENT" newline = r"\n" -invalid = r"\S" # A single non-space character that's not caught by any of the other patterns -matcher = re.compile(choice(id_re, number_re, str_re, char, newline, macro, comment_re, *operators.values(), invalid)) -letter = re.compile(r'[a-zA-Z_]') - -kwds = ( - 'AUTO', 'BREAK', 'CASE', 'CHAR', 'CONST', - 'CONTINUE', 'DEFAULT', 'DO', 'DOUBLE', 'ELSE', 'ENUM', 'EXTERN', - 'FLOAT', 'FOR', 'GOTO', 'IF', 'INLINE', 'INT', 'LONG', 'OVERRIDE', - 'REGISTER', 'OFFSETOF', - 'RESTRICT', 'RETURN', 'SHORT', 'SIGNED', 'SIZEOF', 'STATIC', 'STRUCT', - 'SWITCH', 'TYPEDEF', 'UNION', 'UNSIGNED', 'VOID', - 'VOLATILE', 'WHILE' +invalid = ( + r"\S" # A single non-space character that's not caught by any of the other patterns ) -for name in kwds: - globals()[name] = name -keywords = { name.lower() : name for name in kwds } +matcher = re.compile( + choice( + id_re, + number_re, + str_re, + char, + newline, + macro, + comment_re, + *operators.values(), + invalid, + ) +) +letter = re.compile(r"[a-zA-Z_]") + + +kwds = [] +AUTO = "AUTO" +kwds.append(AUTO) +BREAK = "BREAK" +kwds.append(BREAK) +CASE = "CASE" +kwds.append(CASE) +CHAR = "CHAR" +kwds.append(CHAR) +CONST = "CONST" +kwds.append(CONST) +CONTINUE = "CONTINUE" +kwds.append(CONTINUE) +DEFAULT = "DEFAULT" +kwds.append(DEFAULT) +DO = "DO" +kwds.append(DO) +DOUBLE = "DOUBLE" +kwds.append(DOUBLE) +ELSE = "ELSE" +kwds.append(ELSE) +ENUM = "ENUM" +kwds.append(ENUM) +EXTERN = "EXTERN" +kwds.append(EXTERN) +FLOAT = "FLOAT" +kwds.append(FLOAT) +FOR = "FOR" +kwds.append(FOR) +GOTO = "GOTO" +kwds.append(GOTO) +IF = "IF" +kwds.append(IF) +INLINE = "INLINE" +kwds.append(INLINE) +INT = "INT" +kwds.append(INT) +LONG = "LONG" +kwds.append(LONG) +OVERRIDE = "OVERRIDE" +kwds.append(OVERRIDE) +REGISTER = "REGISTER" +kwds.append(REGISTER) +OFFSETOF = "OFFSETOF" +kwds.append(OFFSETOF) +RESTRICT = "RESTRICT" +kwds.append(RESTRICT) +RETURN = "RETURN" +kwds.append(RETURN) +SHORT = "SHORT" +kwds.append(SHORT) +SIGNED = "SIGNED" +kwds.append(SIGNED) +SIZEOF = "SIZEOF" +kwds.append(SIZEOF) +STATIC = "STATIC" +kwds.append(STATIC) +STRUCT = "STRUCT" +kwds.append(STRUCT) +SWITCH = "SWITCH" +kwds.append(SWITCH) +TYPEDEF = "TYPEDEF" +kwds.append(TYPEDEF) +UNION = "UNION" +kwds.append(UNION) +UNSIGNED = "UNSIGNED" +kwds.append(UNSIGNED) +VOID = "VOID" +kwds.append(VOID) +VOLATILE = "VOLATILE" +kwds.append(VOLATILE) +WHILE = "WHILE" +kwds.append(WHILE) +keywords = {name.lower(): name for name in kwds} + +__all__ = [] +__all__.extend(kwds) def make_syntax_error( - message: str, filename: str, line: int, column: int, line_text: str, + message: str, + filename: str | None, + line: int, + column: int, + line_text: str, ) -> SyntaxError: return SyntaxError(message, (filename, line, column, line_text)) @@ -142,30 +231,30 @@ class Token: end: tuple[int, int] @property - def line(self): + def line(self) -> int: return self.begin[0] @property - def column(self): + def column(self) -> int: return self.begin[1] @property - def end_line(self): + def end_line(self) -> int: return self.end[0] @property - def end_column(self): + def end_column(self) -> int: return self.end[1] @property - def width(self): + def width(self) -> int: return self.end[1] - self.begin[1] - def replaceText(self, txt): + def replaceText(self, txt: str) -> "Token": assert isinstance(txt, str) return Token(self.kind, txt, self.begin, self.end) - def __repr__(self): + def __repr__(self) -> str: b0, b1 = self.begin e0, e1 = self.end if b0 == e0: @@ -174,7 +263,7 @@ def __repr__(self): return f"{self.kind}({self.text!r}, {b0}:{b1}, {e0}:{e1})" -def tokenize(src, line=1, filename=None): +def tokenize(src: str, line: int = 1, filename: str | None = None) -> Iterator[Token]: linestart = -1 for m in matcher.finditer(src): start, end = m.span() @@ -183,73 +272,75 @@ def tokenize(src, line=1, filename=None): kind = keywords[text] elif letter.match(text): kind = IDENTIFIER - elif text == '...': + elif text == "...": kind = ELLIPSIS - elif text == '.': + elif text == ".": kind = PERIOD - elif text[0] in '0123456789.': + elif text[0] in "0123456789.": kind = NUMBER elif text[0] == '"': kind = STRING elif text in opmap: kind = opmap[text] - elif text == '\n': + elif text == "\n": linestart = start line += 1 - kind = '\n' + kind = "\n" elif text[0] == "'": kind = CHARACTER - elif text[0] == '#': + elif text[0] == "#": kind = MACRO - elif text[0] == '/' and text[1] in '/*': + elif text[0] == "/" and text[1] in "/*": kind = COMMENT else: lineend = src.find("\n", start) if lineend == -1: lineend = len(src) - raise make_syntax_error(f"Bad token: {text}", - filename, line, start-linestart+1, src[linestart:lineend]) + raise make_syntax_error( + f"Bad token: {text}", + filename, + line, + start - linestart + 1, + src[linestart:lineend], + ) if kind == COMMENT: - begin = line, start-linestart - newlines = text.count('\n') + begin = line, start - linestart + newlines = text.count("\n") if newlines: - linestart = start + text.rfind('\n') + linestart = start + text.rfind("\n") line += newlines else: - begin = line, start-linestart + begin = line, start - linestart if kind != "\n": - yield Token(kind, text, begin, (line, start-linestart+len(text))) - - -__all__ = [] -__all__.extend([kind for kind in globals() if kind.upper() == kind]) + yield Token(kind, text, begin, (line, start - linestart + len(text))) def to_text(tkns: list[Token], dedent: int = 0) -> str: res: list[str] = [] - line, col = -1, 1+dedent + line, col = -1, 1 + dedent for tkn in tkns: if line == -1: line, _ = tkn.begin l, c = tkn.begin - #assert(l >= line), (line, txt, start, end) + # assert(l >= line), (line, txt, start, end) while l > line: line += 1 - res.append('\n') - col = 1+dedent - res.append(' '*(c-col)) + res.append("\n") + col = 1 + dedent + res.append(" " * (c - col)) text = tkn.text - if dedent != 0 and tkn.kind == 'COMMENT' and '\n' in text: + if dedent != 0 and tkn.kind == "COMMENT" and "\n" in text: if dedent < 0: - text = text.replace('\n', '\n' + ' '*-dedent) + text = text.replace("\n", "\n" + " " * -dedent) # TODO: dedent > 0 res.append(text) line, col = tkn.end - return ''.join(res) + return "".join(res) if __name__ == "__main__": import sys + filename = sys.argv[1] if filename == "-c": src = sys.argv[2] diff --git a/Tools/cases_generator/mypy.ini b/Tools/cases_generator/mypy.ini new file mode 100644 index 0000000000000..7480841bf07ed --- /dev/null +++ b/Tools/cases_generator/mypy.ini @@ -0,0 +1,14 @@ +[mypy] +files = Tools/cases_generator/ +pretty = True + +python_version = 3.10 + +# Be strict: +strict = True +strict_concatenate = True +enable_error_code = ignore-without-code,redundant-expr,truthy-bool + +# Don't enable this one yet; +# it has a lot of false positives on `cases_generator` +warn_unreachable = False diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index cdd20d7a0b3f5..25de3a5adc2cf 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -32,7 +32,7 @@ class Context(NamedTuple): end: int owner: PLexer - def __repr__(self): + def __repr__(self) -> str: return f"<{self.owner.filename}: {self.begin}-{self.end}>" @@ -75,7 +75,7 @@ class StackEffect(Node): size: str = "" # Optional `[size]` # Note: size cannot be combined with type or cond - def __repr__(self): + def __repr__(self) -> str: items = [self.name, self.type, self.cond, self.size] while items and items[-1] == "": del items[-1] diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 632298a567dd4..1e117f1182593 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -413,22 +413,22 @@ def write_components( return next_instr_is_set -def write_single_instr_for_abstract_interp( - instr: Instruction, out: Formatter -): +def write_single_instr_for_abstract_interp(instr: Instruction, out: Formatter) -> None: try: _write_components_for_abstract_interp( [Component(instr, instr.active_caches)], out, ) except AssertionError as err: - raise AssertionError(f"Error writing abstract instruction {instr.name}") from err + raise AssertionError( + f"Error writing abstract instruction {instr.name}" + ) from err def _write_components_for_abstract_interp( parts: list[Component], out: Formatter, -): +) -> None: managers = get_managers(parts) for mgr in managers: if mgr is managers[-1]: @@ -438,5 +438,7 @@ def _write_components_for_abstract_interp( # NULL out the output stack effects for poke in mgr.pokes: if not poke.effect.size and poke.effect.name not in mgr.instr.unmoved_names: - out.emit(f"PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)" - f"PARTITIONNODE_NULLROOT, PEEK(-({poke.offset.as_index()})), true);") + out.emit( + f"PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)" + f"PARTITIONNODE_NULLROOT, PEEK(-({poke.offset.as_index()})), true);" + ) diff --git a/Tools/clinic/requirements-dev.txt b/Tools/clinic/requirements-dev.txt deleted file mode 100644 index e9529f3527e95..0000000000000 --- a/Tools/clinic/requirements-dev.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Requirements file for external linters and checks we run on Tools/clinic/ in CI -mypy==1.4.1 diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt new file mode 100644 index 0000000000000..111773f4f4737 --- /dev/null +++ b/Tools/requirements-dev.txt @@ -0,0 +1,3 @@ +# Requirements file for external linters and checks we run on +# Tools/clinic and Tools/cases_generator/ in CI +mypy==1.5.1 From webhook-mailer at python.org Fri Aug 18 11:44:42 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Fri, 18 Aug 2023 15:44:42 -0000 Subject: [Python-checkins] gh-107995: Fix doctest collection of functools.cached_property objects (#107996) Message-ID: https://github.com/python/cpython/commit/9bb576cb07b42f34fd882b8502642b024f771c62 commit: 9bb576cb07b42f34fd882b8502642b024f771c62 branch: main author: Tyler Smart <40041862+tjsmart at users.noreply.github.com> committer: AlexWaygood date: 2023-08-18T15:44:38Z summary: gh-107995: Fix doctest collection of functools.cached_property objects (#107996) Co-authored-by: Alex Waygood files: A Misc/NEWS.d/next/Library/2023-08-16-00-24-07.gh-issue-107995.TlTp5t.rst M Lib/functools.py M Lib/test/test_doctest.py M Lib/test/test_functools.py M Misc/ACKS diff --git a/Lib/functools.py b/Lib/functools.py index be44ccdae6b69..a2fc28779dbdd 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -984,6 +984,7 @@ def __init__(self, func): self.func = func self.attrname = None self.__doc__ = func.__doc__ + self.__module__ = func.__module__ def __set_name__(self, owner, name): if self.attrname is None: diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index bea52c6de7ec6..c364e81a6b149 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -111,6 +111,14 @@ def a_classmethod_property(cls): """ return cls.a_class_attribute + @functools.cached_property + def a_cached_property(self): + """ + >>> print(SampleClass(29).get()) + 29 + """ + return "hello" + class NestedClass: """ >>> x = SampleClass.NestedClass(5) @@ -515,6 +523,7 @@ def basics(): r""" 3 SampleClass.NestedClass 1 SampleClass.NestedClass.__init__ 1 SampleClass.__init__ + 1 SampleClass.a_cached_property 2 SampleClass.a_classmethod 1 SampleClass.a_classmethod_property 1 SampleClass.a_property @@ -571,6 +580,7 @@ def basics(): r""" 3 some_module.SampleClass.NestedClass 1 some_module.SampleClass.NestedClass.__init__ 1 some_module.SampleClass.__init__ + 1 some_module.SampleClass.a_cached_property 2 some_module.SampleClass.a_classmethod 1 some_module.SampleClass.a_classmethod_property 1 some_module.SampleClass.a_property @@ -613,6 +623,7 @@ def basics(): r""" 3 SampleClass.NestedClass 1 SampleClass.NestedClass.__init__ 1 SampleClass.__init__ + 1 SampleClass.a_cached_property 2 SampleClass.a_classmethod 1 SampleClass.a_classmethod_property 1 SampleClass.a_property @@ -634,6 +645,7 @@ def basics(): r""" 0 SampleClass.NestedClass.get 0 SampleClass.NestedClass.square 1 SampleClass.__init__ + 1 SampleClass.a_cached_property 2 SampleClass.a_classmethod 1 SampleClass.a_classmethod_property 1 SampleClass.a_property diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 50770f066a5e1..5ba7f51c91f3b 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -3105,6 +3105,9 @@ def test_access_from_class(self): def test_doc(self): self.assertEqual(CachedCostItem.cost.__doc__, "The cost of the item.") + def test_module(self): + self.assertEqual(CachedCostItem.cost.__module__, CachedCostItem.__module__) + def test_subclass_with___set__(self): """Caching still works for a subclass defining __set__.""" class readonly_cached_property(py_functools.cached_property): diff --git a/Misc/ACKS b/Misc/ACKS index 8b8c5ad8434bd..475c6ec3dab73 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1706,6 +1706,7 @@ Roman Skurikhin Ville Skytt? Michael Sloan Nick Sloan +Tyler Smart Radek Smejkal V?clav ?milauer Casper W. Smet diff --git a/Misc/NEWS.d/next/Library/2023-08-16-00-24-07.gh-issue-107995.TlTp5t.rst b/Misc/NEWS.d/next/Library/2023-08-16-00-24-07.gh-issue-107995.TlTp5t.rst new file mode 100644 index 0000000000000..7247f6b3abc9b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-16-00-24-07.gh-issue-107995.TlTp5t.rst @@ -0,0 +1,5 @@ +The ``__module__`` attribute on instances of :class:`functools.cached_property` +is now set to the name of the module in which the cached_property is defined, +rather than "functools". This means that doctests in ``cached_property`` +docstrings are now properly collected by the :mod:`doctest` module. Patch by +Tyler Smart. From webhook-mailer at python.org Fri Aug 18 13:14:01 2023 From: webhook-mailer at python.org (rhettinger) Date: Fri, 18 Aug 2023 17:14:01 -0000 Subject: [Python-checkins] Minor code clean-up for the factor() recipe (GH-108114) Message-ID: https://github.com/python/cpython/commit/6db39b1460727e8820b45e5c083e5839af0352aa commit: 6db39b1460727e8820b45e5c083e5839af0352aa branch: main author: Raymond Hettinger committer: rhettinger date: 2023-08-18T12:13:58-05:00 summary: Minor code clean-up for the factor() recipe (GH-108114) files: M Doc/library/itertools.rst diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 730736bbb59ed..d0bb469697019 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -1048,9 +1048,7 @@ The following recipes have a more mathematical flavor: # factor(1_000_000_000_000_007) --> 47 59 360620266859 # factor(1_000_000_000_000_403) --> 1000000000000403 for prime in sieve(math.isqrt(n) + 1): - while True: - if n % prime: - break + while not n % prime: yield prime n //= prime if n == 1: From webhook-mailer at python.org Fri Aug 18 13:53:55 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Fri, 18 Aug 2023 17:53:55 -0000 Subject: [Python-checkins] gh-107801: Improve the accuracy of os.lseek docs (#107935) Message-ID: https://github.com/python/cpython/commit/dd4442c8f597af1ec3eaf20f7ad89c4ac7e2dbc9 commit: dd4442c8f597af1ec3eaf20f7ad89c4ac7e2dbc9 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-18T19:53:51+02:00 summary: gh-107801: Improve the accuracy of os.lseek docs (#107935) - name the last parameter *whence*, like it is for seek() methods on file objects - add param docstrings - structure the valid *whence* params Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/os.rst M Modules/clinic/posixmodule.c.h M Modules/posixmodule.c diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 4668984b10c69..7d63ac15027d5 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1163,17 +1163,22 @@ as internal buffering of data. .. versionadded:: 3.11 -.. function:: lseek(fd, pos, how, /) +.. function:: lseek(fd, pos, whence, /) Set the current position of file descriptor *fd* to position *pos*, modified - by *how*: :const:`SEEK_SET` or ``0`` to set the position relative to the - beginning of the file; :const:`SEEK_CUR` or ``1`` to set it relative to the - current position; :const:`SEEK_END` or ``2`` to set it relative to the end of - the file. Return the new cursor position in bytes, starting from the beginning. + by *whence*, and return the new position in bytes relative to + the start of the file. + Valid values for *whence* are: + + * :const:`SEEK_SET` or ``0`` -- set *pos* relative to the beginning of the file + * :const:`SEEK_CUR` or ``1`` -- set *pos* relative to the current file position + * :const:`SEEK_END` or ``2`` -- set *pos* relative to the end of the file + * :const:`SEEK_HOLE` -- set *pos* to the next data location, relative to *pos* + * :const:`SEEK_DATA` -- set *pos* to the next data hole, relative to *pos* .. versionchanged:: 3.3 - Add support for :const:`SEEK_HOLE` and :const:`SEEK_DATA`. + Add support for :const:`!SEEK_HOLE` and :const:`!SEEK_DATA`. .. data:: SEEK_SET diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 81e3162e679d1..9b3fa8ded2676 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -6496,13 +6496,22 @@ os_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_LOCKF) */ PyDoc_STRVAR(os_lseek__doc__, -"lseek($module, fd, position, how, /)\n" +"lseek($module, fd, position, whence, /)\n" "--\n" "\n" "Set the position of a file descriptor. Return the new position.\n" "\n" -"Return the new cursor position in number of bytes\n" -"relative to the beginning of the file."); +" fd\n" +" An open file descriptor, as returned by os.open().\n" +" position\n" +" Position, interpreted relative to \'whence\'.\n" +" whence\n" +" The relative position to seek from. Valid values are:\n" +" - SEEK_SET: seek from the start of the file.\n" +" - SEEK_CUR: seek from the current file position.\n" +" - SEEK_END: seek from the end of the file.\n" +"\n" +"The return value is the number of bytes relative to the beginning of the file."); #define OS_LSEEK_METHODDEF \ {"lseek", _PyCFunction_CAST(os_lseek), METH_FASTCALL, os_lseek__doc__}, @@ -11981,4 +11990,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=a85a386b212b0631 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9a5f78bb65470528 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c391aab6fdce1..8026080912a7d 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -10424,19 +10424,24 @@ os_lockf_impl(PyObject *module, int fd, int command, Py_off_t length) os.lseek -> Py_off_t fd: int + An open file descriptor, as returned by os.open(). position: Py_off_t - how: int + Position, interpreted relative to 'whence'. + whence as how: int + The relative position to seek from. Valid values are: + - SEEK_SET: seek from the start of the file. + - SEEK_CUR: seek from the current file position. + - SEEK_END: seek from the end of the file. / Set the position of a file descriptor. Return the new position. -Return the new cursor position in number of bytes -relative to the beginning of the file. +The return value is the number of bytes relative to the beginning of the file. [clinic start generated code]*/ static Py_off_t os_lseek_impl(PyObject *module, int fd, Py_off_t position, int how) -/*[clinic end generated code: output=971e1efb6b30bd2f input=902654ad3f96a6d3]*/ +/*[clinic end generated code: output=971e1efb6b30bd2f input=f096e754c5367504]*/ { Py_off_t result; From webhook-mailer at python.org Fri Aug 18 15:48:24 2023 From: webhook-mailer at python.org (ned-deily) Date: Fri, 18 Aug 2023 19:48:24 -0000 Subject: [Python-checkins] gh-107565: Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, and 3.1.2. (GH-107896) Message-ID: https://github.com/python/cpython/commit/ed25f097160b5cbb0c9a1f9a746d2f1bbc96515a commit: ed25f097160b5cbb0c9a1f9a746d2f1bbc96515a branch: main author: Ned Deily committer: ned-deily date: 2023-08-18T15:48:20-04:00 summary: gh-107565: Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, and 3.1.2. (GH-107896) files: A Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst M .github/workflows/build.yml M Tools/ssl/multissltests.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 06551b13219c2..d37eb4447ad11 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -244,7 +244,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 1.1.1v PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout at v3 @@ -313,7 +313,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1u, 3.0.9, 3.1.1] + openssl_ver: [1.1.1v, 3.0.10, 3.1.2] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -365,7 +365,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 1.1.1v PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout at v3 @@ -474,7 +474,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 1.1.1v PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst new file mode 100644 index 0000000000000..c43ee680e8158 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst @@ -0,0 +1,2 @@ +Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, +and 3.1.2. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index f9809c9b54665..fc261c770d694 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -46,9 +46,9 @@ ] OPENSSL_RECENT_VERSIONS = [ - "1.1.1u", - "3.0.9", - "3.1.1", + "1.1.1v", + "3.0.10", + "3.1.2", ] LIBRESSL_OLD_VERSIONS = [ From webhook-mailer at python.org Fri Aug 18 16:30:38 2023 From: webhook-mailer at python.org (ned-deily) Date: Fri, 18 Aug 2023 20:30:38 -0000 Subject: [Python-checkins] [3.12] gh-107565: Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, and 3.1.2. (GH-108118) Message-ID: https://github.com/python/cpython/commit/359cff5c41f87fed3506e6a9a41b6828f4f5cd40 commit: 359cff5c41f87fed3506e6a9a41b6828f4f5cd40 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ned-deily date: 2023-08-18T20:30:34Z summary: [3.12] gh-107565: Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, and 3.1.2. (GH-108118) Co-authored-by: Ned Deily files: A Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst M .github/workflows/build.yml M Tools/ssl/multissltests.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 20208dbdd2f5f..ea6d43947449f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -287,7 +287,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 1.1.1v PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout at v3 @@ -356,7 +356,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1u, 3.0.9, 3.1.1] + openssl_ver: [1.1.1v, 3.0.10, 3.1.2] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -408,7 +408,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 1.1.1v PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout at v3 @@ -517,7 +517,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 1.1.1v PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst new file mode 100644 index 0000000000000..c43ee680e8158 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst @@ -0,0 +1,2 @@ +Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, +and 3.1.2. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index f9809c9b54665..fc261c770d694 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -46,9 +46,9 @@ ] OPENSSL_RECENT_VERSIONS = [ - "1.1.1u", - "3.0.9", - "3.1.1", + "1.1.1v", + "3.0.10", + "3.1.2", ] LIBRESSL_OLD_VERSIONS = [ From webhook-mailer at python.org Fri Aug 18 16:50:15 2023 From: webhook-mailer at python.org (ned-deily) Date: Fri, 18 Aug 2023 20:50:15 -0000 Subject: [Python-checkins] [3.11] gh-107565: Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, and 3.1.2. (GH-108119) Message-ID: https://github.com/python/cpython/commit/441797d4ffb12acda257370b9e5e19ed8d6e8a71 commit: 441797d4ffb12acda257370b9e5e19ed8d6e8a71 branch: 3.11 author: Ned Deily committer: ned-deily date: 2023-08-18T20:50:11Z summary: [3.11] gh-107565: Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, and 3.1.2. (GH-108119) files: A Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst M .github/workflows/build.yml M Tools/ssl/multissltests.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7d3488ef8d9e..8e7d831f4232d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -273,7 +273,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 1.1.1v PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout at v3 @@ -342,7 +342,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1u, 3.0.9, 3.1.1] + openssl_ver: [1.1.1v, 3.0.10, 3.1.2] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -394,7 +394,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 1.1.1v PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst new file mode 100644 index 0000000000000..c43ee680e8158 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst @@ -0,0 +1,2 @@ +Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, +and 3.1.2. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index a9c6e89a1571d..96500b024c00b 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -47,9 +47,9 @@ ] OPENSSL_RECENT_VERSIONS = [ - "1.1.1u", - "3.0.9", - "3.1.1", + "1.1.1v", + "3.0.10", + "3.1.2", ] LIBRESSL_OLD_VERSIONS = [ From webhook-mailer at python.org Fri Aug 18 17:21:20 2023 From: webhook-mailer at python.org (Yhg1s) Date: Fri, 18 Aug 2023 21:21:20 -0000 Subject: [Python-checkins] [3.12] gh-101100: Docs: Check Sphinx warnings and fail if improved (GH-106460) (#108116) Message-ID: https://github.com/python/cpython/commit/daed54d8de1f45a5ca07ea8dd37bee5882769bd7 commit: daed54d8de1f45a5ca07ea8dd37bee5882769bd7 branch: 3.12 author: C.A.M. Gerlach committer: Yhg1s date: 2023-08-18T23:21:16+02:00 summary: [3.12] gh-101100: Docs: Check Sphinx warnings and fail if improved (GH-106460) (#108116) * gh-101100: Docs: Check Sphinx warnings and fail if improved (#106460) (cherry picked from commit 806d7c98a5da5c1fd2e52a5b666f36ca4f545092) * [3.12] gh-101100: Docs: Check Sphinx warnings and fail if improved (GH-106460). (cherry picked from commit 806d7c98a5da5c1fd2e52a5b666f36ca4f545092) Co-authored-by: Hugo van Kemenade --------- Co-authored-by: Hugo van Kemenade files: A Doc/tools/check-warnings.py D Doc/tools/touch-clean-files.py D Doc/tools/warnings-to-gh-actions.py M .github/workflows/reusable-docs.yml M Doc/tools/.nitignore diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 77142a64a4b22..39e5ad62924ad 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -26,10 +26,8 @@ jobs: cache-dependency-path: 'Doc/requirements.txt' - name: 'Install build dependencies' run: make -C Doc/ venv - - name: 'Build HTML documentation' - run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html - # Add pull request annotations for Sphinx nitpicks (missing references) + # To annotate PRs with Sphinx nitpicks (missing references) - name: 'Get list of changed files' if: github.event_name == 'pull_request' id: changed_files @@ -37,24 +35,19 @@ jobs: with: filter: "Doc/**" format: csv # works for paths with spaces - - name: 'Build changed files in nit-picky mode' - if: github.event_name == 'pull_request' + - name: 'Build HTML documentation' continue-on-error: true run: | set -Eeuo pipefail - # Mark files the pull request modified - python Doc/tools/touch-clean-files.py --clean '${{ steps.changed_files.outputs.added_modified }}' - # Build docs with the '-n' (nit-picky) option; convert warnings to annotations - make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n --keep-going" html 2>&1 | - python Doc/tools/warnings-to-gh-actions.py - - # Ensure some files always pass Sphinx nit-picky mode (no missing references) - - name: 'Build known-good files in nit-picky mode' + # Build docs with the '-n' (nit-picky) option; write warnings to file + make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n -W --keep-going -w sphinx-warnings.txt" html + - name: 'Check warnings' + if: github.event_name == 'pull_request' run: | - # Mark files that must pass nit-picky - python Doc/tools/touch-clean-files.py - # Build docs with the '-n' (nit-picky) option, convert warnings to errors (-W) - make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n -W --keep-going" html 2>&1 + python Doc/tools/check-warnings.py \ + --check-and-annotate '${{ steps.changed_files.outputs.added_modified }}' \ + --fail-if-regression \ + --fail-if-improved # This build doesn't use problem matchers or check annotations build_doc_oldest_supported_sphinx: diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index f7f2d6d8f3289..88ac905467299 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -1,17 +1,14 @@ # All RST files under Doc/ -- except these -- must pass Sphinx nit-picky mode, -# as tested on the CI via touch-clean-files.py in doc.yml. -# Add blank lines between files and keep them sorted lexicographically -# to help avoid merge conflicts. +# as tested on the CI via check-warnings.py in reusable-docs.yml. +# Keep lines sorted lexicographically to help avoid merge conflicts. Doc/c-api/arg.rst Doc/c-api/datetime.rst Doc/c-api/descriptor.rst -Doc/c-api/dict.rst Doc/c-api/exceptions.rst Doc/c-api/file.rst Doc/c-api/float.rst Doc/c-api/gcsupport.rst -Doc/c-api/import.rst Doc/c-api/init.rst Doc/c-api/init_config.rst Doc/c-api/intro.rst @@ -58,7 +55,6 @@ Doc/library/bz2.rst Doc/library/calendar.rst Doc/library/cgi.rst Doc/library/chunk.rst -Doc/library/cmath.rst Doc/library/cmd.rst Doc/library/codecs.rst Doc/library/collections.abc.rst diff --git a/Doc/tools/check-warnings.py b/Doc/tools/check-warnings.py new file mode 100644 index 0000000000000..c17d0f51cd127 --- /dev/null +++ b/Doc/tools/check-warnings.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +Check the output of running Sphinx in nit-picky mode (missing references). +""" +import argparse +import csv +import os +import re +import sys +from pathlib import Path + +# Exclude these whether they're dirty or clean, +# because they trigger a rebuild of dirty files. +EXCLUDE_FILES = { + "Doc/whatsnew/changelog.rst", +} + +# Subdirectories of Doc/ to exclude. +EXCLUDE_SUBDIRS = { + ".env", + ".venv", + "env", + "includes", + "venv", +} + +PATTERN = re.compile(r"(?P[^:]+):(?P\d+): WARNING: (?P.+)") + + +def check_and_annotate(warnings: list[str], files_to_check: str) -> None: + """ + Convert Sphinx warning messages to GitHub Actions. + + Converts lines like: + .../Doc/library/cgi.rst:98: WARNING: reference target not found + to: + ::warning file=.../Doc/library/cgi.rst,line=98::reference target not found + + Non-matching lines are echoed unchanged. + + see: + https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-warning-message + """ + files_to_check = next(csv.reader([files_to_check])) + for warning in warnings: + if any(filename in warning for filename in files_to_check): + if match := PATTERN.fullmatch(warning): + print("::warning file={file},line={line}::{msg}".format_map(match)) + + +def fail_if_regression( + warnings: list[str], files_with_expected_nits: set[str], files_with_nits: set[str] +) -> int: + """ + Ensure some files always pass Sphinx nit-picky mode (no missing references). + These are files which are *not* in .nitignore. + """ + all_rst = { + str(rst) + for rst in Path("Doc/").rglob("*.rst") + if rst.parts[1] not in EXCLUDE_SUBDIRS + } + should_be_clean = all_rst - files_with_expected_nits - EXCLUDE_FILES + problem_files = sorted(should_be_clean & files_with_nits) + if problem_files: + print("\nError: must not contain warnings:\n") + for filename in problem_files: + print(filename) + for warning in warnings: + if filename in warning: + if match := PATTERN.fullmatch(warning): + print(" {line}: {msg}".format_map(match)) + return -1 + return 0 + + +def fail_if_improved( + files_with_expected_nits: set[str], files_with_nits: set[str] +) -> int: + """ + We may have fixed warnings in some files so that the files are now completely clean. + Good news! Let's add them to .nitignore to prevent regression. + """ + files_with_no_nits = files_with_expected_nits - files_with_nits + if files_with_no_nits: + print("\nCongratulations! You improved:\n") + for filename in sorted(files_with_no_nits): + print(filename) + print("\nPlease remove from Doc/tools/.nitignore\n") + return -1 + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--check-and-annotate", + help="Comma-separated list of files to check, " + "and annotate those with warnings on GitHub Actions", + ) + parser.add_argument( + "--fail-if-regression", + action="store_true", + help="Fail if known-good files have warnings", + ) + parser.add_argument( + "--fail-if-improved", + action="store_true", + help="Fail if new files with no nits are found", + ) + args = parser.parse_args() + exit_code = 0 + + wrong_directory_msg = "Must run this script from the repo root" + assert Path("Doc").exists() and Path("Doc").is_dir(), wrong_directory_msg + + with Path("Doc/sphinx-warnings.txt").open() as f: + warnings = f.read().splitlines() + + cwd = str(Path.cwd()) + os.path.sep + files_with_nits = { + warning.removeprefix(cwd).split(":")[0] + for warning in warnings + if "Doc/" in warning + } + + with Path("Doc/tools/.nitignore").open() as clean_files: + files_with_expected_nits = { + filename.strip() + for filename in clean_files + if filename.strip() and not filename.startswith("#") + } + + if args.check_and_annotate: + check_and_annotate(warnings, args.check_and_annotate) + + if args.fail_if_regression: + exit_code += fail_if_regression( + warnings, files_with_expected_nits, files_with_nits + ) + + if args.fail_if_improved: + exit_code += fail_if_improved(files_with_expected_nits, files_with_nits) + + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Doc/tools/touch-clean-files.py b/Doc/tools/touch-clean-files.py deleted file mode 100644 index 2b045bd68a0cf..0000000000000 --- a/Doc/tools/touch-clean-files.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 -""" -Touch files that must pass Sphinx nit-picky mode -so they are rebuilt and we can catch regressions. -""" -import argparse -import csv -import sys -from pathlib import Path - -wrong_directory_msg = "Must run this script from the repo root" -assert Path("Doc").exists() and Path("Doc").is_dir(), wrong_directory_msg - -# Exclude these whether they're dirty or clean, -# because they trigger a rebuild of dirty files. -EXCLUDE_FILES = { - Path("Doc/whatsnew/changelog.rst"), -} - -# Subdirectories of Doc/ to exclude. -EXCLUDE_SUBDIRS = { - ".env", - ".venv", - "env", - "includes", - "venv", -} - -ALL_RST = { - rst for rst in Path("Doc/").rglob("*.rst") if rst.parts[1] not in EXCLUDE_SUBDIRS -} - - -parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter -) -parser.add_argument("-c", "--clean", help="Comma-separated list of clean files") -args = parser.parse_args() - -if args.clean: - clean_files = next(csv.reader([args.clean])) - CLEAN = { - Path(filename.strip()) - for filename in clean_files - if Path(filename.strip()).is_file() - } -elif args.clean is not None: - print( - "Not touching any files: an empty string `--clean` arg value passed.", - ) - sys.exit(0) -else: - with Path("Doc/tools/.nitignore").open() as ignored_files: - IGNORED = { - Path(filename.strip()) - for filename in ignored_files - if filename.strip() and not filename.startswith("#") - } - CLEAN = ALL_RST - IGNORED - EXCLUDE_FILES - -print("Touching:") -for filename in sorted(CLEAN): - print(filename) - filename.touch() -print(f"Touched {len(CLEAN)} files") diff --git a/Doc/tools/warnings-to-gh-actions.py b/Doc/tools/warnings-to-gh-actions.py deleted file mode 100644 index da33a4ede07ab..0000000000000 --- a/Doc/tools/warnings-to-gh-actions.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 - -""" -Convert Sphinx warning messages to GitHub Actions. - -Converts lines like: - .../Doc/library/cgi.rst:98: WARNING: reference target not found -to: - ::warning file=.../Doc/library/cgi.rst,line=98::reference target not found - -Non-matching lines are echoed unchanged. - -see: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-warning-message -""" - -import re -import sys - -pattern = re.compile(r'(?P[^:]+):(?P\d+): WARNING: (?P.+)') - -for line in sys.stdin: - if match := pattern.fullmatch(line.strip()): - print('::warning file={file},line={line}::{msg}'.format_map(match)) - else: - print(line) From webhook-mailer at python.org Fri Aug 18 17:21:23 2023 From: webhook-mailer at python.org (ned-deily) Date: Fri, 18 Aug 2023 21:21:23 -0000 Subject: [Python-checkins] [3.10] gh-107565: Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, and 3.1.2. (GH-108120) Message-ID: https://github.com/python/cpython/commit/56e8c87e84ede369b53e502f328876c1ad03fea6 commit: 56e8c87e84ede369b53e502f328876c1ad03fea6 branch: 3.10 author: Ned Deily committer: ned-deily date: 2023-08-18T21:21:19Z summary: [3.10] gh-107565: Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, and 3.1.2. (GH-108120) (cherry picked from commit 441797d4ffb12acda257370b9e5e19ed8d6e8a71) files: A Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst M .github/workflows/build.yml M Tools/ssl/multissltests.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e91a17cd1744..21e3f3aefe5d8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -203,7 +203,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 1.1.1v PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout at v3 @@ -247,7 +247,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1u, 3.0.9, 3.1.1] + openssl_ver: [1.1.1v, 3.0.10, 3.1.2] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst new file mode 100644 index 0000000000000..c43ee680e8158 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst @@ -0,0 +1,2 @@ +Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, +and 3.1.2. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index c15405b704cea..cdaa75488fa86 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -47,9 +47,9 @@ ] OPENSSL_RECENT_VERSIONS = [ - "1.1.1u", - "3.0.9", - "3.1.1", + "1.1.1v", + "3.0.10", + "3.1.2", ] LIBRESSL_OLD_VERSIONS = [ From webhook-mailer at python.org Fri Aug 18 17:38:28 2023 From: webhook-mailer at python.org (ned-deily) Date: Fri, 18 Aug 2023 21:38:28 -0000 Subject: [Python-checkins] gh-107565: Update macOS installer to use OpenSSL 3.0.10. (GH-107897) Message-ID: https://github.com/python/cpython/commit/dc7b630b2359663bb7b8212d9f2f720c978d3daa commit: dc7b630b2359663bb7b8212d9f2f720c978d3daa branch: main author: Ned Deily committer: ned-deily date: 2023-08-18T17:38:24-04:00 summary: gh-107565: Update macOS installer to use OpenSSL 3.0.10. (GH-107897) files: A Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst M Mac/BuildScript/build-installer.py diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 0fc25ec74565d..c108ee21a7a2a 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.9", - url="https://www.openssl.org/source/openssl-3.0.9.tar.gz", - checksum='eb1ab04781474360f77c318ab89d8c5a03abc38e63d65a603cabbf1b00a1dc90', + name="OpenSSL 3.0.10", + url="https://www.openssl.org/source/openssl-3.0.10.tar.gz", + checksum='1761d4f5b13a1028b9b6f3d4b8e17feb0cedc9370f6afe61d7193d2cdce83323', buildrecipe=build_universal_openssl, configure=None, install=None, diff --git a/Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst b/Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst new file mode 100644 index 0000000000000..c238c4760239e --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst @@ -0,0 +1 @@ +Update macOS installer to use OpenSSL 3.0.10. From webhook-mailer at python.org Fri Aug 18 20:43:32 2023 From: webhook-mailer at python.org (CAM-Gerlach) Date: Sat, 19 Aug 2023 00:43:32 -0000 Subject: [Python-checkins] gh-101100: Only show GitHub check annotations on changed doc paragraphs (#108065) Message-ID: https://github.com/python/cpython/commit/eb953d6e4484339067837020f77eecac61f8d4f8 commit: eb953d6e4484339067837020f77eecac61f8d4f8 branch: main author: C.A.M. Gerlach committer: CAM-Gerlach date: 2023-08-18T19:43:28-05:00 summary: gh-101100: Only show GitHub check annotations on changed doc paragraphs (#108065) * Only show GitHub check annotations on changed doc paragraphs * Improve check-warnings script arg parsing following Hugo's suggestions * Factor filtering warnings by modified diffs into helper function * Build docs on unmerged branch so warning lines match & avoid deep clone --------- Co-authored-by: Hugo van Kemenade Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M .github/workflows/reusable-docs.yml M Doc/tools/check-warnings.py diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 39e5ad62924ad..6150b1a7d416a 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -16,8 +16,30 @@ jobs: name: 'Docs' runs-on: ubuntu-latest timeout-minutes: 60 + env: + branch_base: 'origin/${{ github.event.pull_request.base.ref }}' + branch_pr: 'origin/${{ github.event.pull_request.head.ref }}' + refspec_base: '+${{ github.event.pull_request.base.sha }}:remotes/origin/${{ github.event.pull_request.base.ref }}' + refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - - uses: actions/checkout at v3 + - name: 'Check out latest PR branch commit' + uses: actions/checkout at v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721 + - name: 'Fetch commits to get branch diff' + run: | + # Fetch enough history to find a common ancestor commit (aka merge-base): + git fetch origin ${{ env.refspec_pr }} --depth=$(( ${{ github.event.pull_request.commits }} + 1 )) \ + --no-tags --prune --no-recurse-submodules + + # This should get the oldest commit in the local fetched history (which may not be the commit the PR branched from): + COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 ${{ env.branch_pr }} ) + DATE=$( git log --date=iso8601 --format=%cd "${COMMON_ANCESTOR}" ) + + # Get all commits since that commit date from the base branch (eg: master or main): + git fetch origin ${{ env.refspec_base }} --shallow-since="${DATE}" \ + --no-tags --prune --no-recurse-submodules - name: 'Set up Python' uses: actions/setup-python at v4 with: @@ -28,13 +50,6 @@ jobs: run: make -C Doc/ venv # To annotate PRs with Sphinx nitpicks (missing references) - - name: 'Get list of changed files' - if: github.event_name == 'pull_request' - id: changed_files - uses: Ana06/get-changed-files at v2.2.0 - with: - filter: "Doc/**" - format: csv # works for paths with spaces - name: 'Build HTML documentation' continue-on-error: true run: | @@ -45,7 +60,7 @@ jobs: if: github.event_name == 'pull_request' run: | python Doc/tools/check-warnings.py \ - --check-and-annotate '${{ steps.changed_files.outputs.added_modified }}' \ + --annotate-diff '${{ env.branch_base }}' '${{ env.branch_pr }}' \ --fail-if-regression \ --fail-if-improved diff --git a/Doc/tools/check-warnings.py b/Doc/tools/check-warnings.py index c17d0f51cd127..809a8d63087e1 100644 --- a/Doc/tools/check-warnings.py +++ b/Doc/tools/check-warnings.py @@ -2,12 +2,16 @@ """ Check the output of running Sphinx in nit-picky mode (missing references). """ +from __future__ import annotations + import argparse -import csv +import itertools import os import re +import subprocess import sys from pathlib import Path +from typing import TextIO # Exclude these whether they're dirty or clean, # because they trigger a rebuild of dirty files. @@ -24,28 +28,178 @@ "venv", } -PATTERN = re.compile(r"(?P[^:]+):(?P\d+): WARNING: (?P.+)") +# Regex pattern to match the parts of a Sphinx warning +WARNING_PATTERN = re.compile( + r"(?P([A-Za-z]:[\\/])?[^:]+):(?P\d+): WARNING: (?P.+)" +) + +# Regex pattern to match the line numbers in a Git unified diff +DIFF_PATTERN = re.compile( + r"^@@ -(?P\d+)(?:,(?P\d+))? \+(?P\d+)(?:,(?P\d+))? @@", + flags=re.MULTILINE, +) + + +def get_diff_files(ref_a: str, ref_b: str, filter_mode: str = "") -> set[Path]: + """List the files changed between two Git refs, filtered by change type.""" + added_files_result = subprocess.run( + [ + "git", + "diff", + f"--diff-filter={filter_mode}", + "--name-only", + f"{ref_a}...{ref_b}", + "--", + ], + stdout=subprocess.PIPE, + check=True, + text=True, + encoding="UTF-8", + ) + + added_files = added_files_result.stdout.strip().split("\n") + return {Path(file.strip()) for file in added_files if file.strip()} + + +def get_diff_lines(ref_a: str, ref_b: str, file: Path) -> list[int]: + """List the lines changed between two Git refs for a specific file.""" + diff_output = subprocess.run( + [ + "git", + "diff", + "--unified=0", + f"{ref_a}...{ref_b}", + "--", + str(file), + ], + stdout=subprocess.PIPE, + check=True, + text=True, + encoding="UTF-8", + ) + + # Scrape line offsets + lengths from diff and convert to line numbers + line_matches = DIFF_PATTERN.finditer(diff_output.stdout) + # Removed and added line counts are 1 if not printed + line_match_values = [ + line_match.groupdict(default=1) for line_match in line_matches + ] + line_ints = [ + (int(match_value["lineb"]), int(match_value["added"])) + for match_value in line_match_values + ] + line_ranges = [ + range(line_b, line_b + added) for line_b, added in line_ints + ] + line_numbers = list(itertools.chain(*line_ranges)) + + return line_numbers + + +def get_para_line_numbers(file_obj: TextIO) -> list[list[int]]: + """Get the line numbers of text in a file object, grouped by paragraph.""" + paragraphs = [] + prev_line = None + for lineno, line in enumerate(file_obj): + lineno = lineno + 1 + if prev_line is None or (line.strip() and not prev_line.strip()): + paragraph = [lineno - 1] + paragraphs.append(paragraph) + paragraph.append(lineno) + prev_line = line + return paragraphs + + +def filter_and_parse_warnings( + warnings: list[str], files: set[Path] +) -> list[re.Match[str]]: + """Get the warnings matching passed files and parse them with regex.""" + filtered_warnings = [ + warning + for warning in warnings + if any(str(file) in warning for file in files) + ] + warning_matches = [ + WARNING_PATTERN.fullmatch(warning.strip()) + for warning in filtered_warnings + ] + non_null_matches = [warning for warning in warning_matches if warning] + return non_null_matches + + +def filter_warnings_by_diff( + warnings: list[re.Match[str]], ref_a: str, ref_b: str, file: Path +) -> list[re.Match[str]]: + """Filter the passed per-file warnings to just those on changed lines.""" + diff_lines = get_diff_lines(ref_a, ref_b, file) + with file.open(encoding="UTF-8") as file_obj: + paragraphs = get_para_line_numbers(file_obj) + touched_paras = [ + para_lines + for para_lines in paragraphs + if set(diff_lines) & set(para_lines) + ] + touched_para_lines = set(itertools.chain(*touched_paras)) + warnings_infile = [ + warning for warning in warnings if str(file) in warning["file"] + ] + warnings_touched = [ + warning + for warning in warnings_infile + if int(warning["line"]) in touched_para_lines + ] + return warnings_touched + +def process_touched_warnings( + warnings: list[str], ref_a: str, ref_b: str +) -> list[re.Match[str]]: + """Filter a list of Sphinx warnings to those affecting touched lines.""" + added_files, modified_files = tuple( + get_diff_files(ref_a, ref_b, filter_mode=mode) for mode in ("A", "M") + ) + + warnings_added = filter_and_parse_warnings(warnings, added_files) + warnings_modified = filter_and_parse_warnings(warnings, modified_files) + + modified_files_warned = { + file + for file in modified_files + if any(str(file) in warning["file"] for warning in warnings_modified) + } -def check_and_annotate(warnings: list[str], files_to_check: str) -> None: + warnings_modified_touched = [ + filter_warnings_by_diff(warnings_modified, ref_a, ref_b, file) + for file in modified_files_warned + ] + warnings_touched = warnings_added + list( + itertools.chain(*warnings_modified_touched) + ) + + return warnings_touched + + +def annotate_diff( + warnings: list[str], ref_a: str = "main", ref_b: str = "HEAD" +) -> None: """ - Convert Sphinx warning messages to GitHub Actions. + Convert Sphinx warning messages to GitHub Actions for changed paragraphs. Converts lines like: .../Doc/library/cgi.rst:98: WARNING: reference target not found to: ::warning file=.../Doc/library/cgi.rst,line=98::reference target not found - Non-matching lines are echoed unchanged. - - see: + See: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-warning-message """ - files_to_check = next(csv.reader([files_to_check])) - for warning in warnings: - if any(filename in warning for filename in files_to_check): - if match := PATTERN.fullmatch(warning): - print("::warning file={file},line={line}::{msg}".format_map(match)) + warnings_touched = process_touched_warnings(warnings, ref_a, ref_b) + print("Emitting doc warnings matching modified lines:") + for warning in warnings_touched: + print("::warning file={file},line={line}::{msg}".format_map(warning)) + print(warning[0]) + if not warnings_touched: + print("None") def fail_if_regression( @@ -68,7 +222,7 @@ def fail_if_regression( print(filename) for warning in warnings: if filename in warning: - if match := PATTERN.fullmatch(warning): + if match := WARNING_PATTERN.fullmatch(warning): print(" {line}: {msg}".format_map(match)) return -1 return 0 @@ -91,12 +245,14 @@ def fail_if_improved( return 0 -def main() -> int: +def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( - "--check-and-annotate", - help="Comma-separated list of files to check, " - "and annotate those with warnings on GitHub Actions", + "--annotate-diff", + nargs="*", + metavar=("BASE_REF", "HEAD_REF"), + help="Add GitHub Actions annotations on the diff for warnings on " + "lines changed between the given refs (main and HEAD, by default)", ) parser.add_argument( "--fail-if-regression", @@ -108,13 +264,19 @@ def main() -> int: action="store_true", help="Fail if new files with no nits are found", ) - args = parser.parse_args() + + args = parser.parse_args(argv) + if args.annotate_diff is not None and len(args.annotate_diff) > 2: + parser.error( + "--annotate-diff takes between 0 and 2 ref args, not " + f"{len(args.annotate_diff)} {tuple(args.annotate_diff)}" + ) exit_code = 0 wrong_directory_msg = "Must run this script from the repo root" assert Path("Doc").exists() and Path("Doc").is_dir(), wrong_directory_msg - with Path("Doc/sphinx-warnings.txt").open() as f: + with Path("Doc/sphinx-warnings.txt").open(encoding="UTF-8") as f: warnings = f.read().splitlines() cwd = str(Path.cwd()) + os.path.sep @@ -124,15 +286,15 @@ def main() -> int: if "Doc/" in warning } - with Path("Doc/tools/.nitignore").open() as clean_files: + with Path("Doc/tools/.nitignore").open(encoding="UTF-8") as clean_files: files_with_expected_nits = { filename.strip() for filename in clean_files if filename.strip() and not filename.startswith("#") } - if args.check_and_annotate: - check_and_annotate(warnings, args.check_and_annotate) + if args.annotate_diff is not None: + annotate_diff(warnings, *args.annotate_diff) if args.fail_if_regression: exit_code += fail_if_regression( From webhook-mailer at python.org Sat Aug 19 03:13:39 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Sat, 19 Aug 2023 07:13:39 -0000 Subject: [Python-checkins] gh-107704: Argument Clinic: add support for deprecating keyword use of parameters (GH-107984) Message-ID: https://github.com/python/cpython/commit/2f311437cd51afaa68fd671bb99ff515cf7b029a commit: 2f311437cd51afaa68fd671bb99ff515cf7b029a branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-19T10:13:35+03:00 summary: gh-107704: Argument Clinic: add support for deprecating keyword use of parameters (GH-107984) It is now possible to deprecate passing keyword arguments for keyword-or-positional parameters with Argument Clinic, using the new '/ [from X.Y]' syntax. (To be read as "positional-only from Python version X.Y") Co-authored-by: Erlend E. Aasland Co-authored-by: Alex Waygood files: A Misc/NEWS.d/next/Tools-Demos/2023-08-15-19-50-49.gh-issue-107704.Uu84vd.rst A Modules/clinic/_testclinic_depr.c.h D Modules/clinic/_testclinic_depr_star.c.h M Doc/howto/clinic.rst M Lib/test/test_clinic.py M Modules/_sqlite/clinic/connection.c.h M Modules/_testclinic.c M Tools/c-analyzer/cpython/globals-to-fix.tsv M Tools/clinic/clinic.py diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 463a938fafa8d..667859e20a5d8 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1941,54 +1941,70 @@ The generated docstring ends up looking like this: .. _clinic-howto-deprecate-positional: +.. _clinic-howto-deprecate-keyword: -How to deprecate passing parameters positionally ------------------------------------------------- +How to deprecate passing parameters positionally or by keyword +-------------------------------------------------------------- Argument Clinic provides syntax that makes it possible to generate code that -deprecates passing :term:`arguments ` positionally. +deprecates passing :term:`arguments ` for positional-or-keyword +:term:`parameters ` positionally or by keyword. For example, say we've got a module-level function :py:func:`!foo.myfunc` -that has three :term:`parameters `: -positional-or-keyword parameters *a* and *b*, and a keyword-only parameter *c*:: +that has five parameters: a positional-only parameter *a*, three +positional-or-keyword parameters *b*, *c* and *d*, and a keyword-only +parameter *e*:: /*[clinic input] module foo myfunc a: int + / b: int - * c: int + d: int + * + e: int [clinic start generated output]*/ -We now want to make the *b* parameter keyword-only; -however, we'll have to wait two releases before making this change, +We now want to make the *b* parameter positional-only and the *d* parameter +keyword-only; +however, we'll have to wait two releases before making these changes, as mandated by Python's backwards-compatibility policy (see :pep:`387`). For this example, imagine we're in the development phase for Python 3.12: that means we'll be allowed to introduce deprecation warnings in Python 3.12 -whenever the *b* parameter is passed positionally, -and we'll be allowed to make it keyword-only in Python 3.14 at the earliest. +whenever an argument for the *b* parameter is passed by keyword or an argument +for the *d* parameter is passed positionally, and we'll be allowed to make +them positional-only and keyword-only respectively in Python 3.14 at +the earliest. We can use Argument Clinic to emit the desired deprecation warnings -using the ``* [from ...]`` syntax, -by adding the line ``* [from 3.14]`` right above the *b* parameter:: +using the ``[from ...]`` syntax, by adding the line ``/ [from 3.14]`` right +below the *b* parameter and adding the line ``* [from 3.14]`` right above +the *d* parameter:: /*[clinic input] module foo myfunc a: int - * [from 3.14] + / b: int - * + / [from 3.14] c: int + * [from 3.14] + d: int + * + e: int [clinic start generated output]*/ Next, regenerate Argument Clinic code (``make clinic``), and add unit tests for the new behaviour. The generated code will now emit a :exc:`DeprecationWarning` -when an :term:`argument` for the :term:`parameter` *b* is passed positionally. +when an :term:`argument` for the :term:`parameter` *d* is passed positionally +(e.g ``myfunc(1, 2, 3, 4, e=5)``) or an argument for the parameter *b* is +passed by keyword (e.g ``myfunc(1, b=2, c=3, d=4, e=5)``). C preprocessor directives are also generated for emitting -compiler warnings if the ``* [from ...]`` line has not been removed +compiler warnings if the ``[from ...]`` lines have not been removed from the Argument Clinic input when the deprecation period is over, which means when the alpha phase of the specified Python version kicks in. @@ -2001,21 +2017,26 @@ Luckily for us, compiler warnings are now generated: .. code-block:: none In file included from Modules/foomodule.c:139: - Modules/clinic/foomodule.c.h:139:8: warning: In 'foomodule.c', update parameter(s) 'a' and 'b' in the clinic input of 'mymod.myfunc' to be keyword-only. [-W#warnings] - # warning "In 'foomodule.c', update parameter(s) 'a' and 'b' in the clinic input of 'mymod.myfunc' to be keyword-only. [-W#warnings]" + Modules/clinic/foomodule.c.h:139:8: warning: In 'foomodule.c', update the clinic input of 'mymod.myfunc'. [-W#warnings] + # warning "In 'foomodule.c', update the clinic input of 'mymod.myfunc'. [-W#warnings]" ^ -We now close the deprecation phase by making *b* keyword-only; -replace the ``* [from ...]`` line above *b* -with the ``*`` from the line above *c*:: +We now close the deprecation phase by making *a* positional-only and *c* +keyword-only; +replace the ``/ [from ...]`` line below *b* with the ``/`` from the line +below *a* and the ``* [from ...]`` line above *d* with the ``*`` from +the line above *e*:: /*[clinic input] module foo myfunc a: int - * b: int + / c: int + * + d: int + e: int [clinic start generated output]*/ Finally, run ``make clinic`` to regenerate the Argument Clinic code, diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 38eabd2f7f21d..934c1e3ffccca 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1611,7 +1611,7 @@ def test_parameters_required_after_star(self): "module foo\nfoo.bar\n this: int\n *", "module foo\nfoo.bar\n this: int\n *\nDocstring.", ) - err = "Function 'foo.bar' specifies '*' without any parameters afterwards." + err = "Function 'bar' specifies '*' without following parameters." for block in dataset: with self.subTest(block=block): self.expect_failure(block, err) @@ -1679,7 +1679,7 @@ def test_depr_star_invalid_format_1(self): Docstring. """ err = ( - "Function 'foo.bar': expected format '* [from major.minor]' " + "Function 'bar': expected format '[from major.minor]' " "where 'major' and 'minor' are integers; got '3'" ) self.expect_failure(block, err, lineno=3) @@ -1693,7 +1693,7 @@ def test_depr_star_invalid_format_2(self): Docstring. """ err = ( - "Function 'foo.bar': expected format '* [from major.minor]' " + "Function 'bar': expected format '[from major.minor]' " "where 'major' and 'minor' are integers; got 'a.b'" ) self.expect_failure(block, err, lineno=3) @@ -1707,7 +1707,7 @@ def test_depr_star_invalid_format_3(self): Docstring. """ err = ( - "Function 'foo.bar': expected format '* [from major.minor]' " + "Function 'bar': expected format '[from major.minor]' " "where 'major' and 'minor' are integers; got '1.2.3'" ) self.expect_failure(block, err, lineno=3) @@ -1721,8 +1721,24 @@ def test_parameters_required_after_depr_star(self): Docstring. """ err = ( - "Function 'foo.bar' specifies '* [from ...]' without " - "any parameters afterwards" + "Function 'bar' specifies '* [from ...]' without " + "following parameters." + ) + self.expect_failure(block, err, lineno=4) + + def test_parameters_required_after_depr_star2(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + * + b: int + Docstring. + """ + err = ( + "Function 'bar' specifies '* [from ...]' without " + "following parameters." ) self.expect_failure(block, err, lineno=4) @@ -1735,7 +1751,7 @@ def test_depr_star_must_come_before_star(self): * [from 3.14] Docstring. """ - err = "Function 'foo.bar': '* [from ...]' must come before '*'" + err = "Function 'bar': '* [from ...]' must come before '*'" self.expect_failure(block, err, lineno=4) def test_depr_star_duplicate(self): @@ -1749,7 +1765,49 @@ def test_depr_star_duplicate(self): c: int Docstring. """ - err = "Function 'foo.bar' uses '[from ...]' more than once" + err = "Function 'bar' uses '* [from ...]' more than once." + self.expect_failure(block, err, lineno=5) + + def test_depr_star_duplicate2(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + b: int + * [from 3.15] + c: int + Docstring. + """ + err = "Function 'bar' uses '* [from ...]' more than once." + self.expect_failure(block, err, lineno=5) + + def test_depr_slash_duplicate(self): + block = """ + module foo + foo.bar + a: int + / [from 3.14] + b: int + / [from 3.14] + c: int + Docstring. + """ + err = "Function 'bar' uses '/ [from ...]' more than once." + self.expect_failure(block, err, lineno=5) + + def test_depr_slash_duplicate2(self): + block = """ + module foo + foo.bar + a: int + / [from 3.14] + b: int + / [from 3.15] + c: int + Docstring. + """ + err = "Function 'bar' uses '/ [from ...]' more than once." self.expect_failure(block, err, lineno=5) def test_single_slash(self): @@ -1765,6 +1823,34 @@ def test_single_slash(self): ) self.expect_failure(block, err) + def test_parameters_required_before_depr_slash(self): + block = """ + module foo + foo.bar + / [from 3.14] + Docstring. + """ + err = ( + "Function 'bar' specifies '/ [from ...]' without " + "preceding parameters." + ) + self.expect_failure(block, err, lineno=2) + + def test_parameters_required_before_depr_slash2(self): + block = """ + module foo + foo.bar + a: int + / + / [from 3.14] + Docstring. + """ + err = ( + "Function 'bar' specifies '/ [from ...]' without " + "preceding parameters." + ) + self.expect_failure(block, err, lineno=4) + def test_double_slash(self): block = """ module foo @@ -1787,12 +1873,61 @@ def test_mix_star_and_slash(self): z: int / """ - err = ( - "Function 'bar' mixes keyword-only and positional-only parameters, " - "which is unsupported." - ) + err = "Function 'bar': '/' must precede '*'" self.expect_failure(block, err) + def test_depr_star_must_come_after_slash(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + / + b: int + Docstring. + """ + err = "Function 'bar': '/' must precede '* [from ...]'" + self.expect_failure(block, err, lineno=4) + + def test_depr_star_must_come_after_depr_slash(self): + block = """ + module foo + foo.bar + a: int + * [from 3.14] + / [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '/ [from ...]' must precede '* [from ...]'" + self.expect_failure(block, err, lineno=4) + + def test_star_must_come_after_depr_slash(self): + block = """ + module foo + foo.bar + a: int + * + / [from 3.14] + b: int + Docstring. + """ + err = "Function 'bar': '/ [from ...]' must precede '*'" + self.expect_failure(block, err, lineno=4) + + def test_depr_slash_must_come_after_slash(self): + block = """ + module foo + foo.bar + a: int + / [from 3.14] + / + b: int + Docstring. + """ + err = "Function 'bar': '/' must precede '/ [from ...]'" + self.expect_failure(block, err, lineno=4) + def test_parameters_not_permitted_after_slash_for_now(self): block = """ module foo @@ -2589,11 +2724,33 @@ class ClinicFunctionalTest(unittest.TestCase): locals().update((name, getattr(ac_tester, name)) for name in dir(ac_tester) if name.startswith('test_')) - def check_depr_star(self, pnames, fn, *args, **kwds): + def check_depr_star(self, pnames, fn, *args, name=None, **kwds): + if name is None: + name = fn.__qualname__ + if isinstance(fn, type): + name = f'{fn.__module__}.{name}' regex = ( fr"Passing( more than)?( [0-9]+)? positional argument(s)? to " - fr"{fn.__name__}\(\) is deprecated. Parameter(s)? {pnames} will " - fr"become( a)? keyword-only parameter(s)? in Python 3\.14" + fr"{re.escape(name)}\(\) is deprecated. Parameters? {pnames} will " + fr"become( a)? keyword-only parameters? in Python 3\.14" + ) + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + # Record the line number, so we're sure we've got the correct stack + # level on the deprecation warning. + _, lineno = fn(*args, **kwds), sys._getframe().f_lineno + self.assertEqual(cm.filename, __file__) + self.assertEqual(cm.lineno, lineno) + + def check_depr_kwd(self, pnames, fn, *args, name=None, **kwds): + if name is None: + name = fn.__qualname__ + if isinstance(fn, type): + name = f'{fn.__module__}.{name}' + pl = 's' if ' ' in pnames else '' + regex = ( + fr"Passing keyword argument{pl} {pnames} to " + fr"{re.escape(name)}\(\) is deprecated. Corresponding parameter{pl} " + fr"will become positional-only in Python 3\.14." ) with self.assertWarnsRegex(DeprecationWarning, regex) as cm: # Record the line number, so we're sure we've got the correct stack @@ -3067,46 +3224,67 @@ def test_cloned_func_with_converter_exception_message(self): self.assertEqual(func(), name) def test_depr_star_new(self): - regex = re.escape( - "Passing positional arguments to _testclinic.DeprStarNew() is " - "deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.14." - ) - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - ac_tester.DeprStarNew(None) - self.assertEqual(cm.filename, __file__) + cls = ac_tester.DeprStarNew + cls() + cls(a=None) + self.check_depr_star("'a'", cls, None) def test_depr_star_new_cloned(self): - regex = re.escape( - "Passing positional arguments to _testclinic.DeprStarNew.cloned() " - "is deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.14." - ) - obj = ac_tester.DeprStarNew(a=None) - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - obj.cloned(None) - self.assertEqual(cm.filename, __file__) + fn = ac_tester.DeprStarNew().cloned + fn() + fn(a=None) + self.check_depr_star("'a'", fn, None, name='_testclinic.DeprStarNew.cloned') def test_depr_star_init(self): - regex = re.escape( - "Passing positional arguments to _testclinic.DeprStarInit() is " - "deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.14." - ) - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - ac_tester.DeprStarInit(None) - self.assertEqual(cm.filename, __file__) + cls = ac_tester.DeprStarInit + cls() + cls(a=None) + self.check_depr_star("'a'", cls, None) def test_depr_star_init_cloned(self): - regex = re.escape( - "Passing positional arguments to _testclinic.DeprStarInit.cloned() " - "is deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.14." - ) - obj = ac_tester.DeprStarInit(a=None) - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - obj.cloned(None) - self.assertEqual(cm.filename, __file__) + fn = ac_tester.DeprStarInit().cloned + fn() + fn(a=None) + self.check_depr_star("'a'", fn, None, name='_testclinic.DeprStarInit.cloned') + + def test_depr_star_init_noinline(self): + cls = ac_tester.DeprStarInitNoInline + self.assertRaises(TypeError, cls, "a") + cls(a="a", b="b") + cls(a="a", b="b", c="c") + cls("a", b="b") + cls("a", b="b", c="c") + check = partial(self.check_depr_star, "'b' and 'c'", cls) + check("a", "b") + check("a", "b", "c") + check("a", "b", c="c") + self.assertRaises(TypeError, cls, "a", "b", "c", "d") + + def test_depr_kwd_new(self): + cls = ac_tester.DeprKwdNew + cls() + cls(None) + self.check_depr_kwd("'a'", cls, a=None) + + def test_depr_kwd_init(self): + cls = ac_tester.DeprKwdInit + cls() + cls(None) + self.check_depr_kwd("'a'", cls, a=None) + + def test_depr_kwd_init_noinline(self): + cls = ac_tester.DeprKwdInitNoInline + cls = ac_tester.depr_star_noinline + self.assertRaises(TypeError, cls, "a") + cls(a="a", b="b") + cls(a="a", b="b", c="c") + cls("a", b="b") + cls("a", b="b", c="c") + check = partial(self.check_depr_star, "'b' and 'c'", cls) + check("a", "b") + check("a", "b", "c") + check("a", "b", c="c") + self.assertRaises(TypeError, cls, "a", "b", "c", "d") def test_depr_star_pos0_len1(self): fn = ac_tester.depr_star_pos0_len1 @@ -3177,6 +3355,103 @@ def test_depr_star_pos2_len2_with_kwd(self): check("a", "b", "c", d=0, e=0) check("a", "b", "c", "d", e=0) + def test_depr_star_noinline(self): + fn = ac_tester.depr_star_noinline + self.assertRaises(TypeError, fn, "a") + fn(a="a", b="b") + fn(a="a", b="b", c="c") + fn("a", b="b") + fn("a", b="b", c="c") + check = partial(self.check_depr_star, "'b' and 'c'", fn) + check("a", "b") + check("a", "b", "c") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + + def test_depr_kwd_required_1(self): + fn = ac_tester.depr_kwd_required_1 + fn("a", "b") + self.assertRaises(TypeError, fn, "a") + self.assertRaises(TypeError, fn, "a", "b", "c") + check = partial(self.check_depr_kwd, "'b'", fn) + check("a", b="b") + self.assertRaises(TypeError, fn, a="a", b="b") + + def test_depr_kwd_required_2(self): + fn = ac_tester.depr_kwd_required_2 + fn("a", "b", "c") + self.assertRaises(TypeError, fn, "a", "b") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", "b", c="c") + check("a", b="b", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + + def test_depr_kwd_optional_1(self): + fn = ac_tester.depr_kwd_optional_1 + fn("a") + fn("a", "b") + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, "a", "b", "c") + check = partial(self.check_depr_kwd, "'b'", fn) + check("a", b="b") + self.assertRaises(TypeError, fn, a="a", b="b") + + def test_depr_kwd_optional_2(self): + fn = ac_tester.depr_kwd_optional_2 + fn("a") + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", b="b") + check("a", c="c") + check("a", b="b", c="c") + check("a", c="c", b="b") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + + def test_depr_kwd_optional_3(self): + fn = ac_tester.depr_kwd_optional_3 + fn() + fn("a") + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'a', 'b' and 'c'", fn) + check("a", "b", c="c") + check("a", b="b") + check(a="a") + + def test_depr_kwd_required_optional(self): + fn = ac_tester.depr_kwd_required_optional + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn) + self.assertRaises(TypeError, fn, "a") + self.assertRaises(TypeError, fn, "a", "b", "c", "d") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", b="b") + check("a", b="b", c="c") + check("a", c="c", b="b") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, "a", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + + def test_depr_kwd_noinline(self): + fn = ac_tester.depr_kwd_noinline + fn("a", "b") + fn("a", "b", "c") + self.assertRaises(TypeError, fn, "a") + check = partial(self.check_depr_kwd, "'b' and 'c'", fn) + check("a", b="b") + check("a", b="b", c="c") + check("a", c="c", b="b") + check("a", "b", c="c") + self.assertRaises(TypeError, fn, "a", c="c") + self.assertRaises(TypeError, fn, a="a", b="b", c="c") + class PermutationTests(unittest.TestCase): """Test permutation support functions.""" diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-15-19-50-49.gh-issue-107704.Uu84vd.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-15-19-50-49.gh-issue-107704.Uu84vd.rst new file mode 100644 index 0000000000000..ffdcfa6a429e6 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-15-19-50-49.gh-issue-107704.Uu84vd.rst @@ -0,0 +1,4 @@ +It is now possible to deprecate passing keyword arguments for +keyword-or-positional parameters with Argument Clinic, using the new ``/ +[from X.Y]`` syntax. (To be read as *"positional-only from Python version +X.Y"*.) See :ref:`clinic-howto-deprecate-keyword` for more information. diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index af98d61ea7ccc..fe2196d0f7ee5 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -16,6 +16,17 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, int cache_size, int uri, enum autocommit_mode autocommit); +// Emit compiler warnings when we get to Python 3.15. +#if PY_VERSION_HEX >= 0x030f00C0 +# error "Update the clinic input of '_sqlite3.Connection.__init__'." +#elif PY_VERSION_HEX >= 0x030f00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_sqlite3.Connection.__init__'.") +# else +# warning "Update the clinic input of '_sqlite3.Connection.__init__'." +# endif +#endif + static int pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) { @@ -59,28 +70,6 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) int uri = 0; enum autocommit_mode autocommit = LEGACY_TRANSACTION_CONTROL; - // Emit compiler warnings when we get to Python 3.15. - #if PY_VERSION_HEX >= 0x030f00C0 - # error \ - "In connection.c, update parameter(s) 'timeout', 'detect_types', " \ - "'isolation_level', 'check_same_thread', 'factory', " \ - "'cached_statements' and 'uri' in the clinic input of " \ - "'_sqlite3.Connection.__init__' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030f00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In connection.c, update parameter(s) 'timeout', 'detect_types', " \ - "'isolation_level', 'check_same_thread', 'factory', " \ - "'cached_statements' and 'uri' in the clinic input of " \ - "'_sqlite3.Connection.__init__' to be keyword-only.") - # else - # warning \ - "In connection.c, update parameter(s) 'timeout', 'detect_types', " \ - "'isolation_level', 'check_same_thread', 'factory', " \ - "'cached_statements' and 'uri' in the clinic input of " \ - "'_sqlite3.Connection.__init__' to be keyword-only." - # endif - #endif if (nargs > 1 && nargs <= 8) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing more than 1 positional argument to _sqlite3.Connection()" @@ -89,7 +78,7 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) "'cached_statements' and 'uri' will become keyword-only " "parameters in Python 3.15.", 1)) { - goto exit; + goto exit; } } fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 8, 0, argsbuf); @@ -1692,4 +1681,4 @@ getconfig(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=5a05e5294ad9d2ce input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0ad9d55977a51b8f input=a9049054013a1b77]*/ diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index c33536234af0b..efec04d99029b 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1195,14 +1195,14 @@ clone_with_conv_f2_impl(PyObject *module, custom_t path) /*[clinic input] output push -destination deprstar new file '{dirname}/clinic/_testclinic_depr_star.c.h' +destination deprstar new file '{dirname}/clinic/_testclinic_depr.c.h' output everything deprstar #output methoddef_ifndef buffer 1 output docstring_prototype suppress output parser_prototype suppress output impl_definition block [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=f88f37038e00fb0a]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=32116eac48a42d34]*/ // Mock Python version 3.8 @@ -1211,7 +1211,7 @@ output impl_definition block #define PY_VERSION_HEX 0x03080000 -#include "clinic/_testclinic_depr_star.c.h" +#include "clinic/_testclinic_depr.c.h" /*[clinic input] @@ -1219,13 +1219,13 @@ class _testclinic.DeprStarNew "PyObject *" "PyObject" @classmethod _testclinic.DeprStarNew.__new__ as depr_star_new * [from 3.14] - a: object + a: object = None The deprecation message should use the class name instead of __new__. [clinic start generated code]*/ static PyObject * depr_star_new_impl(PyTypeObject *type, PyObject *a) -/*[clinic end generated code: output=bdbb36244f90cf46 input=f4ae7dafbc23c378]*/ +/*[clinic end generated code: output=bdbb36244f90cf46 input=fdd640db964b4dc1]*/ { return type->tp_alloc(type, 0); } @@ -1260,13 +1260,13 @@ static PyTypeObject DeprStarNew = { class _testclinic.DeprStarInit "PyObject *" "PyObject" _testclinic.DeprStarInit.__init__ as depr_star_init * [from 3.14] - a: object + a: object = None The deprecation message should use the class name instead of __init__. [clinic start generated code]*/ static int depr_star_init_impl(PyObject *self, PyObject *a) -/*[clinic end generated code: output=8d27b43c286d3ecc input=659ebc748d87fa86]*/ +/*[clinic end generated code: output=8d27b43c286d3ecc input=5575b77229d5e2be]*/ { return 0; } @@ -1298,6 +1298,116 @@ static PyTypeObject DeprStarInit = { }; +/*[clinic input] +class _testclinic.DeprStarInitNoInline "PyObject *" "PyObject" +_testclinic.DeprStarInitNoInline.__init__ as depr_star_init_noinline + a: object + * [from 3.14] + b: object + c: object = None + * + # Force to use _PyArg_ParseTupleAndKeywordsFast. + d: str(accept={str, robuffer}, zeroes=True) = '' +[clinic start generated code]*/ + +static int +depr_star_init_noinline_impl(PyObject *self, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length) +/*[clinic end generated code: output=9b31fc167f1bf9f7 input=5a887543122bca48]*/ +{ + return 0; +} + +static PyTypeObject DeprStarInitNoInline = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.DeprStarInitNoInline", + .tp_basicsize = sizeof(PyObject), + .tp_new = PyType_GenericNew, + .tp_init = depr_star_init_noinline, + .tp_flags = Py_TPFLAGS_DEFAULT, +}; + + +/*[clinic input] +class _testclinic.DeprKwdNew "PyObject *" "PyObject" + at classmethod +_testclinic.DeprKwdNew.__new__ as depr_kwd_new + a: object = None + / [from 3.14] +The deprecation message should use the class name instead of __new__. +[clinic start generated code]*/ + +static PyObject * +depr_kwd_new_impl(PyTypeObject *type, PyObject *a) +/*[clinic end generated code: output=618d07afc5616149 input=6c7d13c471013c10]*/ +{ + return type->tp_alloc(type, 0); +} + +static PyTypeObject DeprKwdNew = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.DeprKwdNew", + .tp_basicsize = sizeof(PyObject), + .tp_new = depr_kwd_new, + .tp_flags = Py_TPFLAGS_DEFAULT, +}; + + +/*[clinic input] +class _testclinic.DeprKwdInit "PyObject *" "PyObject" +_testclinic.DeprKwdInit.__init__ as depr_kwd_init + a: object = None + / [from 3.14] +The deprecation message should use the class name instead of __init__. +[clinic start generated code]*/ + +static int +depr_kwd_init_impl(PyObject *self, PyObject *a) +/*[clinic end generated code: output=6e02eb724a85d840 input=b9bf3c20f012d539]*/ +{ + return 0; +} + +static PyTypeObject DeprKwdInit = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.DeprKwdInit", + .tp_basicsize = sizeof(PyObject), + .tp_new = PyType_GenericNew, + .tp_init = depr_kwd_init, + .tp_flags = Py_TPFLAGS_DEFAULT, +}; + + +/*[clinic input] +class _testclinic.DeprKwdInitNoInline "PyObject *" "PyObject" +_testclinic.DeprKwdInitNoInline.__init__ as depr_kwd_init_noinline + a: object + / + b: object + c: object = None + / [from 3.14] + # Force to use _PyArg_ParseTupleAndKeywordsFast. + d: str(accept={str, robuffer}, zeroes=True) = '' +[clinic start generated code]*/ + +static int +depr_kwd_init_noinline_impl(PyObject *self, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length) +/*[clinic end generated code: output=27759d70ddd25873 input=c19d982c8c70a930]*/ +{ + return 0; +} + +static PyTypeObject DeprKwdInitNoInline = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "_testclinic.DeprKwdInitNoInline", + .tp_basicsize = sizeof(PyObject), + .tp_new = PyType_GenericNew, + .tp_init = depr_kwd_init_noinline, + .tp_flags = Py_TPFLAGS_DEFAULT, +}; + + /*[clinic input] depr_star_pos0_len1 * [from 3.14] @@ -1450,6 +1560,148 @@ depr_star_pos2_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, } +/*[clinic input] +depr_star_noinline + a: object + * [from 3.14] + b: object + c: object = None + * + # Force to use _PyArg_ParseStackAndKeywords. + d: str(accept={str, robuffer}, zeroes=True) = '' +[clinic start generated code]*/ + +static PyObject * +depr_star_noinline_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length) +/*[clinic end generated code: output=cc27dacf5c2754af input=d36cc862a2daef98]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_kwd_required_1 + a: object + / + b: object + / [from 3.14] +[clinic start generated code]*/ + +static PyObject * +depr_kwd_required_1_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=1d8ab19ea78418af input=53f2c398b828462d]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_kwd_required_2 + a: object + / + b: object + c: object + / [from 3.14] +[clinic start generated code]*/ + +static PyObject * +depr_kwd_required_2_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=44a89cb82509ddde input=a2b0ef37de8a01a7]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_kwd_optional_1 + a: object + / + b: object = None + / [from 3.14] +[clinic start generated code]*/ + +static PyObject * +depr_kwd_optional_1_impl(PyObject *module, PyObject *a, PyObject *b) +/*[clinic end generated code: output=a8a3d67efcc7b058 input=e416981eb78c3053]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_kwd_optional_2 + a: object + / + b: object = None + c: object = None + / [from 3.14] +[clinic start generated code]*/ + +static PyObject * +depr_kwd_optional_2_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=aa2d967f26fdb9f6 input=cae3afb783bfc855]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_kwd_optional_3 + a: object = None + b: object = None + c: object = None + / [from 3.14] +[clinic start generated code]*/ + +static PyObject * +depr_kwd_optional_3_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=a26025bf6118fd07 input=c9183b2f9ccaf992]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_kwd_required_optional + a: object + / + b: object + c: object = None + / [from 3.14] +[clinic start generated code]*/ + +static PyObject * +depr_kwd_required_optional_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c) +/*[clinic end generated code: output=e53a8b7a250d8ffc input=23237a046f8388f5]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_kwd_noinline + a: object + / + b: object + c: object = None + / [from 3.14] + # Force to use _PyArg_ParseStackAndKeywords. + d: str(accept={str, robuffer}, zeroes=True) = '' +[clinic start generated code]*/ + +static PyObject * +depr_kwd_noinline_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length) +/*[clinic end generated code: output=f59da8113f2bad7c input=1d6db65bebb069d7]*/ +{ + Py_RETURN_NONE; +} + // Reset PY_VERSION_HEX #undef PY_VERSION_HEX #define PY_VERSION_HEX _SAVED_PY_VERSION @@ -1526,6 +1778,14 @@ static PyMethodDef tester_methods[] = { DEPR_STAR_POS2_LEN1_METHODDEF DEPR_STAR_POS2_LEN2_METHODDEF DEPR_STAR_POS2_LEN2_WITH_KWD_METHODDEF + DEPR_STAR_NOINLINE_METHODDEF + DEPR_KWD_REQUIRED_1_METHODDEF + DEPR_KWD_REQUIRED_2_METHODDEF + DEPR_KWD_OPTIONAL_1_METHODDEF + DEPR_KWD_OPTIONAL_2_METHODDEF + DEPR_KWD_OPTIONAL_3_METHODDEF + DEPR_KWD_REQUIRED_OPTIONAL_METHODDEF + DEPR_KWD_NOINLINE_METHODDEF {NULL, NULL} }; @@ -1549,6 +1809,18 @@ PyInit__testclinic(void) if (PyModule_AddType(m, &DeprStarInit) < 0) { goto error; } + if (PyModule_AddType(m, &DeprStarInitNoInline) < 0) { + goto error; + } + if (PyModule_AddType(m, &DeprKwdNew) < 0) { + goto error; + } + if (PyModule_AddType(m, &DeprKwdInit) < 0) { + goto error; + } + if (PyModule_AddType(m, &DeprKwdInitNoInline) < 0) { + goto error; + } return m; error: diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h new file mode 100644 index 0000000000000..661fdaf2a0718 --- /dev/null +++ b/Modules/clinic/_testclinic_depr.c.h @@ -0,0 +1,2095 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(depr_star_new__doc__, +"DeprStarNew(a=None)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __new__.\n" +"\n" +"Note: Passing positional arguments to _testclinic.DeprStarNew() is\n" +"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +static PyObject * +depr_star_new_impl(PyTypeObject *type, PyObject *a); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprStarNew.__new__'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprStarNew.__new__'.") +# else +# warning "Update the clinic input of '_testclinic.DeprStarNew.__new__'." +# endif +#endif + +static PyObject * +depr_star_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "DeprStarNew", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *a = Py_None; + + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to _testclinic.DeprStarNew() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + a = fastargs[0]; +skip_optional_pos: + return_value = depr_star_new_impl(type, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_new_clone__doc__, +"cloned($self, /, a=None)\n" +"--\n" +"\n" +"Note: Passing positional arguments to _testclinic.DeprStarNew.cloned()\n" +"is deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_NEW_CLONE_METHODDEF \ + {"cloned", _PyCFunction_CAST(depr_star_new_clone), METH_FASTCALL|METH_KEYWORDS, depr_star_new_clone__doc__}, + +static PyObject * +depr_star_new_clone_impl(PyObject *type, PyObject *a); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprStarNew.cloned'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprStarNew.cloned'.") +# else +# warning "Update the clinic input of '_testclinic.DeprStarNew.cloned'." +# endif +#endif + +static PyObject * +depr_star_new_clone(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "cloned", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *a = Py_None; + + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to _testclinic.DeprStarNew.cloned()" + " is deprecated. Parameter 'a' will become a keyword-only " + "parameter in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + a = args[0]; +skip_optional_pos: + return_value = depr_star_new_clone_impl(type, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_init__doc__, +"DeprStarInit(a=None)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __init__.\n" +"\n" +"Note: Passing positional arguments to _testclinic.DeprStarInit() is\n" +"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +static int +depr_star_init_impl(PyObject *self, PyObject *a); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprStarInit.__init__'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprStarInit.__init__'.") +# else +# warning "Update the clinic input of '_testclinic.DeprStarInit.__init__'." +# endif +#endif + +static int +depr_star_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "DeprStarInit", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *a = Py_None; + + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to _testclinic.DeprStarInit() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + a = fastargs[0]; +skip_optional_pos: + return_value = depr_star_init_impl(self, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_init_clone__doc__, +"cloned($self, /, a=None)\n" +"--\n" +"\n" +"Note: Passing positional arguments to\n" +"_testclinic.DeprStarInit.cloned() is deprecated. Parameter \'a\' will\n" +"become a keyword-only parameter in Python 3.14.\n" +""); + +#define DEPR_STAR_INIT_CLONE_METHODDEF \ + {"cloned", _PyCFunction_CAST(depr_star_init_clone), METH_FASTCALL|METH_KEYWORDS, depr_star_init_clone__doc__}, + +static PyObject * +depr_star_init_clone_impl(PyObject *self, PyObject *a); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprStarInit.cloned'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprStarInit.cloned'.") +# else +# warning "Update the clinic input of '_testclinic.DeprStarInit.cloned'." +# endif +#endif + +static PyObject * +depr_star_init_clone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "cloned", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *a = Py_None; + + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to " + "_testclinic.DeprStarInit.cloned() is deprecated. Parameter 'a' " + "will become a keyword-only parameter in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + a = args[0]; +skip_optional_pos: + return_value = depr_star_init_clone_impl(self, a); + +exit: + return return_value; +} + +static int +depr_star_init_noinline_impl(PyObject *self, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprStarInitNoInline.__init__'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprStarInitNoInline.__init__'.") +# else +# warning "Update the clinic input of '_testclinic.DeprStarInitNoInline.__init__'." +# endif +#endif + +static int +depr_star_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "OO|O$s#:DeprStarInitNoInline", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + const char *d = ""; + Py_ssize_t d_length; + + if (nargs > 1 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to " + "_testclinic.DeprStarInitNoInline() is deprecated. Parameters 'b'" + " and 'c' will become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, + &a, &b, &c, &d, &d_length)) { + goto exit; + } + return_value = depr_star_init_noinline_impl(self, a, b, c, d, d_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_new__doc__, +"DeprKwdNew(a=None)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __new__.\n" +"\n" +"Note: Passing keyword argument \'a\' to _testclinic.DeprKwdNew() is\n" +"deprecated. Corresponding parameter will become positional-only in\n" +"Python 3.14.\n" +""); + +static PyObject * +depr_kwd_new_impl(PyTypeObject *type, PyObject *a); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprKwdNew.__new__'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprKwdNew.__new__'.") +# else +# warning "Update the clinic input of '_testclinic.DeprKwdNew.__new__'." +# endif +#endif + +static PyObject * +depr_kwd_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "DeprKwdNew", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *a = Py_None; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (kwargs && PyDict_GET_SIZE(kwargs) && nargs < 1 && fastargs[0]) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword argument 'a' to _testclinic.DeprKwdNew() is " + "deprecated. Corresponding parameter will become positional-only " + "in Python 3.14.", 1)) + { + goto exit; + } + } + if (!noptargs) { + goto skip_optional_pos; + } + a = fastargs[0]; +skip_optional_pos: + return_value = depr_kwd_new_impl(type, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_init__doc__, +"DeprKwdInit(a=None)\n" +"--\n" +"\n" +"The deprecation message should use the class name instead of __init__.\n" +"\n" +"Note: Passing keyword argument \'a\' to _testclinic.DeprKwdInit() is\n" +"deprecated. Corresponding parameter will become positional-only in\n" +"Python 3.14.\n" +""); + +static int +depr_kwd_init_impl(PyObject *self, PyObject *a); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprKwdInit.__init__'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprKwdInit.__init__'.") +# else +# warning "Update the clinic input of '_testclinic.DeprKwdInit.__init__'." +# endif +#endif + +static int +depr_kwd_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "DeprKwdInit", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + PyObject *a = Py_None; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 0, 1, 0, argsbuf); + if (!fastargs) { + goto exit; + } + if (kwargs && PyDict_GET_SIZE(kwargs) && nargs < 1 && fastargs[0]) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword argument 'a' to _testclinic.DeprKwdInit() is " + "deprecated. Corresponding parameter will become positional-only " + "in Python 3.14.", 1)) + { + goto exit; + } + } + if (!noptargs) { + goto skip_optional_pos; + } + a = fastargs[0]; +skip_optional_pos: + return_value = depr_kwd_init_impl(self, a); + +exit: + return return_value; +} + +static int +depr_kwd_init_noinline_impl(PyObject *self, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of '_testclinic.DeprKwdInitNoInline.__init__'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of '_testclinic.DeprKwdInitNoInline.__init__'.") +# else +# warning "Update the clinic input of '_testclinic.DeprKwdInitNoInline.__init__'." +# endif +#endif + +static int +depr_kwd_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "OO|Os#:DeprKwdInitNoInline", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + const char *d = ""; + Py_ssize_t d_length; + + if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, + &a, &b, &c, &d, &d_length)) { + goto exit; + } + if (kwargs && PyDict_GET_SIZE(kwargs) && ((nargs < 2) || (nargs < 3 && PyDict_Contains(kwargs, &_Py_ID(c))))) { + if (PyErr_Occurred()) { // PyDict_Contains() above can fail + goto exit; + } + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'b' and 'c' to " + "_testclinic.DeprKwdInitNoInline() is deprecated. Corresponding " + "parameters will become positional-only in Python 3.14.", 1)) + { + goto exit; + } + } + return_value = depr_kwd_init_noinline_impl(self, a, b, c, d, d_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos0_len1__doc__, +"depr_star_pos0_len1($module, /, a)\n" +"--\n" +"\n" +"Note: Passing positional arguments to depr_star_pos0_len1() is\n" +"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS0_LEN1_METHODDEF \ + {"depr_star_pos0_len1", _PyCFunction_CAST(depr_star_pos0_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len1__doc__}, + +static PyObject * +depr_star_pos0_len1_impl(PyObject *module, PyObject *a); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos0_len1'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos0_len1'.") +# else +# warning "Update the clinic input of 'depr_star_pos0_len1'." +# endif +#endif + +static PyObject * +depr_star_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos0_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *a; + + if (nargs == 1) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to depr_star_pos0_len1() is " + "deprecated. Parameter 'a' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + return_value = depr_star_pos0_len1_impl(module, a); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos0_len2__doc__, +"depr_star_pos0_len2($module, /, a, b)\n" +"--\n" +"\n" +"Note: Passing positional arguments to depr_star_pos0_len2() is\n" +"deprecated. Parameters \'a\' and \'b\' will become keyword-only parameters\n" +"in Python 3.14.\n" +""); + +#define DEPR_STAR_POS0_LEN2_METHODDEF \ + {"depr_star_pos0_len2", _PyCFunction_CAST(depr_star_pos0_len2), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len2__doc__}, + +static PyObject * +depr_star_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos0_len2'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos0_len2'.") +# else +# warning "Update the clinic input of 'depr_star_pos0_len2'." +# endif +#endif + +static PyObject * +depr_star_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos0_len2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + if (nargs > 0 && nargs <= 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to depr_star_pos0_len2() is " + "deprecated. Parameters 'a' and 'b' will become keyword-only " + "parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = depr_star_pos0_len2_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos0_len3_with_kwd__doc__, +"depr_star_pos0_len3_with_kwd($module, /, a, b, c, *, d)\n" +"--\n" +"\n" +"Note: Passing positional arguments to depr_star_pos0_len3_with_kwd()\n" +"is deprecated. Parameters \'a\', \'b\' and \'c\' will become keyword-only\n" +"parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS0_LEN3_WITH_KWD_METHODDEF \ + {"depr_star_pos0_len3_with_kwd", _PyCFunction_CAST(depr_star_pos0_len3_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len3_with_kwd__doc__}, + +static PyObject * +depr_star_pos0_len3_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos0_len3_with_kwd'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos0_len3_with_kwd'.") +# else +# warning "Update the clinic input of 'depr_star_pos0_len3_with_kwd'." +# endif +#endif + +static PyObject * +depr_star_pos0_len3_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos0_len3_with_kwd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + if (nargs > 0 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing positional arguments to depr_star_pos0_len3_with_kwd() " + "is deprecated. Parameters 'a', 'b' and 'c' will become " + "keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = depr_star_pos0_len3_with_kwd_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos1_len1_opt__doc__, +"depr_star_pos1_len1_opt($module, /, a, b=None)\n" +"--\n" +"\n" +"Note: Passing 2 positional arguments to depr_star_pos1_len1_opt() is\n" +"deprecated. Parameter \'b\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS1_LEN1_OPT_METHODDEF \ + {"depr_star_pos1_len1_opt", _PyCFunction_CAST(depr_star_pos1_len1_opt), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len1_opt__doc__}, + +static PyObject * +depr_star_pos1_len1_opt_impl(PyObject *module, PyObject *a, PyObject *b); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos1_len1_opt'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos1_len1_opt'.") +# else +# warning "Update the clinic input of 'depr_star_pos1_len1_opt'." +# endif +#endif + +static PyObject * +depr_star_pos1_len1_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos1_len1_opt", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + + if (nargs == 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 2 positional arguments to depr_star_pos1_len1_opt() is " + "deprecated. Parameter 'b' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + b = args[1]; +skip_optional_pos: + return_value = depr_star_pos1_len1_opt_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos1_len1__doc__, +"depr_star_pos1_len1($module, /, a, b)\n" +"--\n" +"\n" +"Note: Passing 2 positional arguments to depr_star_pos1_len1() is\n" +"deprecated. Parameter \'b\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS1_LEN1_METHODDEF \ + {"depr_star_pos1_len1", _PyCFunction_CAST(depr_star_pos1_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len1__doc__}, + +static PyObject * +depr_star_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos1_len1'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos1_len1'.") +# else +# warning "Update the clinic input of 'depr_star_pos1_len1'." +# endif +#endif + +static PyObject * +depr_star_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos1_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + if (nargs == 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 2 positional arguments to depr_star_pos1_len1() is " + "deprecated. Parameter 'b' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + return_value = depr_star_pos1_len1_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos1_len2_with_kwd__doc__, +"depr_star_pos1_len2_with_kwd($module, /, a, b, c, *, d)\n" +"--\n" +"\n" +"Note: Passing more than 1 positional argument to\n" +"depr_star_pos1_len2_with_kwd() is deprecated. Parameters \'b\' and \'c\'\n" +"will become keyword-only parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS1_LEN2_WITH_KWD_METHODDEF \ + {"depr_star_pos1_len2_with_kwd", _PyCFunction_CAST(depr_star_pos1_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len2_with_kwd__doc__}, + +static PyObject * +depr_star_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos1_len2_with_kwd'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos1_len2_with_kwd'.") +# else +# warning "Update the clinic input of 'depr_star_pos1_len2_with_kwd'." +# endif +#endif + +static PyObject * +depr_star_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos1_len2_with_kwd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + if (nargs > 1 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to " + "depr_star_pos1_len2_with_kwd() is deprecated. Parameters 'b' and" + " 'c' will become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = depr_star_pos1_len2_with_kwd_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos2_len1__doc__, +"depr_star_pos2_len1($module, /, a, b, c)\n" +"--\n" +"\n" +"Note: Passing 3 positional arguments to depr_star_pos2_len1() is\n" +"deprecated. Parameter \'c\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_STAR_POS2_LEN1_METHODDEF \ + {"depr_star_pos2_len1", _PyCFunction_CAST(depr_star_pos2_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len1__doc__}, + +static PyObject * +depr_star_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos2_len1'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos2_len1'.") +# else +# warning "Update the clinic input of 'depr_star_pos2_len1'." +# endif +#endif + +static PyObject * +depr_star_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos2_len1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *c; + + if (nargs == 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing 3 positional arguments to depr_star_pos2_len1() is " + "deprecated. Parameter 'c' will become a keyword-only parameter " + "in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + return_value = depr_star_pos2_len1_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos2_len2__doc__, +"depr_star_pos2_len2($module, /, a, b, c, d)\n" +"--\n" +"\n" +"Note: Passing more than 2 positional arguments to\n" +"depr_star_pos2_len2() is deprecated. Parameters \'c\' and \'d\' will\n" +"become keyword-only parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS2_LEN2_METHODDEF \ + {"depr_star_pos2_len2", _PyCFunction_CAST(depr_star_pos2_len2), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len2__doc__}, + +static PyObject * +depr_star_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos2_len2'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos2_len2'.") +# else +# warning "Update the clinic input of 'depr_star_pos2_len2'." +# endif +#endif + +static PyObject * +depr_star_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos2_len2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + + if (nargs > 2 && nargs <= 4) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 2 positional arguments to " + "depr_star_pos2_len2() is deprecated. Parameters 'c' and 'd' will" + " become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + return_value = depr_star_pos2_len2_impl(module, a, b, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_pos2_len2_with_kwd__doc__, +"depr_star_pos2_len2_with_kwd($module, /, a, b, c, d, *, e)\n" +"--\n" +"\n" +"Note: Passing more than 2 positional arguments to\n" +"depr_star_pos2_len2_with_kwd() is deprecated. Parameters \'c\' and \'d\'\n" +"will become keyword-only parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_POS2_LEN2_WITH_KWD_METHODDEF \ + {"depr_star_pos2_len2_with_kwd", _PyCFunction_CAST(depr_star_pos2_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len2_with_kwd__doc__}, + +static PyObject * +depr_star_pos2_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, PyObject *d, PyObject *e); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_pos2_len2_with_kwd'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_pos2_len2_with_kwd'.") +# else +# warning "Update the clinic input of 'depr_star_pos2_len2_with_kwd'." +# endif +#endif + +static PyObject * +depr_star_pos2_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 5 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", "e", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_pos2_len2_with_kwd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[5]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + PyObject *e; + + if (nargs > 2 && nargs <= 4) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 2 positional arguments to " + "depr_star_pos2_len2_with_kwd() is deprecated. Parameters 'c' and" + " 'd' will become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + e = args[4]; + return_value = depr_star_pos2_len2_with_kwd_impl(module, a, b, c, d, e); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_star_noinline__doc__, +"depr_star_noinline($module, /, a, b, c=None, *, d=\'\')\n" +"--\n" +"\n" +"Note: Passing more than 1 positional argument to depr_star_noinline()\n" +"is deprecated. Parameters \'b\' and \'c\' will become keyword-only\n" +"parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_NOINLINE_METHODDEF \ + {"depr_star_noinline", _PyCFunction_CAST(depr_star_noinline), METH_FASTCALL|METH_KEYWORDS, depr_star_noinline__doc__}, + +static PyObject * +depr_star_noinline_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_noinline'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_noinline'.") +# else +# warning "Update the clinic input of 'depr_star_noinline'." +# endif +#endif + +static PyObject * +depr_star_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "OO|O$s#:depr_star_noinline", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + const char *d = ""; + Py_ssize_t d_length; + + if (nargs > 1 && nargs <= 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to depr_star_noinline() " + "is deprecated. Parameters 'b' and 'c' will become keyword-only " + "parameters in Python 3.14.", 1)) + { + goto exit; + } + } + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &a, &b, &c, &d, &d_length)) { + goto exit; + } + return_value = depr_star_noinline_impl(module, a, b, c, d, d_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_required_1__doc__, +"depr_kwd_required_1($module, a, /, b)\n" +"--\n" +"\n" +"Note: Passing keyword argument \'b\' to depr_kwd_required_1() is\n" +"deprecated. Corresponding parameter will become positional-only in\n" +"Python 3.14.\n" +""); + +#define DEPR_KWD_REQUIRED_1_METHODDEF \ + {"depr_kwd_required_1", _PyCFunction_CAST(depr_kwd_required_1), METH_FASTCALL|METH_KEYWORDS, depr_kwd_required_1__doc__}, + +static PyObject * +depr_kwd_required_1_impl(PyObject *module, PyObject *a, PyObject *b); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_required_1'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_required_1'.") +# else +# warning "Update the clinic input of 'depr_kwd_required_1'." +# endif +#endif + +static PyObject * +depr_kwd_required_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_kwd_required_1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *a; + PyObject *b; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + if (nargs < 2) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword argument 'b' to depr_kwd_required_1() is " + "deprecated. Corresponding parameter will become positional-only " + "in Python 3.14.", 1)) + { + goto exit; + } + } + a = args[0]; + b = args[1]; + return_value = depr_kwd_required_1_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_required_2__doc__, +"depr_kwd_required_2($module, a, /, b, c)\n" +"--\n" +"\n" +"Note: Passing keyword arguments \'b\' and \'c\' to depr_kwd_required_2()\n" +"is deprecated. Corresponding parameters will become positional-only in\n" +"Python 3.14.\n" +""); + +#define DEPR_KWD_REQUIRED_2_METHODDEF \ + {"depr_kwd_required_2", _PyCFunction_CAST(depr_kwd_required_2), METH_FASTCALL|METH_KEYWORDS, depr_kwd_required_2__doc__}, + +static PyObject * +depr_kwd_required_2_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_required_2'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_required_2'.") +# else +# warning "Update the clinic input of 'depr_kwd_required_2'." +# endif +#endif + +static PyObject * +depr_kwd_required_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_kwd_required_2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *c; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (nargs < 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'b' and 'c' to depr_kwd_required_2() " + "is deprecated. Corresponding parameters will become " + "positional-only in Python 3.14.", 1)) + { + goto exit; + } + } + a = args[0]; + b = args[1]; + c = args[2]; + return_value = depr_kwd_required_2_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_optional_1__doc__, +"depr_kwd_optional_1($module, a, /, b=None)\n" +"--\n" +"\n" +"Note: Passing keyword argument \'b\' to depr_kwd_optional_1() is\n" +"deprecated. Corresponding parameter will become positional-only in\n" +"Python 3.14.\n" +""); + +#define DEPR_KWD_OPTIONAL_1_METHODDEF \ + {"depr_kwd_optional_1", _PyCFunction_CAST(depr_kwd_optional_1), METH_FASTCALL|METH_KEYWORDS, depr_kwd_optional_1__doc__}, + +static PyObject * +depr_kwd_optional_1_impl(PyObject *module, PyObject *a, PyObject *b); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_optional_1'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_optional_1'.") +# else +# warning "Update the clinic input of 'depr_kwd_optional_1'." +# endif +#endif + +static PyObject * +depr_kwd_optional_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_kwd_optional_1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + if (kwnames && PyTuple_GET_SIZE(kwnames) && nargs < 2 && args[1]) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword argument 'b' to depr_kwd_optional_1() is " + "deprecated. Corresponding parameter will become positional-only " + "in Python 3.14.", 1)) + { + goto exit; + } + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + b = args[1]; +skip_optional_pos: + return_value = depr_kwd_optional_1_impl(module, a, b); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_optional_2__doc__, +"depr_kwd_optional_2($module, a, /, b=None, c=None)\n" +"--\n" +"\n" +"Note: Passing keyword arguments \'b\' and \'c\' to depr_kwd_optional_2()\n" +"is deprecated. Corresponding parameters will become positional-only in\n" +"Python 3.14.\n" +""); + +#define DEPR_KWD_OPTIONAL_2_METHODDEF \ + {"depr_kwd_optional_2", _PyCFunction_CAST(depr_kwd_optional_2), METH_FASTCALL|METH_KEYWORDS, depr_kwd_optional_2__doc__}, + +static PyObject * +depr_kwd_optional_2_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_optional_2'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_optional_2'.") +# else +# warning "Update the clinic input of 'depr_kwd_optional_2'." +# endif +#endif + +static PyObject * +depr_kwd_optional_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_kwd_optional_2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *b = Py_None; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2 && args[1]) || (nargs < 3 && args[2]))) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'b' and 'c' to depr_kwd_optional_2() " + "is deprecated. Corresponding parameters will become " + "positional-only in Python 3.14.", 1)) + { + goto exit; + } + } + a = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + c = args[2]; +skip_optional_pos: + return_value = depr_kwd_optional_2_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_optional_3__doc__, +"depr_kwd_optional_3($module, /, a=None, b=None, c=None)\n" +"--\n" +"\n" +"Note: Passing keyword arguments \'a\', \'b\' and \'c\' to\n" +"depr_kwd_optional_3() is deprecated. Corresponding parameters will\n" +"become positional-only in Python 3.14.\n" +""); + +#define DEPR_KWD_OPTIONAL_3_METHODDEF \ + {"depr_kwd_optional_3", _PyCFunction_CAST(depr_kwd_optional_3), METH_FASTCALL|METH_KEYWORDS, depr_kwd_optional_3__doc__}, + +static PyObject * +depr_kwd_optional_3_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_optional_3'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_optional_3'.") +# else +# warning "Update the clinic input of 'depr_kwd_optional_3'." +# endif +#endif + +static PyObject * +depr_kwd_optional_3(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_kwd_optional_3", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *a = Py_None; + PyObject *b = Py_None; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 1 && args[0]) || (nargs < 2 && args[1]) || (nargs < 3 && args[2]))) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'a', 'b' and 'c' to " + "depr_kwd_optional_3() is deprecated. Corresponding parameters " + "will become positional-only in Python 3.14.", 1)) + { + goto exit; + } + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + a = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + b = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + c = args[2]; +skip_optional_pos: + return_value = depr_kwd_optional_3_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_required_optional__doc__, +"depr_kwd_required_optional($module, a, /, b, c=None)\n" +"--\n" +"\n" +"Note: Passing keyword arguments \'b\' and \'c\' to\n" +"depr_kwd_required_optional() is deprecated. Corresponding parameters\n" +"will become positional-only in Python 3.14.\n" +""); + +#define DEPR_KWD_REQUIRED_OPTIONAL_METHODDEF \ + {"depr_kwd_required_optional", _PyCFunction_CAST(depr_kwd_required_optional), METH_FASTCALL|METH_KEYWORDS, depr_kwd_required_optional__doc__}, + +static PyObject * +depr_kwd_required_optional_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_required_optional'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_required_optional'.") +# else +# warning "Update the clinic input of 'depr_kwd_required_optional'." +# endif +#endif + +static PyObject * +depr_kwd_required_optional(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), &_Py_ID(c), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", "c", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_kwd_required_optional", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2) || (nargs < 3 && args[2]))) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'b' and 'c' to " + "depr_kwd_required_optional() is deprecated. Corresponding " + "parameters will become positional-only in Python 3.14.", 1)) + { + goto exit; + } + } + a = args[0]; + b = args[1]; + if (!noptargs) { + goto skip_optional_pos; + } + c = args[2]; +skip_optional_pos: + return_value = depr_kwd_required_optional_impl(module, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_kwd_noinline__doc__, +"depr_kwd_noinline($module, a, /, b, c=None, d=\'\')\n" +"--\n" +"\n" +"Note: Passing keyword arguments \'b\' and \'c\' to depr_kwd_noinline() is\n" +"deprecated. Corresponding parameters will become positional-only in\n" +"Python 3.14.\n" +""); + +#define DEPR_KWD_NOINLINE_METHODDEF \ + {"depr_kwd_noinline", _PyCFunction_CAST(depr_kwd_noinline), METH_FASTCALL|METH_KEYWORDS, depr_kwd_noinline__doc__}, + +static PyObject * +depr_kwd_noinline_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *c, const char *d, Py_ssize_t d_length); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_noinline'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_noinline'.") +# else +# warning "Update the clinic input of 'depr_kwd_noinline'." +# endif +#endif + +static PyObject * +depr_kwd_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", "c", "d", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .format = "OO|Os#:depr_kwd_noinline", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *a; + PyObject *b; + PyObject *c = Py_None; + const char *d = ""; + Py_ssize_t d_length; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &a, &b, &c, &d, &d_length)) { + goto exit; + } + if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2) || (nargs < 3 && PySequence_Contains(kwnames, &_Py_ID(c))))) { + if (PyErr_Occurred()) { // PySequence_Contains() above can fail + goto exit; + } + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'b' and 'c' to depr_kwd_noinline() is " + "deprecated. Corresponding parameters will become positional-only" + " in Python 3.14.", 1)) + { + goto exit; + } + } + return_value = depr_kwd_noinline_impl(module, a, b, c, d, d_length); + +exit: + return return_value; +} +/*[clinic end generated code: output=fc558c1efdcab076 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_depr_star.c.h b/Modules/clinic/_testclinic_depr_star.c.h deleted file mode 100644 index 1aa42dd405977..0000000000000 --- a/Modules/clinic/_testclinic_depr_star.c.h +++ /dev/null @@ -1,1140 +0,0 @@ -/*[clinic input] -preserve -[clinic start generated code]*/ - -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif - - -PyDoc_STRVAR(depr_star_new__doc__, -"DeprStarNew(a)\n" -"--\n" -"\n" -"The deprecation message should use the class name instead of __new__.\n" -"\n" -"Note: Passing positional arguments to _testclinic.DeprStarNew() is\n" -"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" -"Python 3.14.\n" -""); - -static PyObject * -depr_star_new_impl(PyTypeObject *type, PyObject *a); - -static PyObject * -depr_star_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "DeprStarNew", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject * const *fastargs; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarNew.__new__' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarNew.__new__' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarNew.__new__' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to _testclinic.DeprStarNew() is " - "deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.14.", 1)) - { - goto exit; - } - } - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); - if (!fastargs) { - goto exit; - } - a = fastargs[0]; - return_value = depr_star_new_impl(type, a); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_new_clone__doc__, -"cloned($self, /, a)\n" -"--\n" -"\n" -"Note: Passing positional arguments to _testclinic.DeprStarNew.cloned()\n" -"is deprecated. Parameter \'a\' will become a keyword-only parameter in\n" -"Python 3.14.\n" -""); - -#define DEPR_STAR_NEW_CLONE_METHODDEF \ - {"cloned", _PyCFunction_CAST(depr_star_new_clone), METH_FASTCALL|METH_KEYWORDS, depr_star_new_clone__doc__}, - -static PyObject * -depr_star_new_clone_impl(PyObject *type, PyObject *a); - -static PyObject * -depr_star_new_clone(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "cloned", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarNew.cloned' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarNew.cloned' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarNew.cloned' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to _testclinic.DeprStarNew.cloned()" - " is deprecated. Parameter 'a' will become a keyword-only " - "parameter in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - return_value = depr_star_new_clone_impl(type, a); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_init__doc__, -"DeprStarInit(a)\n" -"--\n" -"\n" -"The deprecation message should use the class name instead of __init__.\n" -"\n" -"Note: Passing positional arguments to _testclinic.DeprStarInit() is\n" -"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" -"Python 3.14.\n" -""); - -static int -depr_star_init_impl(PyObject *self, PyObject *a); - -static int -depr_star_init(PyObject *self, PyObject *args, PyObject *kwargs) -{ - int return_value = -1; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "DeprStarInit", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject * const *fastargs; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarInit.__init__' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarInit.__init__' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarInit.__init__' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to _testclinic.DeprStarInit() is " - "deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.14.", 1)) - { - goto exit; - } - } - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); - if (!fastargs) { - goto exit; - } - a = fastargs[0]; - return_value = depr_star_init_impl(self, a); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_init_clone__doc__, -"cloned($self, /, a)\n" -"--\n" -"\n" -"Note: Passing positional arguments to\n" -"_testclinic.DeprStarInit.cloned() is deprecated. Parameter \'a\' will\n" -"become a keyword-only parameter in Python 3.14.\n" -""); - -#define DEPR_STAR_INIT_CLONE_METHODDEF \ - {"cloned", _PyCFunction_CAST(depr_star_init_clone), METH_FASTCALL|METH_KEYWORDS, depr_star_init_clone__doc__}, - -static PyObject * -depr_star_init_clone_impl(PyObject *self, PyObject *a); - -static PyObject * -depr_star_init_clone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "cloned", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarInit.cloned' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarInit.cloned' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " '_testclinic.DeprStarInit.cloned' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to " - "_testclinic.DeprStarInit.cloned() is deprecated. Parameter 'a' " - "will become a keyword-only parameter in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - return_value = depr_star_init_clone_impl(self, a); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos0_len1__doc__, -"depr_star_pos0_len1($module, /, a)\n" -"--\n" -"\n" -"Note: Passing positional arguments to depr_star_pos0_len1() is\n" -"deprecated. Parameter \'a\' will become a keyword-only parameter in\n" -"Python 3.14.\n" -""); - -#define DEPR_STAR_POS0_LEN1_METHODDEF \ - {"depr_star_pos0_len1", _PyCFunction_CAST(depr_star_pos0_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len1__doc__}, - -static PyObject * -depr_star_pos0_len1_impl(PyObject *module, PyObject *a); - -static PyObject * -depr_star_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos0_len1", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; - PyObject *a; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " 'depr_star_pos0_len1' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " 'depr_star_pos0_len1' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'a' in the clinic input of" \ - " 'depr_star_pos0_len1' to be keyword-only." - # endif - #endif - if (nargs == 1) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to depr_star_pos0_len1() is " - "deprecated. Parameter 'a' will become a keyword-only parameter " - "in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - return_value = depr_star_pos0_len1_impl(module, a); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos0_len2__doc__, -"depr_star_pos0_len2($module, /, a, b)\n" -"--\n" -"\n" -"Note: Passing positional arguments to depr_star_pos0_len2() is\n" -"deprecated. Parameters \'a\' and \'b\' will become keyword-only parameters\n" -"in Python 3.14.\n" -""); - -#define DEPR_STAR_POS0_LEN2_METHODDEF \ - {"depr_star_pos0_len2", _PyCFunction_CAST(depr_star_pos0_len2), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len2__doc__}, - -static PyObject * -depr_star_pos0_len2_impl(PyObject *module, PyObject *a, PyObject *b); - -static PyObject * -depr_star_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos0_len2", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - PyObject *a; - PyObject *b; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'a' and 'b' in the clinic " \ - "input of 'depr_star_pos0_len2' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'a' and 'b' in the clinic " \ - "input of 'depr_star_pos0_len2' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'a' and 'b' in the clinic " \ - "input of 'depr_star_pos0_len2' to be keyword-only." - # endif - #endif - if (nargs > 0 && nargs <= 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to depr_star_pos0_len2() is " - "deprecated. Parameters 'a' and 'b' will become keyword-only " - "parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - return_value = depr_star_pos0_len2_impl(module, a, b); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos0_len3_with_kwd__doc__, -"depr_star_pos0_len3_with_kwd($module, /, a, b, c, *, d)\n" -"--\n" -"\n" -"Note: Passing positional arguments to depr_star_pos0_len3_with_kwd()\n" -"is deprecated. Parameters \'a\', \'b\' and \'c\' will become keyword-only\n" -"parameters in Python 3.14.\n" -""); - -#define DEPR_STAR_POS0_LEN3_WITH_KWD_METHODDEF \ - {"depr_star_pos0_len3_with_kwd", _PyCFunction_CAST(depr_star_pos0_len3_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos0_len3_with_kwd__doc__}, - -static PyObject * -depr_star_pos0_len3_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, - PyObject *c, PyObject *d); - -static PyObject * -depr_star_pos0_len3_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 4 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos0_len3_with_kwd", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[4]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *d; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'a', 'b' and 'c' in the " \ - "clinic input of 'depr_star_pos0_len3_with_kwd' to be " \ - "keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'a', 'b' and 'c' in the " \ - "clinic input of 'depr_star_pos0_len3_with_kwd' to be " \ - "keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'a', 'b' and 'c' in the " \ - "clinic input of 'depr_star_pos0_len3_with_kwd' to be " \ - "keyword-only." - # endif - #endif - if (nargs > 0 && nargs <= 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing positional arguments to depr_star_pos0_len3_with_kwd() " - "is deprecated. Parameters 'a', 'b' and 'c' will become " - "keyword-only parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - d = args[3]; - return_value = depr_star_pos0_len3_with_kwd_impl(module, a, b, c, d); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos1_len1_opt__doc__, -"depr_star_pos1_len1_opt($module, /, a, b=None)\n" -"--\n" -"\n" -"Note: Passing 2 positional arguments to depr_star_pos1_len1_opt() is\n" -"deprecated. Parameter \'b\' will become a keyword-only parameter in\n" -"Python 3.14.\n" -""); - -#define DEPR_STAR_POS1_LEN1_OPT_METHODDEF \ - {"depr_star_pos1_len1_opt", _PyCFunction_CAST(depr_star_pos1_len1_opt), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len1_opt__doc__}, - -static PyObject * -depr_star_pos1_len1_opt_impl(PyObject *module, PyObject *a, PyObject *b); - -static PyObject * -depr_star_pos1_len1_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos1_len1_opt", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - PyObject *a; - PyObject *b = Py_None; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ - " 'depr_star_pos1_len1_opt' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ - " 'depr_star_pos1_len1_opt' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ - " 'depr_star_pos1_len1_opt' to be keyword-only." - # endif - #endif - if (nargs == 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing 2 positional arguments to depr_star_pos1_len1_opt() is " - "deprecated. Parameter 'b' will become a keyword-only parameter " - "in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - if (!noptargs) { - goto skip_optional_pos; - } - b = args[1]; -skip_optional_pos: - return_value = depr_star_pos1_len1_opt_impl(module, a, b); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos1_len1__doc__, -"depr_star_pos1_len1($module, /, a, b)\n" -"--\n" -"\n" -"Note: Passing 2 positional arguments to depr_star_pos1_len1() is\n" -"deprecated. Parameter \'b\' will become a keyword-only parameter in\n" -"Python 3.14.\n" -""); - -#define DEPR_STAR_POS1_LEN1_METHODDEF \ - {"depr_star_pos1_len1", _PyCFunction_CAST(depr_star_pos1_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len1__doc__}, - -static PyObject * -depr_star_pos1_len1_impl(PyObject *module, PyObject *a, PyObject *b); - -static PyObject * -depr_star_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos1_len1", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - PyObject *a; - PyObject *b; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ - " 'depr_star_pos1_len1' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ - " 'depr_star_pos1_len1' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'b' in the clinic input of" \ - " 'depr_star_pos1_len1' to be keyword-only." - # endif - #endif - if (nargs == 2) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing 2 positional arguments to depr_star_pos1_len1() is " - "deprecated. Parameter 'b' will become a keyword-only parameter " - "in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - return_value = depr_star_pos1_len1_impl(module, a, b); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos1_len2_with_kwd__doc__, -"depr_star_pos1_len2_with_kwd($module, /, a, b, c, *, d)\n" -"--\n" -"\n" -"Note: Passing more than 1 positional argument to\n" -"depr_star_pos1_len2_with_kwd() is deprecated. Parameters \'b\' and \'c\'\n" -"will become keyword-only parameters in Python 3.14.\n" -""); - -#define DEPR_STAR_POS1_LEN2_WITH_KWD_METHODDEF \ - {"depr_star_pos1_len2_with_kwd", _PyCFunction_CAST(depr_star_pos1_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos1_len2_with_kwd__doc__}, - -static PyObject * -depr_star_pos1_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, - PyObject *c, PyObject *d); - -static PyObject * -depr_star_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 4 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos1_len2_with_kwd", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[4]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *d; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'b' and 'c' in the clinic " \ - "input of 'depr_star_pos1_len2_with_kwd' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'b' and 'c' in the clinic " \ - "input of 'depr_star_pos1_len2_with_kwd' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'b' and 'c' in the clinic " \ - "input of 'depr_star_pos1_len2_with_kwd' to be keyword-only." - # endif - #endif - if (nargs > 1 && nargs <= 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 1 positional argument to " - "depr_star_pos1_len2_with_kwd() is deprecated. Parameters 'b' and" - " 'c' will become keyword-only parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 1, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - d = args[3]; - return_value = depr_star_pos1_len2_with_kwd_impl(module, a, b, c, d); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos2_len1__doc__, -"depr_star_pos2_len1($module, /, a, b, c)\n" -"--\n" -"\n" -"Note: Passing 3 positional arguments to depr_star_pos2_len1() is\n" -"deprecated. Parameter \'c\' will become a keyword-only parameter in\n" -"Python 3.14.\n" -""); - -#define DEPR_STAR_POS2_LEN1_METHODDEF \ - {"depr_star_pos2_len1", _PyCFunction_CAST(depr_star_pos2_len1), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len1__doc__}, - -static PyObject * -depr_star_pos2_len1_impl(PyObject *module, PyObject *a, PyObject *b, - PyObject *c); - -static PyObject * -depr_star_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 3 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos2_len1", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[3]; - PyObject *a; - PyObject *b; - PyObject *c; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'c' in the clinic input of" \ - " 'depr_star_pos2_len1' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'c' in the clinic input of" \ - " 'depr_star_pos2_len1' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'c' in the clinic input of" \ - " 'depr_star_pos2_len1' to be keyword-only." - # endif - #endif - if (nargs == 3) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing 3 positional arguments to depr_star_pos2_len1() is " - "deprecated. Parameter 'c' will become a keyword-only parameter " - "in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - return_value = depr_star_pos2_len1_impl(module, a, b, c); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos2_len2__doc__, -"depr_star_pos2_len2($module, /, a, b, c, d)\n" -"--\n" -"\n" -"Note: Passing more than 2 positional arguments to\n" -"depr_star_pos2_len2() is deprecated. Parameters \'c\' and \'d\' will\n" -"become keyword-only parameters in Python 3.14.\n" -""); - -#define DEPR_STAR_POS2_LEN2_METHODDEF \ - {"depr_star_pos2_len2", _PyCFunction_CAST(depr_star_pos2_len2), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len2__doc__}, - -static PyObject * -depr_star_pos2_len2_impl(PyObject *module, PyObject *a, PyObject *b, - PyObject *c, PyObject *d); - -static PyObject * -depr_star_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 4 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "d", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos2_len2", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[4]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *d; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'depr_star_pos2_len2' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'depr_star_pos2_len2' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'depr_star_pos2_len2' to be keyword-only." - # endif - #endif - if (nargs > 2 && nargs <= 4) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 2 positional arguments to " - "depr_star_pos2_len2() is deprecated. Parameters 'c' and 'd' will" - " become keyword-only parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 0, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - d = args[3]; - return_value = depr_star_pos2_len2_impl(module, a, b, c, d); - -exit: - return return_value; -} - -PyDoc_STRVAR(depr_star_pos2_len2_with_kwd__doc__, -"depr_star_pos2_len2_with_kwd($module, /, a, b, c, d, *, e)\n" -"--\n" -"\n" -"Note: Passing more than 2 positional arguments to\n" -"depr_star_pos2_len2_with_kwd() is deprecated. Parameters \'c\' and \'d\'\n" -"will become keyword-only parameters in Python 3.14.\n" -""); - -#define DEPR_STAR_POS2_LEN2_WITH_KWD_METHODDEF \ - {"depr_star_pos2_len2_with_kwd", _PyCFunction_CAST(depr_star_pos2_len2_with_kwd), METH_FASTCALL|METH_KEYWORDS, depr_star_pos2_len2_with_kwd__doc__}, - -static PyObject * -depr_star_pos2_len2_with_kwd_impl(PyObject *module, PyObject *a, PyObject *b, - PyObject *c, PyObject *d, PyObject *e); - -static PyObject * -depr_star_pos2_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 5 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"a", "b", "c", "d", "e", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "depr_star_pos2_len2_with_kwd", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[5]; - PyObject *a; - PyObject *b; - PyObject *c; - PyObject *d; - PyObject *e; - - // Emit compiler warnings when we get to Python 3.14. - #if PY_VERSION_HEX >= 0x030e00C0 - # error \ - "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'depr_star_pos2_len2_with_kwd' to be keyword-only." - #elif PY_VERSION_HEX >= 0x030e00A0 - # ifdef _MSC_VER - # pragma message ( \ - "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'depr_star_pos2_len2_with_kwd' to be keyword-only.") - # else - # warning \ - "In _testclinic.c, update parameter(s) 'c' and 'd' in the clinic " \ - "input of 'depr_star_pos2_len2_with_kwd' to be keyword-only." - # endif - #endif - if (nargs > 2 && nargs <= 4) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Passing more than 2 positional arguments to " - "depr_star_pos2_len2_with_kwd() is deprecated. Parameters 'c' and" - " 'd' will become keyword-only parameters in Python 3.14.", 1)) - { - goto exit; - } - } - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 4, 4, 1, argsbuf); - if (!args) { - goto exit; - } - a = args[0]; - b = args[1]; - c = args[2]; - d = args[3]; - e = args[4]; - return_value = depr_star_pos2_len2_with_kwd_impl(module, a, b, c, d, e); - -exit: - return return_value; -} -/*[clinic end generated code: output=7a16fee4d6742d54 input=a9049054013a1b77]*/ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 28bd2a4430d14..58f9c8e8a3e9a 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -323,7 +323,11 @@ Modules/_testcapi/vectorcall.c - MethodDescriptorDerived_Type - Modules/_testcapi/vectorcall.c - MethodDescriptorNopGet_Type - Modules/_testcapi/vectorcall.c - MethodDescriptor2_Type - Modules/_testclinic.c - DeprStarInit - +Modules/_testclinic.c - DeprStarInitNoInline - Modules/_testclinic.c - DeprStarNew - +Modules/_testclinic.c - DeprKwdInit - +Modules/_testclinic.c - DeprKwdInitNoInline - +Modules/_testclinic.c - DeprKwdNew - ################################## diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 1593dc49e07e1..fe84e810760b4 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -849,25 +849,24 @@ class CLanguage(Language): #define {methoddef_name} #endif /* !defined({methoddef_name}) */ """) - DEPRECATED_POSITIONAL_PROTOTYPE: Final[str] = r""" + COMPILER_DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" // Emit compiler warnings when we get to Python {major}.{minor}. #if PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00C0 - # error \ - {cpp_message} + # error {message} #elif PY_VERSION_HEX >= 0x{major:02x}{minor:02x}00A0 # ifdef _MSC_VER - # pragma message ( \ - {cpp_message}) + # pragma message ({message}) # else - # warning \ - {cpp_message} + # warning {message} # endif #endif - if ({condition}) {{{{ + """ + DEPRECATION_WARNING_PROTOTYPE: Final[str] = r""" + if ({condition}) {{{{{errcheck} if (PyErr_WarnEx(PyExc_DeprecationWarning, - {depr_message}, 1)) + {message}, 1)) {{{{ - goto exit; + goto exit; }}}} }}}} """ @@ -893,6 +892,30 @@ def render( function = o return self.render_function(clinic, function) + def compiler_deprecated_warning( + self, + func: Function, + parameters: list[Parameter], + ) -> str | None: + minversion: VersionTuple | None = None + for p in parameters: + for version in p.deprecated_positional, p.deprecated_keyword: + if version and (not minversion or minversion > version): + minversion = version + if not minversion: + return None + + # Format the preprocessor warning and error messages. + assert isinstance(self.cpp.filename, str) + source = os.path.basename(self.cpp.filename) + message = f"Update the clinic input of {func.full_name!r}." + code = self.COMPILER_DEPRECATION_WARNING_PROTOTYPE.format( + major=minversion[0], + minor=minversion[1], + message=c_repr(message), + ) + return normalize_snippet(code) + def deprecate_positional_use( self, func: Function, @@ -910,15 +933,7 @@ def deprecate_positional_use( assert first_param.deprecated_positional == last_param.deprecated_positional thenceforth = first_param.deprecated_positional assert thenceforth is not None - - # Format the preprocessor warning and error messages. - assert isinstance(self.cpp.filename, str) - source = os.path.basename(self.cpp.filename) major, minor = thenceforth - cpp_message = ( - f"In {source}, update parameter(s) {pstr} in the clinic " - f"input of {func.full_name!r} to be keyword-only." - ) # Format the deprecation message. if first_pos == 0: @@ -927,7 +942,7 @@ def deprecate_positional_use( condition = f"nargs == {first_pos+1}" if first_pos: preamble = f"Passing {first_pos+1} positional arguments to " - depr_message = preamble + ( + message = preamble + ( f"{func.fulldisplayname}() is deprecated. Parameter {pstr} will " f"become a keyword-only parameter in Python {major}.{minor}." ) @@ -938,26 +953,93 @@ def deprecate_positional_use( f"Passing more than {first_pos} positional " f"argument{'s' if first_pos != 1 else ''} to " ) - depr_message = preamble + ( + message = preamble + ( f"{func.fulldisplayname}() is deprecated. Parameters {pstr} will " f"become keyword-only parameters in Python {major}.{minor}." ) # Append deprecation warning to docstring. - lines = textwrap.wrap(f"Note: {depr_message}") - docstring = "\n".join(lines) + docstring = textwrap.fill(f"Note: {message}") func.docstring += f"\n\n{docstring}\n" + # Format and return the code block. + code = self.DEPRECATION_WARNING_PROTOTYPE.format( + condition=condition, + errcheck="", + message=wrapped_c_string_literal(message, width=64, + subsequent_indent=20), + ) + return normalize_snippet(code, indent=4) + + def deprecate_keyword_use( + self, + func: Function, + params: dict[int, Parameter], + argname_fmt: str | None, + ) -> str: + assert len(params) > 0 + names = [repr(p.name) for p in params.values()] + first_param = next(iter(params.values())) + last_param = next(reversed(params.values())) + + # Pretty-print list of names. + pstr = pprint_words(names) + + # For now, assume there's only one deprecation level. + assert first_param.deprecated_keyword == last_param.deprecated_keyword + thenceforth = first_param.deprecated_keyword + assert thenceforth is not None + major, minor = thenceforth + # Format the deprecation message. + containscheck = "" + conditions = [] + for i, p in params.items(): + if p.is_optional(): + if argname_fmt: + conditions.append(f"nargs < {i+1} && {argname_fmt % i}") + elif func.kind.new_or_init: + conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, &_Py_ID({p.name}))") + containscheck = "PyDict_Contains" + else: + conditions.append(f"nargs < {i+1} && PySequence_Contains(kwnames, &_Py_ID({p.name}))") + containscheck = "PySequence_Contains" + else: + conditions = [f"nargs < {i+1}"] + condition = ") || (".join(conditions) + if len(conditions) > 1: + condition = f"(({condition}))" + if last_param.is_optional(): + if func.kind.new_or_init: + condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}" + else: + condition = f"kwnames && PyTuple_GET_SIZE(kwnames) && {condition}" + if len(params) == 1: + what1 = "argument" + what2 = "parameter" + else: + what1 = "arguments" + what2 = "parameters" + message = ( + f"Passing keyword {what1} {pstr} to {func.fulldisplayname}() is deprecated. " + f"Corresponding {what2} will become positional-only in Python {major}.{minor}." + ) + if containscheck: + errcheck = f""" + if (PyErr_Occurred()) {{{{ // {containscheck}() above can fail + goto exit; + }}}}""" + else: + errcheck = "" + if argname_fmt: + # Append deprecation warning to docstring. + docstring = textwrap.fill(f"Note: {message}") + func.docstring += f"\n\n{docstring}\n" # Format and return the code block. - code = self.DEPRECATED_POSITIONAL_PROTOTYPE.format( + code = self.DEPRECATION_WARNING_PROTOTYPE.format( condition=condition, - major=major, - minor=minor, - cpp_message=wrapped_c_string_literal(cpp_message, suffix=" \\", - width=64, - subsequent_indent=16), - depr_message=wrapped_c_string_literal(depr_message, width=64, - subsequent_indent=20), + errcheck=errcheck, + message=wrapped_c_string_literal(message, width=64, + subsequent_indent=20), ) return normalize_snippet(code, indent=4) @@ -1258,6 +1340,14 @@ def parser_body( parser_definition = parser_body(parser_prototype, *parser_code) else: + deprecated_positionals: dict[int, Parameter] = {} + deprecated_keywords: dict[int, Parameter] = {} + for i, p in enumerate(parameters): + if p.deprecated_positional: + deprecated_positionals[i] = p + if p.deprecated_keyword: + deprecated_keywords[i] = p + has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters) - int(vararg != NO_VARARG)) if vararg == NO_VARARG: args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( @@ -1310,7 +1400,10 @@ def parser_body( flags = 'METH_METHOD|' + flags parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS - deprecated_positionals: dict[int, Parameter] = {} + if deprecated_keywords: + code = self.deprecate_keyword_use(f, deprecated_keywords, argname_fmt) + parser_code.append(code) + add_label: str | None = None for i, p in enumerate(parameters): if isinstance(p.converter, defining_class_converter): @@ -1325,8 +1418,6 @@ def parser_body( parser_code.append("%s:" % add_label) add_label = None if not p.is_optional(): - if p.deprecated_positional: - deprecated_positionals[i] = p parser_code.append(normalize_snippet(parsearg, indent=4)) elif i < pos_only: add_label = 'skip_optional_posonly' @@ -1356,8 +1447,6 @@ def parser_body( goto %s; }} """ % add_label, indent=4)) - if p.deprecated_positional: - deprecated_positionals[i] = p if i + 1 == len(parameters): parser_code.append(normalize_snippet(parsearg, indent=4)) else: @@ -1373,12 +1462,6 @@ def parser_body( }} """ % add_label, indent=4)) - if deprecated_positionals: - code = self.deprecate_positional_use(f, deprecated_positionals) - assert parser_code is not None - # Insert the deprecation code before parameter parsing. - parser_code.insert(0, code) - if parser_code is not None: if add_label: parser_code.append("%s:" % add_label) @@ -1398,6 +1481,17 @@ def parser_body( goto exit; }} """, indent=4)] + if deprecated_positionals or deprecated_keywords: + declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" + if deprecated_keywords: + code = self.deprecate_keyword_use(f, deprecated_keywords, None) + parser_code.append(code) + + if deprecated_positionals: + code = self.deprecate_positional_use(f, deprecated_positionals) + # Insert the deprecation code before parameter parsing. + parser_code.insert(0, code) + parser_definition = parser_body(parser_prototype, *parser_code, declarations=declarations) @@ -1478,6 +1572,10 @@ def parser_body( parser_definition = parser_definition.replace("{return_value_declaration}", return_value_declaration) + compiler_warning = self.compiler_deprecated_warning(f, parameters) + if compiler_warning: + parser_definition = compiler_warning + "\n\n" + parser_definition + d = { "docstring_prototype" : docstring_prototype, "docstring_definition" : docstring_definition, @@ -2739,6 +2837,7 @@ class Parameter: group: int = 0 # (`None` signifies that there is no deprecation) deprecated_positional: VersionTuple | None = None + deprecated_keyword: VersionTuple | None = None right_bracket_count: int = dc.field(init=False, default=0) def __repr__(self) -> str: @@ -4576,6 +4675,7 @@ class DSLParser: keyword_only: bool positional_only: bool deprecated_positional: VersionTuple | None + deprecated_keyword: VersionTuple | None group: int parameter_state: ParamState indent: IndentStack @@ -4583,11 +4683,7 @@ class DSLParser: coexist: bool parameter_continuation: str preserve_output: bool - star_from_version_re = create_regex( - before="* [from ", - after="]", - word=False, - ) + from_version_re = re.compile(r'([*/]) +\[from +(.+)\]') def __init__(self, clinic: Clinic) -> None: self.clinic = clinic @@ -4612,6 +4708,7 @@ def reset(self) -> None: self.keyword_only = False self.positional_only = False self.deprecated_positional = None + self.deprecated_keyword = None self.group = 0 self.parameter_state: ParamState = ParamState.START self.indent = IndentStack() @@ -5089,21 +5186,22 @@ def state_parameter(self, line: str) -> None: return line = line.lstrip() - match = self.star_from_version_re.match(line) + version: VersionTuple | None = None + match = self.from_version_re.fullmatch(line) if match: - self.parse_deprecated_positional(match.group(1)) - return + line = match[1] + version = self.parse_version(match[2]) func = self.function match line: case '*': - self.parse_star(func) + self.parse_star(func, version) case '[': self.parse_opening_square_bracket(func) case ']': self.parse_closing_square_bracket(func) case '/': - self.parse_slash(func) + self.parse_slash(func, version) case param: self.parse_parameter(param) @@ -5404,29 +5502,36 @@ def parse_converter( "Annotations must be either a name, a function call, or a string." ) - def parse_deprecated_positional(self, thenceforth: str) -> None: + def parse_version(self, thenceforth: str) -> VersionTuple: + """Parse Python version in `[from ...]` marker.""" assert isinstance(self.function, Function) - fname = self.function.full_name - if self.keyword_only: - fail(f"Function {fname!r}: '* [from ...]' must come before '*'") - if self.deprecated_positional: - fail(f"Function {fname!r} uses '[from ...]' more than once.") try: major, minor = thenceforth.split(".") - self.deprecated_positional = int(major), int(minor) + return int(major), int(minor) except ValueError: fail( - f"Function {fname!r}: expected format '* [from major.minor]' " + f"Function {self.function.name!r}: expected format '[from major.minor]' " f"where 'major' and 'minor' are integers; got {thenceforth!r}" ) - def parse_star(self, function: Function) -> None: - """Parse keyword-only parameter marker '*'.""" - if self.keyword_only: - fail(f"Function {function.name!r} uses '*' more than once.") - self.deprecated_positional = None - self.keyword_only = True + def parse_star(self, function: Function, version: VersionTuple | None) -> None: + """Parse keyword-only parameter marker '*'. + + The 'version' parameter signifies the future version from which + the marker will take effect (None means it is already in effect). + """ + if version is None: + if self.keyword_only: + fail(f"Function {function.name!r} uses '*' more than once.") + self.check_remaining_star() + self.keyword_only = True + else: + if self.keyword_only: + fail(f"Function {function.name!r}: '* [from ...]' must come before '*'") + if self.deprecated_positional: + fail(f"Function {function.name!r} uses '* [from ...]' more than once.") + self.deprecated_positional = version def parse_opening_square_bracket(self, function: Function) -> None: """Parse opening parameter group symbol '['.""" @@ -5460,11 +5565,38 @@ def parse_closing_square_bracket(self, function: Function) -> None: f"has an unsupported group configuration. " f"(Unexpected state {st}.c)") - def parse_slash(self, function: Function) -> None: - """Parse positional-only parameter marker '/'.""" - if self.positional_only: - fail(f"Function {function.name!r} uses '/' more than once.") + def parse_slash(self, function: Function, version: VersionTuple | None) -> None: + """Parse positional-only parameter marker '/'. + + The 'version' parameter signifies the future version from which + the marker will take effect (None means it is already in effect). + """ + if version is None: + if self.deprecated_keyword: + fail(f"Function {function.name!r}: '/' must precede '/ [from ...]'") + if self.deprecated_positional: + fail(f"Function {function.name!r}: '/' must precede '* [from ...]'") + if self.keyword_only: + fail(f"Function {function.name!r}: '/' must precede '*'") + if self.positional_only: + fail(f"Function {function.name!r} uses '/' more than once.") + else: + if self.deprecated_keyword: + fail(f"Function {function.name!r} uses '/ [from ...]' more than once.") + if self.deprecated_positional: + fail(f"Function {function.name!r}: '/ [from ...]' must precede '* [from ...]'") + if self.keyword_only: + fail(f"Function {function.name!r}: '/ [from ...]' must precede '*'") self.positional_only = True + self.deprecated_keyword = version + if version is not None: + found = False + for p in reversed(function.parameters.values()): + found = p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD + break + if not found: + fail(f"Function {function.name!r} specifies '/ [from ...]' " + f"without preceding parameters.") # REQUIRED and OPTIONAL are allowed here, that allows positional-only # without option groups to work (and have default values!) allowed = { @@ -5476,19 +5608,13 @@ def parse_slash(self, function: Function) -> None: if (self.parameter_state not in allowed) or self.group: fail(f"Function {function.name!r} has an unsupported group configuration. " f"(Unexpected state {self.parameter_state}.d)") - if self.keyword_only: - fail(f"Function {function.name!r} mixes keyword-only and " - "positional-only parameters, which is unsupported.") # fixup preceding parameters for p in function.parameters.values(): - if p.is_vararg(): - continue - if (p.kind is not inspect.Parameter.POSITIONAL_OR_KEYWORD and - not isinstance(p.converter, self_converter) - ): - fail(f"Function {function.name!r} mixes keyword-only and " - "positional-only parameters, which is unsupported.") - p.kind = inspect.Parameter.POSITIONAL_ONLY + if p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD: + if version is None: + p.kind = inspect.Parameter.POSITIONAL_ONLY + else: + p.deprecated_keyword = version def state_parameter_docstring_start(self, line: str) -> None: assert self.indent.margin is not None, "self.margin.infer() has not yet been called to set the margin" @@ -5773,6 +5899,29 @@ def format_docstring(self) -> str: signature=signature, parameters=parameters).rstrip() + def check_remaining_star(self, lineno: int | None = None) -> None: + assert isinstance(self.function, Function) + + if self.keyword_only: + symbol = '*' + elif self.deprecated_positional: + symbol = '* [from ...]' + else: + return + + no_param_after_symbol = True + for p in reversed(self.function.parameters.values()): + if self.keyword_only: + if p.kind == inspect.Parameter.KEYWORD_ONLY: + return + elif self.deprecated_positional: + if p.deprecated_positional == self.deprecated_positional: + return + break + + fail(f"Function {self.function.name!r} specifies {symbol!r} " + f"without following parameters.", line_number=lineno) + def do_post_block_processing_cleanup(self, lineno: int) -> None: """ Called when processing the block is done. @@ -5780,28 +5929,7 @@ def do_post_block_processing_cleanup(self, lineno: int) -> None: if not self.function: return - def check_remaining( - symbol: str, - condition: Callable[[Parameter], bool] - ) -> None: - assert isinstance(self.function, Function) - - if values := self.function.parameters.values(): - last_param = next(reversed(values)) - no_param_after_symbol = condition(last_param) - else: - no_param_after_symbol = True - if no_param_after_symbol: - fname = self.function.full_name - fail(f"Function {fname!r} specifies {symbol!r} " - "without any parameters afterwards.", line_number=lineno) - - if self.keyword_only: - check_remaining("*", lambda p: p.kind != inspect.Parameter.KEYWORD_ONLY) - - if self.deprecated_positional: - check_remaining("* [from ...]", lambda p: not p.deprecated_positional) - + self.check_remaining_star(lineno) self.function.docstring = self.format_docstring() From webhook-mailer at python.org Sat Aug 19 04:34:38 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Sat, 19 Aug 2023 08:34:38 -0000 Subject: [Python-checkins] Docs: format sys.float_info properly (#108107) Message-ID: https://github.com/python/cpython/commit/ca0c6c1f1ef79d10bc49b61d638d87cde265aa94 commit: ca0c6c1f1ef79d10bc49b61d638d87cde265aa94 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-19T08:34:34Z summary: Docs: format sys.float_info properly (#108107) - Normalise capitalisation and punctuation - Use attribute markup for named tuple attributes - Use :c:macro: markup for C macros - Use a list for the 'rounds' attribute values - Use list-table, for better .rst readability - Remove one unneeded sys.float_info.dig link Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/sys.rst diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index da2435aa4bc23..a82e043f6209a 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -574,61 +574,82 @@ always available. programming language; see section 5.2.4.2.2 of the 1999 ISO/IEC C standard [C99]_, 'Characteristics of floating types', for details. - .. tabularcolumns:: |l|l|L| - - +---------------------+---------------------+--------------------------------------------------+ - | attribute | float.h macro | explanation | - +=====================+=====================+==================================================+ - | ``epsilon`` | ``DBL_EPSILON`` | difference between 1.0 and the least value | - | | | greater than 1.0 that is representable as a float| - | | | | - | | | See also :func:`math.ulp`. | - +---------------------+---------------------+--------------------------------------------------+ - | ``dig`` | ``DBL_DIG`` | maximum number of decimal digits that can be | - | | | faithfully represented in a float; see below | - +---------------------+---------------------+--------------------------------------------------+ - | ``mant_dig`` | ``DBL_MANT_DIG`` | float precision: the number of base-``radix`` | - | | | digits in the significand of a float | - +---------------------+---------------------+--------------------------------------------------+ - | ``max`` | ``DBL_MAX`` | maximum representable positive finite float | - +---------------------+---------------------+--------------------------------------------------+ - | ``max_exp`` | ``DBL_MAX_EXP`` | maximum integer *e* such that ``radix**(e-1)`` is| - | | | a representable finite float | - +---------------------+---------------------+--------------------------------------------------+ - | ``max_10_exp`` | ``DBL_MAX_10_EXP`` | maximum integer *e* such that ``10**e`` is in the| - | | | range of representable finite floats | - +---------------------+---------------------+--------------------------------------------------+ - | ``min`` | ``DBL_MIN`` | minimum representable positive *normalized* float| - | | | | - | | | Use :func:`math.ulp(0.0) ` to get the | - | | | smallest positive *denormalized* representable | - | | | float. | - +---------------------+---------------------+--------------------------------------------------+ - | ``min_exp`` | ``DBL_MIN_EXP`` | minimum integer *e* such that ``radix**(e-1)`` is| - | | | a normalized float | - +---------------------+---------------------+--------------------------------------------------+ - | ``min_10_exp`` | ``DBL_MIN_10_EXP`` | minimum integer *e* such that ``10**e`` is a | - | | | normalized float | - +---------------------+---------------------+--------------------------------------------------+ - | ``radix`` | ``FLT_RADIX`` | radix of exponent representation | - +---------------------+---------------------+--------------------------------------------------+ - | ``rounds`` | ``FLT_ROUNDS`` | integer representing the rounding mode for | - | | | floating-point arithmetic. This reflects the | - | | | value of the system ``FLT_ROUNDS`` macro at | - | | | interpreter startup time: | - | | | ``-1`` indeterminable, | - | | | ``0`` toward zero, | - | | | ``1`` to nearest, | - | | | ``2`` toward positive infinity, | - | | | ``3`` toward negative infinity | - | | | | - | | | All other values for ``FLT_ROUNDS`` characterize | - | | | implementation-defined rounding behavior. | - +---------------------+---------------------+--------------------------------------------------+ + .. list-table:: Attributes of the :data:`!float_info` :term:`named tuple` + :header-rows: 1 + + * - attribute + - float.h macro + - explanation + + * - .. attribute:: float_info.epsilon + - :c:macro:`!DBL_EPSILON` + - difference between 1.0 and the least value greater than 1.0 that is + representable as a float. + + See also :func:`math.ulp`. + + * - .. attribute:: float_info.dig + - :c:macro:`!DBL_DIG` + - The maximum number of decimal digits that can be faithfully + represented in a float; see below. + + * - .. attribute:: float_info.mant_dig + - :c:macro:`!DBL_MANT_DIG` + - Float precision: the number of base-``radix`` digits in the + significand of a float. + + * - .. attribute:: float_info.max + - :c:macro:`!DBL_MAX` + - The maximum representable positive finite float. + + * - .. attribute:: float_info.max_exp + - :c:macro:`!DBL_MAX_EXP` + - The maximum integer *e* such that ``radix**(e-1)`` is a representable + finite float. + + * - .. attribute:: float_info.max_10_exp + - :c:macro:`!DBL_MAX_10_EXP` + - The maximum integer *e* such that ``10**e`` is in the range of + representable finite floats. + + * - .. attribute:: float_info.min + - :c:macro:`!DBL_MIN` + - The minimum representable positive *normalized* float. + + Use :func:`math.ulp(0.0) ` to get the smallest positive + *denormalized* representable float. + + * - .. attribute:: float_info.min_exp + - :c:macro:`!DBL_MIN_EXP` + - The minimum integer *e* such that ``radix**(e-1)`` is a normalized + float. + + * - .. attribute:: float_info.min_10_exp + - :c:macro:`!DBL_MIN_10_EXP` + - The minimum integer *e* such that ``10**e`` is a normalized float. + + * - .. attribute:: float_info.radix + - :c:macro:`!FLT_RADIX` + - The radix of exponent representation. + + * - .. attribute:: float_info.rounds + - :c:macro:`!FLT_ROUNDS` + - An integer representing the rounding mode for floating-point arithmetic. + This reflects the value of the system :c:macro:`!FLT_ROUNDS` macro + at interpreter startup time: + + * ``-1``: indeterminable + * ``0``: toward zero + * ``1``: to nearest + * ``2``: toward positive infinity + * ``3``: toward negative infinity + + All other values for :c:macro:`!FLT_ROUNDS` characterize + implementation-defined rounding behavior. The attribute :attr:`sys.float_info.dig` needs further explanation. If ``s`` is any string representing a decimal number with at most - :attr:`sys.float_info.dig` significant digits, then converting ``s`` to a + :attr:`!sys.float_info.dig` significant digits, then converting ``s`` to a float and back again will recover a string representing the same decimal value:: From webhook-mailer at python.org Sat Aug 19 05:13:12 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Sat, 19 Aug 2023 09:13:12 -0000 Subject: [Python-checkins] [3.11] Docs: format sys.float_info properly (GH-108107) (#108131) Message-ID: https://github.com/python/cpython/commit/0673d4c762d6dc4c742e71e76cee66181c457f24 commit: 0673d4c762d6dc4c742e71e76cee66181c457f24 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: AlexWaygood date: 2023-08-19T10:13:09+01:00 summary: [3.11] Docs: format sys.float_info properly (GH-108107) (#108131) Docs: format sys.float_info properly (GH-108107) - Normalise capitalisation and punctuation - Use attribute markup for named tuple attributes - Use :c:macro: markup for C macros - Use a list for the 'rounds' attribute values - Use list-table, for better .rst readability - Remove one unneeded sys.float_info.dig link (cherry picked from commit ca0c6c1f1ef79d10bc49b61d638d87cde265aa94) Co-authored-by: Erlend E. Aasland Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/sys.rst diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index b5ad0b6860b95..08bf2d13572f4 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -570,61 +570,82 @@ always available. programming language; see section 5.2.4.2.2 of the 1999 ISO/IEC C standard [C99]_, 'Characteristics of floating types', for details. - .. tabularcolumns:: |l|l|L| - - +---------------------+---------------------+--------------------------------------------------+ - | attribute | float.h macro | explanation | - +=====================+=====================+==================================================+ - | ``epsilon`` | ``DBL_EPSILON`` | difference between 1.0 and the least value | - | | | greater than 1.0 that is representable as a float| - | | | | - | | | See also :func:`math.ulp`. | - +---------------------+---------------------+--------------------------------------------------+ - | ``dig`` | ``DBL_DIG`` | maximum number of decimal digits that can be | - | | | faithfully represented in a float; see below | - +---------------------+---------------------+--------------------------------------------------+ - | ``mant_dig`` | ``DBL_MANT_DIG`` | float precision: the number of base-``radix`` | - | | | digits in the significand of a float | - +---------------------+---------------------+--------------------------------------------------+ - | ``max`` | ``DBL_MAX`` | maximum representable positive finite float | - +---------------------+---------------------+--------------------------------------------------+ - | ``max_exp`` | ``DBL_MAX_EXP`` | maximum integer *e* such that ``radix**(e-1)`` is| - | | | a representable finite float | - +---------------------+---------------------+--------------------------------------------------+ - | ``max_10_exp`` | ``DBL_MAX_10_EXP`` | maximum integer *e* such that ``10**e`` is in the| - | | | range of representable finite floats | - +---------------------+---------------------+--------------------------------------------------+ - | ``min`` | ``DBL_MIN`` | minimum representable positive *normalized* float| - | | | | - | | | Use :func:`math.ulp(0.0) ` to get the | - | | | smallest positive *denormalized* representable | - | | | float. | - +---------------------+---------------------+--------------------------------------------------+ - | ``min_exp`` | ``DBL_MIN_EXP`` | minimum integer *e* such that ``radix**(e-1)`` is| - | | | a normalized float | - +---------------------+---------------------+--------------------------------------------------+ - | ``min_10_exp`` | ``DBL_MIN_10_EXP`` | minimum integer *e* such that ``10**e`` is a | - | | | normalized float | - +---------------------+---------------------+--------------------------------------------------+ - | ``radix`` | ``FLT_RADIX`` | radix of exponent representation | - +---------------------+---------------------+--------------------------------------------------+ - | ``rounds`` | ``FLT_ROUNDS`` | integer representing the rounding mode for | - | | | floating-point arithmetic. This reflects the | - | | | value of the system ``FLT_ROUNDS`` macro at | - | | | interpreter startup time: | - | | | ``-1`` indeterminable, | - | | | ``0`` toward zero, | - | | | ``1`` to nearest, | - | | | ``2`` toward positive infinity, | - | | | ``3`` toward negative infinity | - | | | | - | | | All other values for ``FLT_ROUNDS`` characterize | - | | | implementation-defined rounding behavior. | - +---------------------+---------------------+--------------------------------------------------+ + .. list-table:: Attributes of the :data:`!float_info` :term:`named tuple` + :header-rows: 1 + + * - attribute + - float.h macro + - explanation + + * - .. attribute:: float_info.epsilon + - :c:macro:`!DBL_EPSILON` + - difference between 1.0 and the least value greater than 1.0 that is + representable as a float. + + See also :func:`math.ulp`. + + * - .. attribute:: float_info.dig + - :c:macro:`!DBL_DIG` + - The maximum number of decimal digits that can be faithfully + represented in a float; see below. + + * - .. attribute:: float_info.mant_dig + - :c:macro:`!DBL_MANT_DIG` + - Float precision: the number of base-``radix`` digits in the + significand of a float. + + * - .. attribute:: float_info.max + - :c:macro:`!DBL_MAX` + - The maximum representable positive finite float. + + * - .. attribute:: float_info.max_exp + - :c:macro:`!DBL_MAX_EXP` + - The maximum integer *e* such that ``radix**(e-1)`` is a representable + finite float. + + * - .. attribute:: float_info.max_10_exp + - :c:macro:`!DBL_MAX_10_EXP` + - The maximum integer *e* such that ``10**e`` is in the range of + representable finite floats. + + * - .. attribute:: float_info.min + - :c:macro:`!DBL_MIN` + - The minimum representable positive *normalized* float. + + Use :func:`math.ulp(0.0) ` to get the smallest positive + *denormalized* representable float. + + * - .. attribute:: float_info.min_exp + - :c:macro:`!DBL_MIN_EXP` + - The minimum integer *e* such that ``radix**(e-1)`` is a normalized + float. + + * - .. attribute:: float_info.min_10_exp + - :c:macro:`!DBL_MIN_10_EXP` + - The minimum integer *e* such that ``10**e`` is a normalized float. + + * - .. attribute:: float_info.radix + - :c:macro:`!FLT_RADIX` + - The radix of exponent representation. + + * - .. attribute:: float_info.rounds + - :c:macro:`!FLT_ROUNDS` + - An integer representing the rounding mode for floating-point arithmetic. + This reflects the value of the system :c:macro:`!FLT_ROUNDS` macro + at interpreter startup time: + + * ``-1``: indeterminable + * ``0``: toward zero + * ``1``: to nearest + * ``2``: toward positive infinity + * ``3``: toward negative infinity + + All other values for :c:macro:`!FLT_ROUNDS` characterize + implementation-defined rounding behavior. The attribute :attr:`sys.float_info.dig` needs further explanation. If ``s`` is any string representing a decimal number with at most - :attr:`sys.float_info.dig` significant digits, then converting ``s`` to a + :attr:`!sys.float_info.dig` significant digits, then converting ``s`` to a float and back again will recover a string representing the same decimal value:: From webhook-mailer at python.org Sat Aug 19 07:48:05 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Sat, 19 Aug 2023 11:48:05 -0000 Subject: [Python-checkins] gh-72684: Tkinter: provide interface for "tk busy" subcommands (GH-107684) Message-ID: https://github.com/python/cpython/commit/79db9d9a0e8f51ad4ea5caae31d7ae4b29985271 commit: 79db9d9a0e8f51ad4ea5caae31d7ae4b29985271 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-19T14:48:02+03:00 summary: gh-72684: Tkinter: provide interface for "tk busy" subcommands (GH-107684) Add tkinter.Misc methods: tk_busy_hold(), tk_busy_configure(), tk_busy_cget(), tk_busy_forget(), tk_busy_current(), and tk_busy_status(). files: A Misc/NEWS.d/next/Library/2023-08-06-10-52-12.gh-issue-72684.Ls2mSf.rst M Doc/whatsnew/3.13.rst M Lib/test/test_tkinter/test_misc.py M Lib/tkinter/__init__.py diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 85aa81b4a98d2..47b868bad3192 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -144,6 +144,15 @@ pathlib :meth:`~pathlib.Path.is_dir`. (Contributed by Barney Gale in :gh:`77609` and :gh:`105793`.) +tkinter +------- + +* Add :mod:`tkinter` widget methods: + :meth:`!tk_busy_hold`, :meth:`!tk_busy_configure`, + :meth:`!tk_busy_cget`, :meth:`!tk_busy_forget`, + :meth:`!tk_busy_current`, and :meth:`!tk_busy_status`. + (Contributed by Miguel, klappnase and Serhiy Storchaka in :gh:`72684`.) + traceback --------- diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index d1aca58d15fbd..0ae27610be607 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -1,9 +1,10 @@ import functools import unittest import tkinter +from tkinter import TclError import enum from test import support -from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest +from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest, requires_tk support.requires('gui') @@ -36,6 +37,59 @@ def test_generated_names(self): for name in str(b).split('.'): self.assertFalse(name.isidentifier(), msg=repr(name)) + @requires_tk(8, 6, 6) + def test_tk_busy(self): + root = self.root + f = tkinter.Frame(root, name='myframe') + f2 = tkinter.Frame(root) + f.pack() + f2.pack() + b = tkinter.Button(f) + b.pack() + f.tk_busy_hold() + with self.assertRaisesRegex(TclError, 'unknown option "-spam"'): + f.tk_busy_configure(spam='eggs') + with self.assertRaisesRegex(TclError, 'unknown option "-spam"'): + f.tk_busy_cget('spam') + with self.assertRaisesRegex(TclError, 'unknown option "-spam"'): + f.tk_busy_configure('spam') + self.assertIsInstance(f.tk_busy_configure(), dict) + + self.assertTrue(f.tk_busy_status()) + self.assertFalse(root.tk_busy_status()) + self.assertFalse(f2.tk_busy_status()) + self.assertFalse(b.tk_busy_status()) + self.assertIn(f, f.tk_busy_current()) + self.assertIn(f, f.tk_busy_current('*.m?f*me')) + self.assertNotIn(f, f.tk_busy_current('*spam')) + + f.tk_busy_forget() + self.assertFalse(f.tk_busy_status()) + self.assertFalse(f.tk_busy_current()) + with self.assertRaisesRegex(TclError, "can't find busy window"): + f.tk_busy_configure() + with self.assertRaisesRegex(TclError, "can't find busy window"): + f.tk_busy_forget() + + @requires_tk(8, 6, 6) + def test_tk_busy_with_cursor(self): + root = self.root + if root._windowingsystem == 'aqua': + self.skipTest('the cursor option is not supported on OSX/Aqua') + f = tkinter.Frame(root, name='myframe') + f.pack() + f.tk_busy_hold(cursor='gumby') + + self.assertEqual(f.tk_busy_cget('cursor'), 'gumby') + f.tk_busy_configure(cursor='heart') + self.assertEqual(f.tk_busy_cget('cursor'), 'heart') + self.assertEqual(f.tk_busy_configure()['cursor'][4], 'heart') + self.assertEqual(f.tk_busy_configure('cursor')[4], 'heart') + + f.tk_busy_forget() + with self.assertRaisesRegex(TclError, "can't find busy window"): + f.tk_busy_cget('cursor') + def test_tk_setPalette(self): root = self.root root.tk_setPalette('black') diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index c59f8d11e8a9d..440e7f100c8c4 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -901,6 +901,85 @@ def bell(self, displayof=0): """Ring a display's bell.""" self.tk.call(('bell',) + self._displayof(displayof)) + def tk_busy_cget(self, option): + """Return the value of busy configuration option. + + The widget must have been previously made busy by + tk_busy_hold(). Option may have any of the values accepted by + tk_busy_hold(). + """ + return self.tk.call('tk', 'busy', 'cget', self._w, '-'+option) + busy_cget = tk_busy_cget + + def tk_busy_configure(self, cnf=None, **kw): + """Query or modify the busy configuration options. + + The widget must have been previously made busy by + tk_busy_hold(). Options may have any of the values accepted by + tk_busy_hold(). + + Please note that the option database is referenced by the widget + name or class. For example, if a Frame widget with name "frame" + is to be made busy, the busy cursor can be specified for it by + either call: + + w.option_add('*frame.busyCursor', 'gumby') + w.option_add('*Frame.BusyCursor', 'gumby') + """ + if kw: + cnf = _cnfmerge((cnf, kw)) + elif cnf: + cnf = _cnfmerge(cnf) + if cnf is None: + return self._getconfigure( + 'tk', 'busy', 'configure', self._w) + if isinstance(cnf, str): + return self._getconfigure1( + 'tk', 'busy', 'configure', self._w, '-'+cnf) + self.tk.call('tk', 'busy', 'configure', self._w, *self._options(cnf)) + busy_config = busy_configure = tk_busy_config = tk_busy_configure + + def tk_busy_current(self, pattern=None): + """Return a list of widgets that are currently busy. + + If a pattern is given, only busy widgets whose path names match + a pattern are returned. + """ + return [self._nametowidget(x) for x in + self.tk.splitlist(self.tk.call( + 'tk', 'busy', 'current', pattern))] + busy_current = tk_busy_current + + def tk_busy_forget(self): + """Make this widget no longer busy. + + User events will again be received by the widget. + """ + self.tk.call('tk', 'busy', 'forget', self._w) + busy_forget = tk_busy_forget + + def tk_busy_hold(self, **kw): + """Make this widget appear busy. + + The specified widget and its descendants will be blocked from + user interactions. Normally update() should be called + immediately afterward to insure that the hold operation is in + effect before the application starts its processing. + + The only supported configuration option is: + + cursor: the cursor to be displayed when the widget is made + busy. + """ + self.tk.call('tk', 'busy', 'hold', self._w, *self._options(kw)) + busy = busy_hold = tk_busy = tk_busy_hold + + def tk_busy_status(self): + """Return True if the widget is busy, False otherwise.""" + return self.tk.getboolean(self.tk.call( + 'tk', 'busy', 'status', self._w)) + busy_status = tk_busy_status + # Clipboard handling: def clipboard_get(self, **kw): """Retrieve data from the clipboard on window's display. diff --git a/Misc/NEWS.d/next/Library/2023-08-06-10-52-12.gh-issue-72684.Ls2mSf.rst b/Misc/NEWS.d/next/Library/2023-08-06-10-52-12.gh-issue-72684.Ls2mSf.rst new file mode 100644 index 0000000000000..1ac0e6d790456 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-06-10-52-12.gh-issue-72684.Ls2mSf.rst @@ -0,0 +1,3 @@ +Add :mod:`tkinter` widget methods: :meth:`!tk_busy_hold`, +:meth:`!tk_busy_configure`, :meth:`!tk_busy_cget`, :meth:`!tk_busy_forget`, +:meth:`!tk_busy_current`, and :meth:`!tk_busy_status`. From webhook-mailer at python.org Sat Aug 19 07:51:06 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Sat, 19 Aug 2023 11:51:06 -0000 Subject: [Python-checkins] gh-107915: Handle errors in C API functions PyErr_Set*() and PyErr_Format() (GH-107918) Message-ID: https://github.com/python/cpython/commit/633ea217a85f6b6ba5bdbc73094254d5811b3485 commit: 633ea217a85f6b6ba5bdbc73094254d5811b3485 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-19T14:51:03+03:00 summary: gh-107915: Handle errors in C API functions PyErr_Set*() and PyErr_Format() (GH-107918) Such C API functions as PyErr_SetString(), PyErr_Format(), PyErr_SetFromErrnoWithFilename() and many others no longer crash or ignore errors if it failed to format the error message or decode the filename. Instead, they keep a corresponding error. files: A Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst M Lib/test/test_capi/test_exceptions.py M Modules/_testcapi/clinic/exceptions.c.h M Modules/_testcapi/exceptions.c M Python/errors.c diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index 118b575cba6df..b96cc7922a0ee 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -1,9 +1,12 @@ +import errno +import os import re import sys import unittest from test import support from test.support import import_helper +from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE from test.support.script_helper import assert_python_failure from test.support.testcase import ExceptionIsLikeMixin @@ -12,6 +15,8 @@ # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') +NULL = None + class Test_Exceptions(unittest.TestCase): def test_exception(self): @@ -189,6 +194,82 @@ def __repr__(self): self.assertEqual(exc.__notes__[0], 'Normalization failed: type=Broken args=') + def test_set_string(self): + """Test PyErr_SetString()""" + setstring = _testcapi.err_setstring + with self.assertRaises(ZeroDivisionError) as e: + setstring(ZeroDivisionError, b'error') + self.assertEqual(e.exception.args, ('error',)) + with self.assertRaises(ZeroDivisionError) as e: + setstring(ZeroDivisionError, '???????'.encode()) + self.assertEqual(e.exception.args, ('???????',)) + + with self.assertRaises(UnicodeDecodeError): + setstring(ZeroDivisionError, b'\xff') + self.assertRaises(SystemError, setstring, list, b'error') + # CRASHES setstring(ZeroDivisionError, NULL) + # CRASHES setstring(NULL, b'error') + + def test_format(self): + """Test PyErr_Format()""" + import_helper.import_module('ctypes') + from ctypes import pythonapi, py_object, c_char_p, c_int + name = "PyErr_Format" + PyErr_Format = getattr(pythonapi, name) + PyErr_Format.argtypes = (py_object, c_char_p,) + PyErr_Format.restype = py_object + with self.assertRaises(ZeroDivisionError) as e: + PyErr_Format(ZeroDivisionError, b'%s %d', b'error', c_int(42)) + self.assertEqual(e.exception.args, ('error 42',)) + with self.assertRaises(ZeroDivisionError) as e: + PyErr_Format(ZeroDivisionError, b'%s', '???????'.encode()) + self.assertEqual(e.exception.args, ('???????',)) + + with self.assertRaisesRegex(OverflowError, 'not in range'): + PyErr_Format(ZeroDivisionError, b'%c', c_int(-1)) + with self.assertRaisesRegex(ValueError, 'format string'): + PyErr_Format(ZeroDivisionError, b'\xff') + self.assertRaises(SystemError, PyErr_Format, list, b'error') + # CRASHES PyErr_Format(ZeroDivisionError, NULL) + # CRASHES PyErr_Format(py_object(), b'error') + + def test_setfromerrnowithfilename(self): + """Test PyErr_SetFromErrnoWithFilename()""" + setfromerrnowithfilename = _testcapi.err_setfromerrnowithfilename + ENOENT = errno.ENOENT + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, b'file') + self.assertEqual(e.exception.args, + (ENOENT, 'No such file or directory')) + self.assertEqual(e.exception.errno, ENOENT) + self.assertEqual(e.exception.filename, 'file') + + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, os.fsencode(TESTFN)) + self.assertEqual(e.exception.filename, TESTFN) + + if TESTFN_UNDECODABLE: + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, TESTFN_UNDECODABLE) + self.assertEqual(e.exception.filename, + os.fsdecode(TESTFN_UNDECODABLE)) + + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, NULL) + self.assertIsNone(e.exception.filename) + + with self.assertRaises(OSError) as e: + setfromerrnowithfilename(0, OSError, b'file') + self.assertEqual(e.exception.args, (0, 'Error')) + self.assertEqual(e.exception.errno, 0) + self.assertEqual(e.exception.filename, 'file') + + with self.assertRaises(ZeroDivisionError) as e: + setfromerrnowithfilename(ENOENT, ZeroDivisionError, b'file') + self.assertEqual(e.exception.args, + (ENOENT, 'No such file or directory', 'file')) + # CRASHES setfromerrnowithfilename(ENOENT, NULL, b'error') + class Test_PyUnstable_Exc_PrepReraiseStar(ExceptionIsLikeMixin, unittest.TestCase): diff --git a/Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst b/Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst new file mode 100644 index 0000000000000..58ee3f169a28c --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst @@ -0,0 +1,4 @@ +Such C API functions as ``PyErr_SetString()``, ``PyErr_Format()``, +``PyErr_SetFromErrnoWithFilename()`` and many others no longer crash or +ignore errors if it failed to format the error message or decode the +filename. Instead, they keep a corresponding error. diff --git a/Modules/_testcapi/clinic/exceptions.c.h b/Modules/_testcapi/clinic/exceptions.c.h index 01730ffa2ed03..16954a5ebf3e5 100644 --- a/Modules/_testcapi/clinic/exceptions.c.h +++ b/Modules/_testcapi/clinic/exceptions.c.h @@ -215,6 +215,68 @@ _testcapi_exc_set_object_fetch(PyObject *module, PyObject *const *args, Py_ssize return return_value; } +PyDoc_STRVAR(_testcapi_err_setstring__doc__, +"err_setstring($module, exc, value, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_ERR_SETSTRING_METHODDEF \ + {"err_setstring", _PyCFunction_CAST(_testcapi_err_setstring), METH_FASTCALL, _testcapi_err_setstring__doc__}, + +static PyObject * +_testcapi_err_setstring_impl(PyObject *module, PyObject *exc, + const char *value, Py_ssize_t value_length); + +static PyObject * +_testcapi_err_setstring(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *exc; + const char *value; + Py_ssize_t value_length; + + if (!_PyArg_ParseStack(args, nargs, "Oz#:err_setstring", + &exc, &value, &value_length)) { + goto exit; + } + return_value = _testcapi_err_setstring_impl(module, exc, value, value_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(_testcapi_err_setfromerrnowithfilename__doc__, +"err_setfromerrnowithfilename($module, error, exc, value, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_ERR_SETFROMERRNOWITHFILENAME_METHODDEF \ + {"err_setfromerrnowithfilename", _PyCFunction_CAST(_testcapi_err_setfromerrnowithfilename), METH_FASTCALL, _testcapi_err_setfromerrnowithfilename__doc__}, + +static PyObject * +_testcapi_err_setfromerrnowithfilename_impl(PyObject *module, int error, + PyObject *exc, const char *value, + Py_ssize_t value_length); + +static PyObject * +_testcapi_err_setfromerrnowithfilename(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int error; + PyObject *exc; + const char *value; + Py_ssize_t value_length; + + if (!_PyArg_ParseStack(args, nargs, "iOz#:err_setfromerrnowithfilename", + &error, &exc, &value, &value_length)) { + goto exit; + } + return_value = _testcapi_err_setfromerrnowithfilename_impl(module, error, exc, value, value_length); + +exit: + return return_value; +} + PyDoc_STRVAR(_testcapi_raise_exception__doc__, "raise_exception($module, exception, num_args, /)\n" "--\n" @@ -426,4 +488,4 @@ _testcapi_unstable_exc_prep_reraise_star(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=fd6aef54f195c77b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d574342d716e98b5 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index a627bf1717fe0..d7edf8c7ce674 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -1,6 +1,8 @@ #include "parts.h" #include "clinic/exceptions.c.h" +#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); + /*[clinic input] module _testcapi [clinic start generated code]*/ @@ -129,6 +131,43 @@ _testcapi_exc_set_object_fetch_impl(PyObject *module, PyObject *exc, return value; } +/*[clinic input] +_testcapi.err_setstring + exc: object + value: str(zeroes=True, accept={robuffer, str, NoneType}) + / +[clinic start generated code]*/ + +static PyObject * +_testcapi_err_setstring_impl(PyObject *module, PyObject *exc, + const char *value, Py_ssize_t value_length) +/*[clinic end generated code: output=fba8705e5703dd3f input=e8a95fad66d9004b]*/ +{ + NULLABLE(exc); + PyErr_SetString(exc, value); + return NULL; +} + +/*[clinic input] +_testcapi.err_setfromerrnowithfilename + error: int + exc: object + value: str(zeroes=True, accept={robuffer, str, NoneType}) + / +[clinic start generated code]*/ + +static PyObject * +_testcapi_err_setfromerrnowithfilename_impl(PyObject *module, int error, + PyObject *exc, const char *value, + Py_ssize_t value_length) +/*[clinic end generated code: output=d02df5749a01850e input=ff7c384234bf097f]*/ +{ + NULLABLE(exc); + errno = error; + PyErr_SetFromErrnoWithFilename(exc, value); + return NULL; +} + /*[clinic input] _testcapi.raise_exception exception as exc: object @@ -338,6 +377,8 @@ static PyMethodDef test_methods[] = { _TESTCAPI_MAKE_EXCEPTION_WITH_DOC_METHODDEF _TESTCAPI_EXC_SET_OBJECT_METHODDEF _TESTCAPI_EXC_SET_OBJECT_FETCH_METHODDEF + _TESTCAPI_ERR_SETSTRING_METHODDEF + _TESTCAPI_ERR_SETFROMERRNOWITHFILENAME_METHODDEF _TESTCAPI_RAISE_EXCEPTION_METHODDEF _TESTCAPI_RAISE_MEMORYERROR_METHODDEF _TESTCAPI_SET_EXC_INFO_METHODDEF diff --git a/Python/errors.c b/Python/errors.c index 916958c9a0ab0..d9086c507251e 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -292,8 +292,10 @@ _PyErr_SetString(PyThreadState *tstate, PyObject *exception, const char *string) { PyObject *value = PyUnicode_FromString(string); - _PyErr_SetObject(tstate, exception, value); - Py_XDECREF(value); + if (value != NULL) { + _PyErr_SetObject(tstate, exception, value); + Py_DECREF(value); + } } void @@ -891,7 +893,13 @@ PyErr_SetFromErrnoWithFilenameObjects(PyObject *exc, PyObject *filenameObject, P PyObject * PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) { - PyObject *name = filename ? PyUnicode_DecodeFSDefault(filename) : NULL; + PyObject *name = NULL; + if (filename) { + name = PyUnicode_DecodeFSDefault(filename); + if (name == NULL) { + return NULL; + } + } PyObject *result = PyErr_SetFromErrnoWithFilenameObjects(exc, name, NULL); Py_XDECREF(name); return result; @@ -988,7 +996,13 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilename( int ierr, const char *filename) { - PyObject *name = filename ? PyUnicode_DecodeFSDefault(filename) : NULL; + PyObject *name = NULL; + if (filename) { + name = PyUnicode_DecodeFSDefault(filename); + if (name == NULL) { + return NULL; + } + } PyObject *ret = PyErr_SetExcFromWindowsErrWithFilenameObjects(exc, ierr, name, @@ -1012,7 +1026,13 @@ PyObject *PyErr_SetFromWindowsErrWithFilename( int ierr, const char *filename) { - PyObject *name = filename ? PyUnicode_DecodeFSDefault(filename) : NULL; + PyObject *name = NULL; + if (filename) { + name = PyUnicode_DecodeFSDefault(filename); + if (name == NULL) { + return NULL; + } + } PyObject *result = PyErr_SetExcFromWindowsErrWithFilenameObjects( PyExc_OSError, ierr, name, NULL); @@ -1137,9 +1157,10 @@ _PyErr_FormatV(PyThreadState *tstate, PyObject *exception, _PyErr_Clear(tstate); string = PyUnicode_FromFormatV(format, vargs); - - _PyErr_SetObject(tstate, exception, string); - Py_XDECREF(string); + if (string != NULL) { + _PyErr_SetObject(tstate, exception, string); + Py_DECREF(string); + } return NULL; } From webhook-mailer at python.org Sat Aug 19 08:22:16 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Sat, 19 Aug 2023 12:22:16 -0000 Subject: [Python-checkins] [3.11] gh-107915: Handle errors in C API functions PyErr_Set*() and PyErr_Format() (GH-107918) (GH-108135) Message-ID: https://github.com/python/cpython/commit/92a578409b70b5b9dabc9e55d4d13dd8239fe2f5 commit: 92a578409b70b5b9dabc9e55d4d13dd8239fe2f5 branch: 3.11 author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-19T12:22:13Z summary: [3.11] gh-107915: Handle errors in C API functions PyErr_Set*() and PyErr_Format() (GH-107918) (GH-108135) Such C API functions as PyErr_SetString(), PyErr_Format(), PyErr_SetFromErrnoWithFilename() and many others no longer crash or ignore errors if it failed to format the error message or decode the filename. Instead, they keep a corresponding error. (cherry picked from commit 633ea217a85f6b6ba5bdbc73094254d5811b3485) files: A Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst M Python/errors.c diff --git a/Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst b/Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst new file mode 100644 index 0000000000000..58ee3f169a28c --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst @@ -0,0 +1,4 @@ +Such C API functions as ``PyErr_SetString()``, ``PyErr_Format()``, +``PyErr_SetFromErrnoWithFilename()`` and many others no longer crash or +ignore errors if it failed to format the error message or decode the +filename. Instead, they keep a corresponding error. diff --git a/Python/errors.c b/Python/errors.c index 3eb8a5ef04d28..d693f3a24126a 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -225,8 +225,10 @@ _PyErr_SetString(PyThreadState *tstate, PyObject *exception, const char *string) { PyObject *value = PyUnicode_FromString(string); - _PyErr_SetObject(tstate, exception, value); - Py_XDECREF(value); + if (value != NULL) { + _PyErr_SetObject(tstate, exception, value); + Py_DECREF(value); + } } void @@ -852,7 +854,13 @@ PyErr_SetFromErrnoWithFilenameObjects(PyObject *exc, PyObject *filenameObject, P PyObject * PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) { - PyObject *name = filename ? PyUnicode_DecodeFSDefault(filename) : NULL; + PyObject *name = NULL; + if (filename) { + name = PyUnicode_DecodeFSDefault(filename); + if (name == NULL) { + return NULL; + } + } PyObject *result = PyErr_SetFromErrnoWithFilenameObjects(exc, name, NULL); Py_XDECREF(name); return result; @@ -949,7 +957,13 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilename( int ierr, const char *filename) { - PyObject *name = filename ? PyUnicode_DecodeFSDefault(filename) : NULL; + PyObject *name = NULL; + if (filename) { + name = PyUnicode_DecodeFSDefault(filename); + if (name == NULL) { + return NULL; + } + } PyObject *ret = PyErr_SetExcFromWindowsErrWithFilenameObjects(exc, ierr, name, @@ -973,7 +987,13 @@ PyObject *PyErr_SetFromWindowsErrWithFilename( int ierr, const char *filename) { - PyObject *name = filename ? PyUnicode_DecodeFSDefault(filename) : NULL; + PyObject *name = NULL; + if (filename) { + name = PyUnicode_DecodeFSDefault(filename); + if (name == NULL) { + return NULL; + } + } PyObject *result = PyErr_SetExcFromWindowsErrWithFilenameObjects( PyExc_OSError, ierr, name, NULL); @@ -1076,9 +1096,10 @@ _PyErr_FormatV(PyThreadState *tstate, PyObject *exception, _PyErr_Clear(tstate); string = PyUnicode_FromFormatV(format, vargs); - - _PyErr_SetObject(tstate, exception, string); - Py_XDECREF(string); + if (string != NULL) { + _PyErr_SetObject(tstate, exception, string); + Py_DECREF(string); + } return NULL; } From webhook-mailer at python.org Sat Aug 19 09:06:40 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Sat, 19 Aug 2023 13:06:40 -0000 Subject: [Python-checkins] [3.11] gh-107801: Improve the accuracy of os.lseek docs (#107935) (#108137) Message-ID: https://github.com/python/cpython/commit/e0ca3424ad6736600ee850348c3bba05f0c05d4b commit: e0ca3424ad6736600ee850348c3bba05f0c05d4b branch: 3.11 author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-19T13:06:36Z summary: [3.11] gh-107801: Improve the accuracy of os.lseek docs (#107935) (#108137) - name the last parameter *whence*, like it is for seek() methods on file objects - add param docstrings - structure the valid *whence* params (cherry picked from commit dd4442c8f597af1ec3eaf20f7ad89c4ac7e2dbc9) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/os.rst M Modules/clinic/posixmodule.c.h M Modules/posixmodule.c diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 993236c521a87..5bbb4089d621a 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1053,17 +1053,22 @@ as internal buffering of data. .. versionadded:: 3.11 -.. function:: lseek(fd, pos, how, /) +.. function:: lseek(fd, pos, whence, /) Set the current position of file descriptor *fd* to position *pos*, modified - by *how*: :const:`SEEK_SET` or ``0`` to set the position relative to the - beginning of the file; :const:`SEEK_CUR` or ``1`` to set it relative to the - current position; :const:`SEEK_END` or ``2`` to set it relative to the end of - the file. Return the new cursor position in bytes, starting from the beginning. + by *whence*, and return the new position in bytes relative to + the start of the file. + Valid values for *whence* are: + + * :const:`SEEK_SET` or ``0`` -- set *pos* relative to the beginning of the file + * :const:`SEEK_CUR` or ``1`` -- set *pos* relative to the current file position + * :const:`SEEK_END` or ``2`` -- set *pos* relative to the end of the file + * :const:`SEEK_HOLE` -- set *pos* to the next data location, relative to *pos* + * :const:`SEEK_DATA` -- set *pos* to the next data hole, relative to *pos* .. versionchanged:: 3.3 - Add support for :const:`SEEK_HOLE` and :const:`SEEK_DATA`. + Add support for :const:`!SEEK_HOLE` and :const:`!SEEK_DATA`. .. data:: SEEK_SET diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index b66cd857e4496..e5efd1a3a547a 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -4803,13 +4803,22 @@ os_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_LOCKF) */ PyDoc_STRVAR(os_lseek__doc__, -"lseek($module, fd, position, how, /)\n" +"lseek($module, fd, position, whence, /)\n" "--\n" "\n" "Set the position of a file descriptor. Return the new position.\n" "\n" -"Return the new cursor position in number of bytes\n" -"relative to the beginning of the file."); +" fd\n" +" An open file descriptor, as returned by os.open().\n" +" position\n" +" Position, interpreted relative to \'whence\'.\n" +" whence\n" +" The relative position to seek from. Valid values are:\n" +" - SEEK_SET: seek from the start of the file.\n" +" - SEEK_CUR: seek from the current file position.\n" +" - SEEK_END: seek from the end of the file.\n" +"\n" +"The return value is the number of bytes relative to the beginning of the file."); #define OS_LSEEK_METHODDEF \ {"lseek", _PyCFunction_CAST(os_lseek), METH_FASTCALL, os_lseek__doc__}, @@ -9379,4 +9388,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=8dd784bf1e41b881 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8277545a0dea1505 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 5ceea7411818a..cfadb6a06602f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -9521,19 +9521,24 @@ os_lockf_impl(PyObject *module, int fd, int command, Py_off_t length) os.lseek -> Py_off_t fd: int + An open file descriptor, as returned by os.open(). position: Py_off_t - how: int + Position, interpreted relative to 'whence'. + whence as how: int + The relative position to seek from. Valid values are: + - SEEK_SET: seek from the start of the file. + - SEEK_CUR: seek from the current file position. + - SEEK_END: seek from the end of the file. / Set the position of a file descriptor. Return the new position. -Return the new cursor position in number of bytes -relative to the beginning of the file. +The return value is the number of bytes relative to the beginning of the file. [clinic start generated code]*/ static Py_off_t os_lseek_impl(PyObject *module, int fd, Py_off_t position, int how) -/*[clinic end generated code: output=971e1efb6b30bd2f input=902654ad3f96a6d3]*/ +/*[clinic end generated code: output=971e1efb6b30bd2f input=f096e754c5367504]*/ { Py_off_t result; From webhook-mailer at python.org Sat Aug 19 13:03:29 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Sat, 19 Aug 2023 17:03:29 -0000 Subject: [Python-checkins] Docs: Remove links to external C functions and macros in os.rst (#108138) Message-ID: https://github.com/python/cpython/commit/c31c61c04e55ef431615ffec959d84ac73a3db81 commit: c31c61c04e55ef431615ffec959d84ac73a3db81 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-19T17:03:26Z summary: Docs: Remove links to external C functions and macros in os.rst (#108138) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/os.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 7d63ac15027d5..4f4d8c9902352 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -215,7 +215,7 @@ process and user. On some platforms, including FreeBSD and macOS, setting ``environ`` may cause memory leaks. Refer to the system documentation for - :c:func:`putenv`. + :c:func:`!putenv`. You can delete items in this mapping to unset environment variables. :func:`unsetenv` will be called automatically when an item is deleted from @@ -564,7 +564,7 @@ process and user. .. note:: On some platforms, including FreeBSD and macOS, setting ``environ`` may - cause memory leaks. Refer to the system documentation for :c:func:`putenv`. + cause memory leaks. Refer to the system documentation for :c:func:`!putenv`. .. audit-event:: os.putenv key,value os.putenv @@ -646,7 +646,7 @@ process and user. .. function:: setpgrp() - Call the system call :c:func:`setpgrp` or ``setpgrp(0, 0)`` depending on + Call the system call :c:func:`!setpgrp` or ``setpgrp(0, 0)`` depending on which version is implemented (if any). See the Unix manual for the semantics. .. availability:: Unix, not Emscripten, not WASI. @@ -654,7 +654,7 @@ process and user. .. function:: setpgid(pid, pgrp, /) - Call the system call :c:func:`setpgid` to set the process group id of the + Call the system call :c:func:`!setpgid` to set the process group id of the process with id *pid* to the process group with id *pgrp*. See the Unix manual for the semantics. @@ -1077,7 +1077,7 @@ as internal buffering of data. .. function:: fsync(fd) Force write of file with filedescriptor *fd* to disk. On Unix, this calls the - native :c:func:`fsync` function; on Windows, the MS :c:func:`_commit` function. + native :c:func:`!fsync` function; on Windows, the MS :c:func:`!_commit` function. If you're starting with a buffered Python :term:`file object` *f*, first do ``f.flush()``, and then do ``os.fsync(f.fileno())``, to ensure that all internal @@ -2304,7 +2304,7 @@ features: .. function:: lstat(path, *, dir_fd=None) - Perform the equivalent of an :c:func:`lstat` system call on the given path. + Perform the equivalent of an :c:func:`!lstat` system call on the given path. Similar to :func:`~os.stat`, but does not follow symbolic links. Return a :class:`stat_result` object. @@ -3147,14 +3147,16 @@ features: Windows file attributes: ``dwFileAttributes`` member of the ``BY_HANDLE_FILE_INFORMATION`` structure returned by - :c:func:`GetFileInformationByHandle`. See the ``FILE_ATTRIBUTE_*`` + :c:func:`!GetFileInformationByHandle`. + See the :const:`!FILE_ATTRIBUTE_* ` constants in the :mod:`stat` module. .. attribute:: st_reparse_tag - When :attr:`st_file_attributes` has the ``FILE_ATTRIBUTE_REPARSE_POINT`` + When :attr:`st_file_attributes` has the :const:`~stat.FILE_ATTRIBUTE_REPARSE_POINT` set, this field contains the tag identifying the type of reparse point. - See the ``IO_REPARSE_TAG_*`` constants in the :mod:`stat` module. + See the :const:`IO_REPARSE_TAG_* ` + constants in the :mod:`stat` module. The standard module :mod:`stat` defines functions and constants that are useful for extracting information from a :c:struct:`stat` structure. (On @@ -3212,7 +3214,7 @@ features: .. function:: statvfs(path) - Perform a :c:func:`statvfs` system call on the given path. The return value is + Perform a :c:func:`!statvfs` system call on the given path. The return value is an object whose attributes describe the filesystem on the given path, and correspond to the members of the :c:struct:`statvfs` structure, namely: :attr:`f_bsize`, :attr:`f_frsize`, :attr:`f_blocks`, :attr:`f_bfree`, @@ -4303,7 +4305,7 @@ written in Python, such as a mail server's external command delivery program. setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ setsigdef=(), scheduler=None) - Wraps the :c:func:`posix_spawn` C library API for use from Python. + Wraps the :c:func:`!posix_spawn` C library API for use from Python. Most users should use :func:`subprocess.run` instead of :func:`posix_spawn`. @@ -4339,16 +4341,16 @@ written in Python, such as a mail server's external command delivery program. Performs ``os.dup2(fd, new_fd)``. These tuples correspond to the C library - :c:func:`posix_spawn_file_actions_addopen`, - :c:func:`posix_spawn_file_actions_addclose`, and - :c:func:`posix_spawn_file_actions_adddup2` API calls used to prepare - for the :c:func:`posix_spawn` call itself. + :c:func:`!posix_spawn_file_actions_addopen`, + :c:func:`!posix_spawn_file_actions_addclose`, and + :c:func:`!posix_spawn_file_actions_adddup2` API calls used to prepare + for the :c:func:`!posix_spawn` call itself. The *setpgroup* argument will set the process group of the child to the value specified. If the value specified is 0, the child's process group ID will be made the same as its process ID. If the value of *setpgroup* is not set, the child will inherit the parent's process group ID. This argument corresponds - to the C library :c:macro:`POSIX_SPAWN_SETPGROUP` flag. + to the C library :c:macro:`!POSIX_SPAWN_SETPGROUP` flag. If the *resetids* argument is ``True`` it will reset the effective UID and GID of the child to the real UID and GID of the parent process. If the @@ -4356,27 +4358,27 @@ written in Python, such as a mail server's external command delivery program. the parent. In either case, if the set-user-ID and set-group-ID permission bits are enabled on the executable file, their effect will override the setting of the effective UID and GID. This argument corresponds to the C - library :c:macro:`POSIX_SPAWN_RESETIDS` flag. + library :c:macro:`!POSIX_SPAWN_RESETIDS` flag. If the *setsid* argument is ``True``, it will create a new session ID - for ``posix_spawn``. *setsid* requires :c:macro:`POSIX_SPAWN_SETSID` - or :c:macro:`POSIX_SPAWN_SETSID_NP` flag. Otherwise, :exc:`NotImplementedError` + for ``posix_spawn``. *setsid* requires :c:macro:`!POSIX_SPAWN_SETSID` + or :c:macro:`!POSIX_SPAWN_SETSID_NP` flag. Otherwise, :exc:`NotImplementedError` is raised. The *setsigmask* argument will set the signal mask to the signal set specified. If the parameter is not used, then the child inherits the parent's signal mask. This argument corresponds to the C library - :c:macro:`POSIX_SPAWN_SETSIGMASK` flag. + :c:macro:`!POSIX_SPAWN_SETSIGMASK` flag. The *sigdef* argument will reset the disposition of all signals in the set specified. This argument corresponds to the C library - :c:macro:`POSIX_SPAWN_SETSIGDEF` flag. + :c:macro:`!POSIX_SPAWN_SETSIGDEF` flag. The *scheduler* argument must be a tuple containing the (optional) scheduler policy and an instance of :class:`sched_param` with the scheduler parameters. A value of ``None`` in the place of the scheduler policy indicates that is not being provided. This argument is a combination of the C library - :c:macro:`POSIX_SPAWN_SETSCHEDPARAM` and :c:macro:`POSIX_SPAWN_SETSCHEDULER` + :c:macro:`!POSIX_SPAWN_SETSCHEDPARAM` and :c:macro:`!POSIX_SPAWN_SETSCHEDULER` flags. .. audit-event:: os.posix_spawn path,argv,env os.posix_spawn @@ -4389,7 +4391,7 @@ written in Python, such as a mail server's external command delivery program. setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ setsigdef=(), scheduler=None) - Wraps the :c:func:`posix_spawnp` C library API for use from Python. + Wraps the :c:func:`!posix_spawnp` C library API for use from Python. Similar to :func:`posix_spawn` except that the system searches for the *executable* file in the list of directories specified by the @@ -4570,7 +4572,7 @@ written in Python, such as a mail server's external command delivery program. Use *show_cmd* to override the default window style. Whether this has any effect will depend on the application being launched. Values are integers as - supported by the Win32 :c:func:`ShellExecute` function. + supported by the Win32 :c:func:`!ShellExecute` function. :func:`startfile` returns as soon as the associated application is launched. There is no option to wait for the application to close, and no way to retrieve @@ -4580,7 +4582,7 @@ written in Python, such as a mail server's external command delivery program. :func:`os.path.normpath` function to ensure that paths are properly encoded for Win32. - To reduce interpreter startup overhead, the Win32 :c:func:`ShellExecute` + To reduce interpreter startup overhead, the Win32 :c:func:`!ShellExecute` function is not resolved until this function is first called. If the function cannot be resolved, :exc:`NotImplementedError` will be raised. From webhook-mailer at python.org Sat Aug 19 13:11:32 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Sat, 19 Aug 2023 17:11:32 -0000 Subject: [Python-checkins] [3.11] Docs: Remove links to external C functions and macros in os.rst (GH-108138) (#108144) Message-ID: https://github.com/python/cpython/commit/49abeb5eb49e5c0f16d8d02d406763592c66f3bd commit: 49abeb5eb49e5c0f16d8d02d406763592c66f3bd branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: erlend-aasland date: 2023-08-19T17:11:29Z summary: [3.11] Docs: Remove links to external C functions and macros in os.rst (GH-108138) (#108144) (cherry picked from commit c31c61c04e55ef431615ffec959d84ac73a3db81) Co-authored-by: Erlend E. Aasland Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/os.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 5bbb4089d621a..38024f78f4f04 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -215,7 +215,7 @@ process and user. On some platforms, including FreeBSD and macOS, setting ``environ`` may cause memory leaks. Refer to the system documentation for - :c:func:`putenv`. + :c:func:`!putenv`. You can delete items in this mapping to unset environment variables. :func:`unsetenv` will be called automatically when an item is deleted from @@ -553,7 +553,7 @@ process and user. .. note:: On some platforms, including FreeBSD and macOS, setting ``environ`` may - cause memory leaks. Refer to the system documentation for :c:func:`putenv`. + cause memory leaks. Refer to the system documentation for :c:func:`!putenv`. .. audit-event:: os.putenv key,value os.putenv @@ -597,7 +597,7 @@ process and user. .. function:: setpgrp() - Call the system call :c:func:`setpgrp` or ``setpgrp(0, 0)`` depending on + Call the system call :c:func:`!setpgrp` or ``setpgrp(0, 0)`` depending on which version is implemented (if any). See the Unix manual for the semantics. .. availability:: Unix, not Emscripten, not WASI. @@ -605,7 +605,7 @@ process and user. .. function:: setpgid(pid, pgrp, /) - Call the system call :c:func:`setpgid` to set the process group id of the + Call the system call :c:func:`!setpgid` to set the process group id of the process with id *pid* to the process group with id *pgrp*. See the Unix manual for the semantics. @@ -971,7 +971,7 @@ as internal buffering of data. .. function:: fsync(fd) Force write of file with filedescriptor *fd* to disk. On Unix, this calls the - native :c:func:`fsync` function; on Windows, the MS :c:func:`_commit` function. + native :c:func:`!fsync` function; on Windows, the MS :c:func:`!_commit` function. If you're starting with a buffered Python :term:`file object` *f*, first do ``f.flush()``, and then do ``os.fsync(f.fileno())``, to ensure that all internal @@ -2127,7 +2127,7 @@ features: .. function:: lstat(path, *, dir_fd=None) - Perform the equivalent of an :c:func:`lstat` system call on the given path. + Perform the equivalent of an :c:func:`!lstat` system call on the given path. Similar to :func:`~os.stat`, but does not follow symbolic links. Return a :class:`stat_result` object. @@ -2936,14 +2936,16 @@ features: Windows file attributes: ``dwFileAttributes`` member of the ``BY_HANDLE_FILE_INFORMATION`` structure returned by - :c:func:`GetFileInformationByHandle`. See the ``FILE_ATTRIBUTE_*`` + :c:func:`!GetFileInformationByHandle`. + See the :const:`!FILE_ATTRIBUTE_* ` constants in the :mod:`stat` module. .. attribute:: st_reparse_tag - When :attr:`st_file_attributes` has the ``FILE_ATTRIBUTE_REPARSE_POINT`` + When :attr:`st_file_attributes` has the :const:`~stat.FILE_ATTRIBUTE_REPARSE_POINT` set, this field contains the tag identifying the type of reparse point. - See the ``IO_REPARSE_TAG_*`` constants in the :mod:`stat` module. + See the :const:`IO_REPARSE_TAG_* ` + constants in the :mod:`stat` module. The standard module :mod:`stat` defines functions and constants that are useful for extracting information from a :c:struct:`stat` structure. (On @@ -2982,7 +2984,7 @@ features: .. function:: statvfs(path) - Perform a :c:func:`statvfs` system call on the given path. The return value is + Perform a :c:func:`!statvfs` system call on the given path. The return value is an object whose attributes describe the filesystem on the given path, and correspond to the members of the :c:struct:`statvfs` structure, namely: :attr:`f_bsize`, :attr:`f_frsize`, :attr:`f_blocks`, :attr:`f_bfree`, @@ -4063,7 +4065,7 @@ written in Python, such as a mail server's external command delivery program. setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ setsigdef=(), scheduler=None) - Wraps the :c:func:`posix_spawn` C library API for use from Python. + Wraps the :c:func:`!posix_spawn` C library API for use from Python. Most users should use :func:`subprocess.run` instead of :func:`posix_spawn`. @@ -4099,16 +4101,16 @@ written in Python, such as a mail server's external command delivery program. Performs ``os.dup2(fd, new_fd)``. These tuples correspond to the C library - :c:func:`posix_spawn_file_actions_addopen`, - :c:func:`posix_spawn_file_actions_addclose`, and - :c:func:`posix_spawn_file_actions_adddup2` API calls used to prepare - for the :c:func:`posix_spawn` call itself. + :c:func:`!posix_spawn_file_actions_addopen`, + :c:func:`!posix_spawn_file_actions_addclose`, and + :c:func:`!posix_spawn_file_actions_adddup2` API calls used to prepare + for the :c:func:`!posix_spawn` call itself. The *setpgroup* argument will set the process group of the child to the value specified. If the value specified is 0, the child's process group ID will be made the same as its process ID. If the value of *setpgroup* is not set, the child will inherit the parent's process group ID. This argument corresponds - to the C library :c:macro:`POSIX_SPAWN_SETPGROUP` flag. + to the C library :c:macro:`!POSIX_SPAWN_SETPGROUP` flag. If the *resetids* argument is ``True`` it will reset the effective UID and GID of the child to the real UID and GID of the parent process. If the @@ -4116,27 +4118,27 @@ written in Python, such as a mail server's external command delivery program. the parent. In either case, if the set-user-ID and set-group-ID permission bits are enabled on the executable file, their effect will override the setting of the effective UID and GID. This argument corresponds to the C - library :c:macro:`POSIX_SPAWN_RESETIDS` flag. + library :c:macro:`!POSIX_SPAWN_RESETIDS` flag. If the *setsid* argument is ``True``, it will create a new session ID - for ``posix_spawn``. *setsid* requires :c:macro:`POSIX_SPAWN_SETSID` - or :c:macro:`POSIX_SPAWN_SETSID_NP` flag. Otherwise, :exc:`NotImplementedError` + for ``posix_spawn``. *setsid* requires :c:macro:`!POSIX_SPAWN_SETSID` + or :c:macro:`!POSIX_SPAWN_SETSID_NP` flag. Otherwise, :exc:`NotImplementedError` is raised. The *setsigmask* argument will set the signal mask to the signal set specified. If the parameter is not used, then the child inherits the parent's signal mask. This argument corresponds to the C library - :c:macro:`POSIX_SPAWN_SETSIGMASK` flag. + :c:macro:`!POSIX_SPAWN_SETSIGMASK` flag. The *sigdef* argument will reset the disposition of all signals in the set specified. This argument corresponds to the C library - :c:macro:`POSIX_SPAWN_SETSIGDEF` flag. + :c:macro:`!POSIX_SPAWN_SETSIGDEF` flag. The *scheduler* argument must be a tuple containing the (optional) scheduler policy and an instance of :class:`sched_param` with the scheduler parameters. A value of ``None`` in the place of the scheduler policy indicates that is not being provided. This argument is a combination of the C library - :c:macro:`POSIX_SPAWN_SETSCHEDPARAM` and :c:macro:`POSIX_SPAWN_SETSCHEDULER` + :c:macro:`!POSIX_SPAWN_SETSCHEDPARAM` and :c:macro:`!POSIX_SPAWN_SETSCHEDULER` flags. .. audit-event:: os.posix_spawn path,argv,env os.posix_spawn @@ -4149,7 +4151,7 @@ written in Python, such as a mail server's external command delivery program. setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ setsigdef=(), scheduler=None) - Wraps the :c:func:`posix_spawnp` C library API for use from Python. + Wraps the :c:func:`!posix_spawnp` C library API for use from Python. Similar to :func:`posix_spawn` except that the system searches for the *executable* file in the list of directories specified by the @@ -4330,7 +4332,7 @@ written in Python, such as a mail server's external command delivery program. Use *show_cmd* to override the default window style. Whether this has any effect will depend on the application being launched. Values are integers as - supported by the Win32 :c:func:`ShellExecute` function. + supported by the Win32 :c:func:`!ShellExecute` function. :func:`startfile` returns as soon as the associated application is launched. There is no option to wait for the application to close, and no way to retrieve @@ -4340,7 +4342,7 @@ written in Python, such as a mail server's external command delivery program. :func:`os.path.normpath` function to ensure that paths are properly encoded for Win32. - To reduce interpreter startup overhead, the Win32 :c:func:`ShellExecute` + To reduce interpreter startup overhead, the Win32 :c:func:`!ShellExecute` function is not resolved until this function is first called. If the function cannot be resolved, :exc:`NotImplementedError` will be raised. From webhook-mailer at python.org Sat Aug 19 16:14:42 2023 From: webhook-mailer at python.org (gvanrossum) Date: Sat, 19 Aug 2023 20:14:42 -0000 Subject: [Python-checkins] gh-107980: fix doc role for asyncio.timeouts (#108126) Message-ID: https://github.com/python/cpython/commit/a47c13cae5b32e6f3d7532cc6dbb4e1ac31219de commit: a47c13cae5b32e6f3d7532cc6dbb4e1ac31219de branch: main author: Tin Tvrtkovi? committer: gvanrossum date: 2023-08-19T13:14:38-07:00 summary: gh-107980: fix doc role for asyncio.timeouts (#108126) files: M Doc/library/asyncio-task.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 0651ff7213e52..1c419728c9a48 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -631,9 +631,9 @@ Shielding From Cancellation Timeouts ======== -.. coroutinefunction:: timeout(delay) +.. function:: timeout(delay) - An :ref:`asynchronous context manager ` + Return an :ref:`asynchronous context manager ` that can be used to limit the amount of time spent waiting on something. @@ -724,7 +724,7 @@ Timeouts .. versionadded:: 3.11 -.. coroutinefunction:: timeout_at(when) +.. function:: timeout_at(when) Similar to :func:`asyncio.timeout`, except *when* is the absolute time to stop waiting, or ``None``. From webhook-mailer at python.org Sat Aug 19 18:59:35 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sat, 19 Aug 2023 22:59:35 -0000 Subject: [Python-checkins] [3.12] gh-107565: Update macOS installer to use OpenSSL 3.0.10. (GH-107897) (#108121) Message-ID: https://github.com/python/cpython/commit/a1808b835876ee70020ad9dbba40614439a5cdde commit: a1808b835876ee70020ad9dbba40614439a5cdde branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-20T00:59:32+02:00 summary: [3.12] gh-107565: Update macOS installer to use OpenSSL 3.0.10. (GH-107897) (#108121) gh-107565: Update macOS installer to use OpenSSL 3.0.10. (GH-107897) (cherry picked from commit dc7b630b2359663bb7b8212d9f2f720c978d3daa) Co-authored-by: Ned Deily files: A Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst M Mac/BuildScript/build-installer.py diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 0fc25ec74565d..c108ee21a7a2a 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.9", - url="https://www.openssl.org/source/openssl-3.0.9.tar.gz", - checksum='eb1ab04781474360f77c318ab89d8c5a03abc38e63d65a603cabbf1b00a1dc90', + name="OpenSSL 3.0.10", + url="https://www.openssl.org/source/openssl-3.0.10.tar.gz", + checksum='1761d4f5b13a1028b9b6f3d4b8e17feb0cedc9370f6afe61d7193d2cdce83323', buildrecipe=build_universal_openssl, configure=None, install=None, diff --git a/Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst b/Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst new file mode 100644 index 0000000000000..c238c4760239e --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst @@ -0,0 +1 @@ +Update macOS installer to use OpenSSL 3.0.10. From webhook-mailer at python.org Sat Aug 19 19:00:13 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sat, 19 Aug 2023 23:00:13 -0000 Subject: [Python-checkins] [3.12] gh-101100: Only show GitHub check annotations on changed doc paragraphs (GH-108065) (#108127) Message-ID: https://github.com/python/cpython/commit/71e3581c969e8a7dbfd21db2407af498fd0027e3 commit: 71e3581c969e8a7dbfd21db2407af498fd0027e3 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-20T01:00:09+02:00 summary: [3.12] gh-101100: Only show GitHub check annotations on changed doc paragraphs (GH-108065) (#108127) gh-101100: Only show GitHub check annotations on changed doc paragraphs (GH-108065) * Only show GitHub check annotations on changed doc paragraphs * Improve check-warnings script arg parsing following Hugo's suggestions * Factor filtering warnings by modified diffs into helper function * Build docs on unmerged branch so warning lines match & avoid deep clone --------- (cherry picked from commit eb953d6e4484339067837020f77eecac61f8d4f8) Co-authored-by: C.A.M. Gerlach Co-authored-by: Hugo van Kemenade Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M .github/workflows/reusable-docs.yml M Doc/tools/check-warnings.py diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 39e5ad62924ad..6150b1a7d416a 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -16,8 +16,30 @@ jobs: name: 'Docs' runs-on: ubuntu-latest timeout-minutes: 60 + env: + branch_base: 'origin/${{ github.event.pull_request.base.ref }}' + branch_pr: 'origin/${{ github.event.pull_request.head.ref }}' + refspec_base: '+${{ github.event.pull_request.base.sha }}:remotes/origin/${{ github.event.pull_request.base.ref }}' + refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - - uses: actions/checkout at v3 + - name: 'Check out latest PR branch commit' + uses: actions/checkout at v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721 + - name: 'Fetch commits to get branch diff' + run: | + # Fetch enough history to find a common ancestor commit (aka merge-base): + git fetch origin ${{ env.refspec_pr }} --depth=$(( ${{ github.event.pull_request.commits }} + 1 )) \ + --no-tags --prune --no-recurse-submodules + + # This should get the oldest commit in the local fetched history (which may not be the commit the PR branched from): + COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 ${{ env.branch_pr }} ) + DATE=$( git log --date=iso8601 --format=%cd "${COMMON_ANCESTOR}" ) + + # Get all commits since that commit date from the base branch (eg: master or main): + git fetch origin ${{ env.refspec_base }} --shallow-since="${DATE}" \ + --no-tags --prune --no-recurse-submodules - name: 'Set up Python' uses: actions/setup-python at v4 with: @@ -28,13 +50,6 @@ jobs: run: make -C Doc/ venv # To annotate PRs with Sphinx nitpicks (missing references) - - name: 'Get list of changed files' - if: github.event_name == 'pull_request' - id: changed_files - uses: Ana06/get-changed-files at v2.2.0 - with: - filter: "Doc/**" - format: csv # works for paths with spaces - name: 'Build HTML documentation' continue-on-error: true run: | @@ -45,7 +60,7 @@ jobs: if: github.event_name == 'pull_request' run: | python Doc/tools/check-warnings.py \ - --check-and-annotate '${{ steps.changed_files.outputs.added_modified }}' \ + --annotate-diff '${{ env.branch_base }}' '${{ env.branch_pr }}' \ --fail-if-regression \ --fail-if-improved diff --git a/Doc/tools/check-warnings.py b/Doc/tools/check-warnings.py index c17d0f51cd127..809a8d63087e1 100644 --- a/Doc/tools/check-warnings.py +++ b/Doc/tools/check-warnings.py @@ -2,12 +2,16 @@ """ Check the output of running Sphinx in nit-picky mode (missing references). """ +from __future__ import annotations + import argparse -import csv +import itertools import os import re +import subprocess import sys from pathlib import Path +from typing import TextIO # Exclude these whether they're dirty or clean, # because they trigger a rebuild of dirty files. @@ -24,28 +28,178 @@ "venv", } -PATTERN = re.compile(r"(?P[^:]+):(?P\d+): WARNING: (?P.+)") +# Regex pattern to match the parts of a Sphinx warning +WARNING_PATTERN = re.compile( + r"(?P([A-Za-z]:[\\/])?[^:]+):(?P\d+): WARNING: (?P.+)" +) + +# Regex pattern to match the line numbers in a Git unified diff +DIFF_PATTERN = re.compile( + r"^@@ -(?P\d+)(?:,(?P\d+))? \+(?P\d+)(?:,(?P\d+))? @@", + flags=re.MULTILINE, +) + + +def get_diff_files(ref_a: str, ref_b: str, filter_mode: str = "") -> set[Path]: + """List the files changed between two Git refs, filtered by change type.""" + added_files_result = subprocess.run( + [ + "git", + "diff", + f"--diff-filter={filter_mode}", + "--name-only", + f"{ref_a}...{ref_b}", + "--", + ], + stdout=subprocess.PIPE, + check=True, + text=True, + encoding="UTF-8", + ) + + added_files = added_files_result.stdout.strip().split("\n") + return {Path(file.strip()) for file in added_files if file.strip()} + + +def get_diff_lines(ref_a: str, ref_b: str, file: Path) -> list[int]: + """List the lines changed between two Git refs for a specific file.""" + diff_output = subprocess.run( + [ + "git", + "diff", + "--unified=0", + f"{ref_a}...{ref_b}", + "--", + str(file), + ], + stdout=subprocess.PIPE, + check=True, + text=True, + encoding="UTF-8", + ) + + # Scrape line offsets + lengths from diff and convert to line numbers + line_matches = DIFF_PATTERN.finditer(diff_output.stdout) + # Removed and added line counts are 1 if not printed + line_match_values = [ + line_match.groupdict(default=1) for line_match in line_matches + ] + line_ints = [ + (int(match_value["lineb"]), int(match_value["added"])) + for match_value in line_match_values + ] + line_ranges = [ + range(line_b, line_b + added) for line_b, added in line_ints + ] + line_numbers = list(itertools.chain(*line_ranges)) + + return line_numbers + + +def get_para_line_numbers(file_obj: TextIO) -> list[list[int]]: + """Get the line numbers of text in a file object, grouped by paragraph.""" + paragraphs = [] + prev_line = None + for lineno, line in enumerate(file_obj): + lineno = lineno + 1 + if prev_line is None or (line.strip() and not prev_line.strip()): + paragraph = [lineno - 1] + paragraphs.append(paragraph) + paragraph.append(lineno) + prev_line = line + return paragraphs + + +def filter_and_parse_warnings( + warnings: list[str], files: set[Path] +) -> list[re.Match[str]]: + """Get the warnings matching passed files and parse them with regex.""" + filtered_warnings = [ + warning + for warning in warnings + if any(str(file) in warning for file in files) + ] + warning_matches = [ + WARNING_PATTERN.fullmatch(warning.strip()) + for warning in filtered_warnings + ] + non_null_matches = [warning for warning in warning_matches if warning] + return non_null_matches + + +def filter_warnings_by_diff( + warnings: list[re.Match[str]], ref_a: str, ref_b: str, file: Path +) -> list[re.Match[str]]: + """Filter the passed per-file warnings to just those on changed lines.""" + diff_lines = get_diff_lines(ref_a, ref_b, file) + with file.open(encoding="UTF-8") as file_obj: + paragraphs = get_para_line_numbers(file_obj) + touched_paras = [ + para_lines + for para_lines in paragraphs + if set(diff_lines) & set(para_lines) + ] + touched_para_lines = set(itertools.chain(*touched_paras)) + warnings_infile = [ + warning for warning in warnings if str(file) in warning["file"] + ] + warnings_touched = [ + warning + for warning in warnings_infile + if int(warning["line"]) in touched_para_lines + ] + return warnings_touched + +def process_touched_warnings( + warnings: list[str], ref_a: str, ref_b: str +) -> list[re.Match[str]]: + """Filter a list of Sphinx warnings to those affecting touched lines.""" + added_files, modified_files = tuple( + get_diff_files(ref_a, ref_b, filter_mode=mode) for mode in ("A", "M") + ) + + warnings_added = filter_and_parse_warnings(warnings, added_files) + warnings_modified = filter_and_parse_warnings(warnings, modified_files) + + modified_files_warned = { + file + for file in modified_files + if any(str(file) in warning["file"] for warning in warnings_modified) + } -def check_and_annotate(warnings: list[str], files_to_check: str) -> None: + warnings_modified_touched = [ + filter_warnings_by_diff(warnings_modified, ref_a, ref_b, file) + for file in modified_files_warned + ] + warnings_touched = warnings_added + list( + itertools.chain(*warnings_modified_touched) + ) + + return warnings_touched + + +def annotate_diff( + warnings: list[str], ref_a: str = "main", ref_b: str = "HEAD" +) -> None: """ - Convert Sphinx warning messages to GitHub Actions. + Convert Sphinx warning messages to GitHub Actions for changed paragraphs. Converts lines like: .../Doc/library/cgi.rst:98: WARNING: reference target not found to: ::warning file=.../Doc/library/cgi.rst,line=98::reference target not found - Non-matching lines are echoed unchanged. - - see: + See: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-a-warning-message """ - files_to_check = next(csv.reader([files_to_check])) - for warning in warnings: - if any(filename in warning for filename in files_to_check): - if match := PATTERN.fullmatch(warning): - print("::warning file={file},line={line}::{msg}".format_map(match)) + warnings_touched = process_touched_warnings(warnings, ref_a, ref_b) + print("Emitting doc warnings matching modified lines:") + for warning in warnings_touched: + print("::warning file={file},line={line}::{msg}".format_map(warning)) + print(warning[0]) + if not warnings_touched: + print("None") def fail_if_regression( @@ -68,7 +222,7 @@ def fail_if_regression( print(filename) for warning in warnings: if filename in warning: - if match := PATTERN.fullmatch(warning): + if match := WARNING_PATTERN.fullmatch(warning): print(" {line}: {msg}".format_map(match)) return -1 return 0 @@ -91,12 +245,14 @@ def fail_if_improved( return 0 -def main() -> int: +def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() parser.add_argument( - "--check-and-annotate", - help="Comma-separated list of files to check, " - "and annotate those with warnings on GitHub Actions", + "--annotate-diff", + nargs="*", + metavar=("BASE_REF", "HEAD_REF"), + help="Add GitHub Actions annotations on the diff for warnings on " + "lines changed between the given refs (main and HEAD, by default)", ) parser.add_argument( "--fail-if-regression", @@ -108,13 +264,19 @@ def main() -> int: action="store_true", help="Fail if new files with no nits are found", ) - args = parser.parse_args() + + args = parser.parse_args(argv) + if args.annotate_diff is not None and len(args.annotate_diff) > 2: + parser.error( + "--annotate-diff takes between 0 and 2 ref args, not " + f"{len(args.annotate_diff)} {tuple(args.annotate_diff)}" + ) exit_code = 0 wrong_directory_msg = "Must run this script from the repo root" assert Path("Doc").exists() and Path("Doc").is_dir(), wrong_directory_msg - with Path("Doc/sphinx-warnings.txt").open() as f: + with Path("Doc/sphinx-warnings.txt").open(encoding="UTF-8") as f: warnings = f.read().splitlines() cwd = str(Path.cwd()) + os.path.sep @@ -124,15 +286,15 @@ def main() -> int: if "Doc/" in warning } - with Path("Doc/tools/.nitignore").open() as clean_files: + with Path("Doc/tools/.nitignore").open(encoding="UTF-8") as clean_files: files_with_expected_nits = { filename.strip() for filename in clean_files if filename.strip() and not filename.startswith("#") } - if args.check_and_annotate: - check_and_annotate(warnings, args.check_and_annotate) + if args.annotate_diff is not None: + annotate_diff(warnings, *args.annotate_diff) if args.fail_if_regression: exit_code += fail_if_regression( From webhook-mailer at python.org Sat Aug 19 19:00:34 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sat, 19 Aug 2023 23:00:34 -0000 Subject: [Python-checkins] [3.12] Docs: format sys.float_info properly (GH-108107) (#108130) Message-ID: https://github.com/python/cpython/commit/28074306578fb7e67667ee64cd7d66509d63c21c commit: 28074306578fb7e67667ee64cd7d66509d63c21c branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-20T01:00:31+02:00 summary: [3.12] Docs: format sys.float_info properly (GH-108107) (#108130) Docs: format sys.float_info properly (GH-108107) - Normalise capitalisation and punctuation - Use attribute markup for named tuple attributes - Use :c:macro: markup for C macros - Use a list for the 'rounds' attribute values - Use list-table, for better .rst readability - Remove one unneeded sys.float_info.dig link (cherry picked from commit ca0c6c1f1ef79d10bc49b61d638d87cde265aa94) Co-authored-by: Erlend E. Aasland Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/sys.rst diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index b6346bafbf625..334a2b077dd6f 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -574,61 +574,82 @@ always available. programming language; see section 5.2.4.2.2 of the 1999 ISO/IEC C standard [C99]_, 'Characteristics of floating types', for details. - .. tabularcolumns:: |l|l|L| - - +---------------------+---------------------+--------------------------------------------------+ - | attribute | float.h macro | explanation | - +=====================+=====================+==================================================+ - | ``epsilon`` | ``DBL_EPSILON`` | difference between 1.0 and the least value | - | | | greater than 1.0 that is representable as a float| - | | | | - | | | See also :func:`math.ulp`. | - +---------------------+---------------------+--------------------------------------------------+ - | ``dig`` | ``DBL_DIG`` | maximum number of decimal digits that can be | - | | | faithfully represented in a float; see below | - +---------------------+---------------------+--------------------------------------------------+ - | ``mant_dig`` | ``DBL_MANT_DIG`` | float precision: the number of base-``radix`` | - | | | digits in the significand of a float | - +---------------------+---------------------+--------------------------------------------------+ - | ``max`` | ``DBL_MAX`` | maximum representable positive finite float | - +---------------------+---------------------+--------------------------------------------------+ - | ``max_exp`` | ``DBL_MAX_EXP`` | maximum integer *e* such that ``radix**(e-1)`` is| - | | | a representable finite float | - +---------------------+---------------------+--------------------------------------------------+ - | ``max_10_exp`` | ``DBL_MAX_10_EXP`` | maximum integer *e* such that ``10**e`` is in the| - | | | range of representable finite floats | - +---------------------+---------------------+--------------------------------------------------+ - | ``min`` | ``DBL_MIN`` | minimum representable positive *normalized* float| - | | | | - | | | Use :func:`math.ulp(0.0) ` to get the | - | | | smallest positive *denormalized* representable | - | | | float. | - +---------------------+---------------------+--------------------------------------------------+ - | ``min_exp`` | ``DBL_MIN_EXP`` | minimum integer *e* such that ``radix**(e-1)`` is| - | | | a normalized float | - +---------------------+---------------------+--------------------------------------------------+ - | ``min_10_exp`` | ``DBL_MIN_10_EXP`` | minimum integer *e* such that ``10**e`` is a | - | | | normalized float | - +---------------------+---------------------+--------------------------------------------------+ - | ``radix`` | ``FLT_RADIX`` | radix of exponent representation | - +---------------------+---------------------+--------------------------------------------------+ - | ``rounds`` | ``FLT_ROUNDS`` | integer representing the rounding mode for | - | | | floating-point arithmetic. This reflects the | - | | | value of the system ``FLT_ROUNDS`` macro at | - | | | interpreter startup time: | - | | | ``-1`` indeterminable, | - | | | ``0`` toward zero, | - | | | ``1`` to nearest, | - | | | ``2`` toward positive infinity, | - | | | ``3`` toward negative infinity | - | | | | - | | | All other values for ``FLT_ROUNDS`` characterize | - | | | implementation-defined rounding behavior. | - +---------------------+---------------------+--------------------------------------------------+ + .. list-table:: Attributes of the :data:`!float_info` :term:`named tuple` + :header-rows: 1 + + * - attribute + - float.h macro + - explanation + + * - .. attribute:: float_info.epsilon + - :c:macro:`!DBL_EPSILON` + - difference between 1.0 and the least value greater than 1.0 that is + representable as a float. + + See also :func:`math.ulp`. + + * - .. attribute:: float_info.dig + - :c:macro:`!DBL_DIG` + - The maximum number of decimal digits that can be faithfully + represented in a float; see below. + + * - .. attribute:: float_info.mant_dig + - :c:macro:`!DBL_MANT_DIG` + - Float precision: the number of base-``radix`` digits in the + significand of a float. + + * - .. attribute:: float_info.max + - :c:macro:`!DBL_MAX` + - The maximum representable positive finite float. + + * - .. attribute:: float_info.max_exp + - :c:macro:`!DBL_MAX_EXP` + - The maximum integer *e* such that ``radix**(e-1)`` is a representable + finite float. + + * - .. attribute:: float_info.max_10_exp + - :c:macro:`!DBL_MAX_10_EXP` + - The maximum integer *e* such that ``10**e`` is in the range of + representable finite floats. + + * - .. attribute:: float_info.min + - :c:macro:`!DBL_MIN` + - The minimum representable positive *normalized* float. + + Use :func:`math.ulp(0.0) ` to get the smallest positive + *denormalized* representable float. + + * - .. attribute:: float_info.min_exp + - :c:macro:`!DBL_MIN_EXP` + - The minimum integer *e* such that ``radix**(e-1)`` is a normalized + float. + + * - .. attribute:: float_info.min_10_exp + - :c:macro:`!DBL_MIN_10_EXP` + - The minimum integer *e* such that ``10**e`` is a normalized float. + + * - .. attribute:: float_info.radix + - :c:macro:`!FLT_RADIX` + - The radix of exponent representation. + + * - .. attribute:: float_info.rounds + - :c:macro:`!FLT_ROUNDS` + - An integer representing the rounding mode for floating-point arithmetic. + This reflects the value of the system :c:macro:`!FLT_ROUNDS` macro + at interpreter startup time: + + * ``-1``: indeterminable + * ``0``: toward zero + * ``1``: to nearest + * ``2``: toward positive infinity + * ``3``: toward negative infinity + + All other values for :c:macro:`!FLT_ROUNDS` characterize + implementation-defined rounding behavior. The attribute :attr:`sys.float_info.dig` needs further explanation. If ``s`` is any string representing a decimal number with at most - :attr:`sys.float_info.dig` significant digits, then converting ``s`` to a + :attr:`!sys.float_info.dig` significant digits, then converting ``s`` to a float and back again will recover a string representing the same decimal value:: From webhook-mailer at python.org Sat Aug 19 19:02:15 2023 From: webhook-mailer at python.org (ned-deily) Date: Sat, 19 Aug 2023 23:02:15 -0000 Subject: [Python-checkins] [3.11] gh-107565: Update macOS installer to use OpenSSL 3.0.10. (GH-107897) (#108122) Message-ID: https://github.com/python/cpython/commit/f8b14fea33a5566dbb369ef9aacea12924de2c7a commit: f8b14fea33a5566dbb369ef9aacea12924de2c7a branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ned-deily date: 2023-08-19T19:02:12-04:00 summary: [3.11] gh-107565: Update macOS installer to use OpenSSL 3.0.10. (GH-107897) (#108122) files: A Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst M Mac/BuildScript/build-installer.py diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 71fbe3c0777e1..c10daa0a2dd07 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.9", - url="https://www.openssl.org/source/openssl-3.0.9.tar.gz", - checksum='eb1ab04781474360f77c318ab89d8c5a03abc38e63d65a603cabbf1b00a1dc90', + name="OpenSSL 3.0.10", + url="https://www.openssl.org/source/openssl-3.0.10.tar.gz", + checksum='1761d4f5b13a1028b9b6f3d4b8e17feb0cedc9370f6afe61d7193d2cdce83323', buildrecipe=build_universal_openssl, configure=None, install=None, diff --git a/Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst b/Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst new file mode 100644 index 0000000000000..c238c4760239e --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-08-12-13-33-57.gh-issue-107565.SJwqf4.rst @@ -0,0 +1 @@ +Update macOS installer to use OpenSSL 3.0.10. From webhook-mailer at python.org Sat Aug 19 19:05:03 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sat, 19 Aug 2023 23:05:03 -0000 Subject: [Python-checkins] [3.12] gh-107801: Improve the accuracy of os.lseek docs (#107935) (#108136) Message-ID: https://github.com/python/cpython/commit/41634edb2b54f488aac286b938a3590f5dac154c commit: 41634edb2b54f488aac286b938a3590f5dac154c branch: 3.12 author: Erlend E. Aasland committer: Yhg1s date: 2023-08-20T01:05:00+02:00 summary: [3.12] gh-107801: Improve the accuracy of os.lseek docs (#107935) (#108136) - name the last parameter *whence*, like it is for seek() methods on file objects - add param docstrings - structure the valid *whence* params (cherry picked from commit dd4442c8f597af1ec3eaf20f7ad89c4ac7e2dbc9) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/os.rst M Modules/clinic/posixmodule.c.h M Modules/posixmodule.c diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 4668984b10c69..7d63ac15027d5 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1163,17 +1163,22 @@ as internal buffering of data. .. versionadded:: 3.11 -.. function:: lseek(fd, pos, how, /) +.. function:: lseek(fd, pos, whence, /) Set the current position of file descriptor *fd* to position *pos*, modified - by *how*: :const:`SEEK_SET` or ``0`` to set the position relative to the - beginning of the file; :const:`SEEK_CUR` or ``1`` to set it relative to the - current position; :const:`SEEK_END` or ``2`` to set it relative to the end of - the file. Return the new cursor position in bytes, starting from the beginning. + by *whence*, and return the new position in bytes relative to + the start of the file. + Valid values for *whence* are: + + * :const:`SEEK_SET` or ``0`` -- set *pos* relative to the beginning of the file + * :const:`SEEK_CUR` or ``1`` -- set *pos* relative to the current file position + * :const:`SEEK_END` or ``2`` -- set *pos* relative to the end of the file + * :const:`SEEK_HOLE` -- set *pos* to the next data location, relative to *pos* + * :const:`SEEK_DATA` -- set *pos* to the next data hole, relative to *pos* .. versionchanged:: 3.3 - Add support for :const:`SEEK_HOLE` and :const:`SEEK_DATA`. + Add support for :const:`!SEEK_HOLE` and :const:`!SEEK_DATA`. .. data:: SEEK_SET diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 3312bd667694d..5924c4ab03943 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -6496,13 +6496,22 @@ os_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_LOCKF) */ PyDoc_STRVAR(os_lseek__doc__, -"lseek($module, fd, position, how, /)\n" +"lseek($module, fd, position, whence, /)\n" "--\n" "\n" "Set the position of a file descriptor. Return the new position.\n" "\n" -"Return the new cursor position in number of bytes\n" -"relative to the beginning of the file."); +" fd\n" +" An open file descriptor, as returned by os.open().\n" +" position\n" +" Position, interpreted relative to \'whence\'.\n" +" whence\n" +" The relative position to seek from. Valid values are:\n" +" - SEEK_SET: seek from the start of the file.\n" +" - SEEK_CUR: seek from the current file position.\n" +" - SEEK_END: seek from the end of the file.\n" +"\n" +"The return value is the number of bytes relative to the beginning of the file."); #define OS_LSEEK_METHODDEF \ {"lseek", _PyCFunction_CAST(os_lseek), METH_FASTCALL, os_lseek__doc__}, @@ -11990,4 +11999,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=9d8b0d6717c9af54 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6646be70849f971f input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index b9f45c0ce5543..44cc31bd4bd70 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -10421,19 +10421,24 @@ os_lockf_impl(PyObject *module, int fd, int command, Py_off_t length) os.lseek -> Py_off_t fd: int + An open file descriptor, as returned by os.open(). position: Py_off_t - how: int + Position, interpreted relative to 'whence'. + whence as how: int + The relative position to seek from. Valid values are: + - SEEK_SET: seek from the start of the file. + - SEEK_CUR: seek from the current file position. + - SEEK_END: seek from the end of the file. / Set the position of a file descriptor. Return the new position. -Return the new cursor position in number of bytes -relative to the beginning of the file. +The return value is the number of bytes relative to the beginning of the file. [clinic start generated code]*/ static Py_off_t os_lseek_impl(PyObject *module, int fd, Py_off_t position, int how) -/*[clinic end generated code: output=971e1efb6b30bd2f input=902654ad3f96a6d3]*/ +/*[clinic end generated code: output=971e1efb6b30bd2f input=f096e754c5367504]*/ { Py_off_t result; From webhook-mailer at python.org Sat Aug 19 19:07:44 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sat, 19 Aug 2023 23:07:44 -0000 Subject: [Python-checkins] [3.12] gh-108083: Don't ignore exceptions in sqlite3.Connection.__init__() and .close() (#108084) (#108141) Message-ID: https://github.com/python/cpython/commit/0c21298f2bc17966498f13995586890488a6f8f1 commit: 0c21298f2bc17966498f13995586890488a6f8f1 branch: 3.12 author: Erlend E. Aasland committer: Yhg1s date: 2023-08-20T01:07:41+02:00 summary: [3.12] gh-108083: Don't ignore exceptions in sqlite3.Connection.__init__() and .close() (#108084) (#108141) - Add explanatory comments - Add return value to connection_close() for propagating errors - Always check the return value of connection_exec_stmt() - Assert pre/post state in remove_callbacks() - Don't log unraisable exceptions in case of interpreter shutdown - Make sure we're not initialized if reinit fails - Try to close the database even if ROLLBACK fails (cherry picked from commit fd195092204aa7fc9f13c5c6d423bc723d0b3520) Co-authored-by: Victor Stinner Co-authored-by: Serhiy Storchaka files: A Misc/NEWS.d/next/Library/2023-08-17-12-59-35.gh-issue-108083.9J7UcT.rst M Modules/_sqlite/connection.c diff --git a/Misc/NEWS.d/next/Library/2023-08-17-12-59-35.gh-issue-108083.9J7UcT.rst b/Misc/NEWS.d/next/Library/2023-08-17-12-59-35.gh-issue-108083.9J7UcT.rst new file mode 100644 index 0000000000000..ff499ced73a30 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-17-12-59-35.gh-issue-108083.9J7UcT.rst @@ -0,0 +1,3 @@ +Fix bugs in the constructor of :mod:`sqlite3.Connection` and +:meth:`sqlite3.Connection.close` where exceptions could be leaked. Patch by +Erlend E. Aasland. diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 82d23c2c30b79..12e5c135aafa5 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -146,7 +146,7 @@ static void _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self); static void free_callback_context(callback_context *ctx); static void set_callback_context(callback_context **ctx_pp, callback_context *ctx); -static void connection_close(pysqlite_Connection *self); +static int connection_close(pysqlite_Connection *self); PyObject *_pysqlite_query_execute(pysqlite_Cursor *, int, PyObject *, PyObject *); static PyObject * @@ -244,10 +244,13 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, } if (self->initialized) { + self->initialized = 0; + PyTypeObject *tp = Py_TYPE(self); tp->tp_clear((PyObject *)self); - connection_close(self); - self->initialized = 0; + if (connection_close(self) < 0) { + return -1; + } } // Create and configure SQLite database object. @@ -331,7 +334,9 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, self->initialized = 1; if (autocommit == AUTOCOMMIT_DISABLED) { - (void)connection_exec_stmt(self, "BEGIN"); + if (connection_exec_stmt(self, "BEGIN") < 0) { + return -1; + } } return 0; @@ -401,52 +406,88 @@ free_callback_contexts(pysqlite_Connection *self) static void remove_callbacks(sqlite3 *db) { + assert(db != NULL); + /* None of these APIs can fail, as long as they are given a valid + * database pointer. */ + int rc; #ifdef HAVE_TRACE_V2 - sqlite3_trace_v2(db, SQLITE_TRACE_STMT, 0, 0); + rc = sqlite3_trace_v2(db, SQLITE_TRACE_STMT, 0, 0); + assert(rc == SQLITE_OK), (void)rc; #else sqlite3_trace(db, 0, (void*)0); #endif + sqlite3_progress_handler(db, 0, 0, (void *)0); - (void)sqlite3_set_authorizer(db, NULL, NULL); + + rc = sqlite3_set_authorizer(db, NULL, NULL); + assert(rc == SQLITE_OK), (void)rc; } -static void +static int connection_close(pysqlite_Connection *self) { - if (self->db) { - if (self->autocommit == AUTOCOMMIT_DISABLED && - !sqlite3_get_autocommit(self->db)) - { - /* If close is implicitly called as a result of interpreter - * tear-down, we must not call back into Python. */ - if (_Py_IsInterpreterFinalizing(PyInterpreterState_Get())) { - remove_callbacks(self->db); - } - (void)connection_exec_stmt(self, "ROLLBACK"); + if (self->db == NULL) { + return 0; + } + + int rc = 0; + if (self->autocommit == AUTOCOMMIT_DISABLED && + !sqlite3_get_autocommit(self->db)) + { + if (connection_exec_stmt(self, "ROLLBACK") < 0) { + rc = -1; } + } - free_callback_contexts(self); + sqlite3 *db = self->db; + self->db = NULL; - sqlite3 *db = self->db; - self->db = NULL; + Py_BEGIN_ALLOW_THREADS + /* The v2 close call always returns SQLITE_OK if given a valid database + * pointer (which we do), so we can safely ignore the return value */ + (void)sqlite3_close_v2(db); + Py_END_ALLOW_THREADS - Py_BEGIN_ALLOW_THREADS - int rc = sqlite3_close_v2(db); - assert(rc == SQLITE_OK), (void)rc; - Py_END_ALLOW_THREADS - } + free_callback_contexts(self); + return rc; } static void -connection_dealloc(pysqlite_Connection *self) +connection_finalize(PyObject *self) { - PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); - tp->tp_clear((PyObject *)self); + pysqlite_Connection *con = (pysqlite_Connection *)self; + PyObject *exc = PyErr_GetRaisedException(); + + /* If close is implicitly called as a result of interpreter + * tear-down, we must not call back into Python. */ + PyInterpreterState *interp = PyInterpreterState_Get(); + int teardown = _Py_IsInterpreterFinalizing(interp); + if (teardown && con->db) { + remove_callbacks(con->db); + } /* Clean up if user has not called .close() explicitly. */ - connection_close(self); + if (connection_close(con) < 0) { + if (teardown) { + PyErr_Clear(); + } + else { + PyErr_WriteUnraisable((PyObject *)self); + } + } + + PyErr_SetRaisedException(exc); +} +static void +connection_dealloc(PyObject *self) +{ + if (PyObject_CallFinalizerFromDealloc(self) < 0) { + return; + } + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + tp->tp_clear(self); tp->tp_free(self); Py_DECREF(tp); } @@ -594,7 +635,9 @@ pysqlite_connection_close_impl(pysqlite_Connection *self) pysqlite_close_all_blobs(self); Py_CLEAR(self->statement_cache); - connection_close(self); + if (connection_close(self) < 0) { + return NULL; + } Py_RETURN_NONE; } @@ -2582,6 +2625,7 @@ static struct PyMemberDef connection_members[] = }; static PyType_Slot connection_slots[] = { + {Py_tp_finalize, connection_finalize}, {Py_tp_dealloc, connection_dealloc}, {Py_tp_doc, (void *)connection_doc}, {Py_tp_methods, connection_methods}, From webhook-mailer at python.org Sat Aug 19 19:08:18 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sat, 19 Aug 2023 23:08:18 -0000 Subject: [Python-checkins] [3.12] Docs: Remove links to external C functions and macros in os.rst (GH-108138) (#108143) Message-ID: https://github.com/python/cpython/commit/540c0d99eabcdaedd7f6b56406abc08a628188b1 commit: 540c0d99eabcdaedd7f6b56406abc08a628188b1 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-20T01:08:14+02:00 summary: [3.12] Docs: Remove links to external C functions and macros in os.rst (GH-108138) (#108143) Docs: Remove links to external C functions and macros in os.rst (GH-108138) (cherry picked from commit c31c61c04e55ef431615ffec959d84ac73a3db81) Co-authored-by: Erlend E. Aasland Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/os.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 7d63ac15027d5..4f4d8c9902352 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -215,7 +215,7 @@ process and user. On some platforms, including FreeBSD and macOS, setting ``environ`` may cause memory leaks. Refer to the system documentation for - :c:func:`putenv`. + :c:func:`!putenv`. You can delete items in this mapping to unset environment variables. :func:`unsetenv` will be called automatically when an item is deleted from @@ -564,7 +564,7 @@ process and user. .. note:: On some platforms, including FreeBSD and macOS, setting ``environ`` may - cause memory leaks. Refer to the system documentation for :c:func:`putenv`. + cause memory leaks. Refer to the system documentation for :c:func:`!putenv`. .. audit-event:: os.putenv key,value os.putenv @@ -646,7 +646,7 @@ process and user. .. function:: setpgrp() - Call the system call :c:func:`setpgrp` or ``setpgrp(0, 0)`` depending on + Call the system call :c:func:`!setpgrp` or ``setpgrp(0, 0)`` depending on which version is implemented (if any). See the Unix manual for the semantics. .. availability:: Unix, not Emscripten, not WASI. @@ -654,7 +654,7 @@ process and user. .. function:: setpgid(pid, pgrp, /) - Call the system call :c:func:`setpgid` to set the process group id of the + Call the system call :c:func:`!setpgid` to set the process group id of the process with id *pid* to the process group with id *pgrp*. See the Unix manual for the semantics. @@ -1077,7 +1077,7 @@ as internal buffering of data. .. function:: fsync(fd) Force write of file with filedescriptor *fd* to disk. On Unix, this calls the - native :c:func:`fsync` function; on Windows, the MS :c:func:`_commit` function. + native :c:func:`!fsync` function; on Windows, the MS :c:func:`!_commit` function. If you're starting with a buffered Python :term:`file object` *f*, first do ``f.flush()``, and then do ``os.fsync(f.fileno())``, to ensure that all internal @@ -2304,7 +2304,7 @@ features: .. function:: lstat(path, *, dir_fd=None) - Perform the equivalent of an :c:func:`lstat` system call on the given path. + Perform the equivalent of an :c:func:`!lstat` system call on the given path. Similar to :func:`~os.stat`, but does not follow symbolic links. Return a :class:`stat_result` object. @@ -3147,14 +3147,16 @@ features: Windows file attributes: ``dwFileAttributes`` member of the ``BY_HANDLE_FILE_INFORMATION`` structure returned by - :c:func:`GetFileInformationByHandle`. See the ``FILE_ATTRIBUTE_*`` + :c:func:`!GetFileInformationByHandle`. + See the :const:`!FILE_ATTRIBUTE_* ` constants in the :mod:`stat` module. .. attribute:: st_reparse_tag - When :attr:`st_file_attributes` has the ``FILE_ATTRIBUTE_REPARSE_POINT`` + When :attr:`st_file_attributes` has the :const:`~stat.FILE_ATTRIBUTE_REPARSE_POINT` set, this field contains the tag identifying the type of reparse point. - See the ``IO_REPARSE_TAG_*`` constants in the :mod:`stat` module. + See the :const:`IO_REPARSE_TAG_* ` + constants in the :mod:`stat` module. The standard module :mod:`stat` defines functions and constants that are useful for extracting information from a :c:struct:`stat` structure. (On @@ -3212,7 +3214,7 @@ features: .. function:: statvfs(path) - Perform a :c:func:`statvfs` system call on the given path. The return value is + Perform a :c:func:`!statvfs` system call on the given path. The return value is an object whose attributes describe the filesystem on the given path, and correspond to the members of the :c:struct:`statvfs` structure, namely: :attr:`f_bsize`, :attr:`f_frsize`, :attr:`f_blocks`, :attr:`f_bfree`, @@ -4303,7 +4305,7 @@ written in Python, such as a mail server's external command delivery program. setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ setsigdef=(), scheduler=None) - Wraps the :c:func:`posix_spawn` C library API for use from Python. + Wraps the :c:func:`!posix_spawn` C library API for use from Python. Most users should use :func:`subprocess.run` instead of :func:`posix_spawn`. @@ -4339,16 +4341,16 @@ written in Python, such as a mail server's external command delivery program. Performs ``os.dup2(fd, new_fd)``. These tuples correspond to the C library - :c:func:`posix_spawn_file_actions_addopen`, - :c:func:`posix_spawn_file_actions_addclose`, and - :c:func:`posix_spawn_file_actions_adddup2` API calls used to prepare - for the :c:func:`posix_spawn` call itself. + :c:func:`!posix_spawn_file_actions_addopen`, + :c:func:`!posix_spawn_file_actions_addclose`, and + :c:func:`!posix_spawn_file_actions_adddup2` API calls used to prepare + for the :c:func:`!posix_spawn` call itself. The *setpgroup* argument will set the process group of the child to the value specified. If the value specified is 0, the child's process group ID will be made the same as its process ID. If the value of *setpgroup* is not set, the child will inherit the parent's process group ID. This argument corresponds - to the C library :c:macro:`POSIX_SPAWN_SETPGROUP` flag. + to the C library :c:macro:`!POSIX_SPAWN_SETPGROUP` flag. If the *resetids* argument is ``True`` it will reset the effective UID and GID of the child to the real UID and GID of the parent process. If the @@ -4356,27 +4358,27 @@ written in Python, such as a mail server's external command delivery program. the parent. In either case, if the set-user-ID and set-group-ID permission bits are enabled on the executable file, their effect will override the setting of the effective UID and GID. This argument corresponds to the C - library :c:macro:`POSIX_SPAWN_RESETIDS` flag. + library :c:macro:`!POSIX_SPAWN_RESETIDS` flag. If the *setsid* argument is ``True``, it will create a new session ID - for ``posix_spawn``. *setsid* requires :c:macro:`POSIX_SPAWN_SETSID` - or :c:macro:`POSIX_SPAWN_SETSID_NP` flag. Otherwise, :exc:`NotImplementedError` + for ``posix_spawn``. *setsid* requires :c:macro:`!POSIX_SPAWN_SETSID` + or :c:macro:`!POSIX_SPAWN_SETSID_NP` flag. Otherwise, :exc:`NotImplementedError` is raised. The *setsigmask* argument will set the signal mask to the signal set specified. If the parameter is not used, then the child inherits the parent's signal mask. This argument corresponds to the C library - :c:macro:`POSIX_SPAWN_SETSIGMASK` flag. + :c:macro:`!POSIX_SPAWN_SETSIGMASK` flag. The *sigdef* argument will reset the disposition of all signals in the set specified. This argument corresponds to the C library - :c:macro:`POSIX_SPAWN_SETSIGDEF` flag. + :c:macro:`!POSIX_SPAWN_SETSIGDEF` flag. The *scheduler* argument must be a tuple containing the (optional) scheduler policy and an instance of :class:`sched_param` with the scheduler parameters. A value of ``None`` in the place of the scheduler policy indicates that is not being provided. This argument is a combination of the C library - :c:macro:`POSIX_SPAWN_SETSCHEDPARAM` and :c:macro:`POSIX_SPAWN_SETSCHEDULER` + :c:macro:`!POSIX_SPAWN_SETSCHEDPARAM` and :c:macro:`!POSIX_SPAWN_SETSCHEDULER` flags. .. audit-event:: os.posix_spawn path,argv,env os.posix_spawn @@ -4389,7 +4391,7 @@ written in Python, such as a mail server's external command delivery program. setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ setsigdef=(), scheduler=None) - Wraps the :c:func:`posix_spawnp` C library API for use from Python. + Wraps the :c:func:`!posix_spawnp` C library API for use from Python. Similar to :func:`posix_spawn` except that the system searches for the *executable* file in the list of directories specified by the @@ -4570,7 +4572,7 @@ written in Python, such as a mail server's external command delivery program. Use *show_cmd* to override the default window style. Whether this has any effect will depend on the application being launched. Values are integers as - supported by the Win32 :c:func:`ShellExecute` function. + supported by the Win32 :c:func:`!ShellExecute` function. :func:`startfile` returns as soon as the associated application is launched. There is no option to wait for the application to close, and no way to retrieve @@ -4580,7 +4582,7 @@ written in Python, such as a mail server's external command delivery program. :func:`os.path.normpath` function to ensure that paths are properly encoded for Win32. - To reduce interpreter startup overhead, the Win32 :c:func:`ShellExecute` + To reduce interpreter startup overhead, the Win32 :c:func:`!ShellExecute` function is not resolved until this function is first called. If the function cannot be resolved, :exc:`NotImplementedError` will be raised. From webhook-mailer at python.org Sat Aug 19 19:08:43 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sat, 19 Aug 2023 23:08:43 -0000 Subject: [Python-checkins] [3.12] gh-107980: fix doc role for asyncio.timeouts (GH-108126) (#108152) Message-ID: https://github.com/python/cpython/commit/0e5eee0130bb4924264250da60c2d636d357dc4d commit: 0e5eee0130bb4924264250da60c2d636d357dc4d branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-20T01:08:40+02:00 summary: [3.12] gh-107980: fix doc role for asyncio.timeouts (GH-108126) (#108152) gh-107980: fix doc role for asyncio.timeouts (GH-108126) (cherry picked from commit a47c13cae5b32e6f3d7532cc6dbb4e1ac31219de) Co-authored-by: Tin Tvrtkovi? files: M Doc/library/asyncio-task.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index d07fbae287e53..f488aa73a62f0 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -628,9 +628,9 @@ Shielding From Cancellation Timeouts ======== -.. coroutinefunction:: timeout(delay) +.. function:: timeout(delay) - An :ref:`asynchronous context manager ` + Return an :ref:`asynchronous context manager ` that can be used to limit the amount of time spent waiting on something. @@ -721,7 +721,7 @@ Timeouts .. versionadded:: 3.11 -.. coroutinefunction:: timeout_at(when) +.. function:: timeout_at(when) Similar to :func:`asyncio.timeout`, except *when* is the absolute time to stop waiting, or ``None``. From webhook-mailer at python.org Sun Aug 20 04:11:44 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Sun, 20 Aug 2023 08:11:44 -0000 Subject: [Python-checkins] Fix misspellings in sysconfig docs (#108156) Message-ID: https://github.com/python/cpython/commit/1dc0c58d2b17819720d184ec0287a8a9b1dc347e commit: 1dc0c58d2b17819720d184ec0287a8a9b1dc347e branch: main author: Rafael Fontenelle committer: AlexWaygood date: 2023-08-20T08:11:39Z summary: Fix misspellings in sysconfig docs (#108156) files: M Doc/library/sysconfig.rst diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index c805c50ffc689..26344ea4e7e2a 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -84,13 +84,13 @@ Python currently supports nine schemes: through Distutils and the *user* option is used. This scheme defines paths located under the user home directory. - *posix_venv*: scheme for :mod:`Python virtual environments ` on POSIX - platforms; by default it is the same as *posix_prefix* . + platforms; by default it is the same as *posix_prefix*. - *nt*: scheme for NT platforms like Windows. - *nt_user*: scheme for NT platforms, when the *user* option is used. - *nt_venv*: scheme for :mod:`Python virtual environments ` on NT - platforms; by default it is the same as *nt* . -- *venv*: a scheme with values from ether *posix_venv* or *nt_venv* depending - on the platform Python runs on + platforms; by default it is the same as *nt*. +- *venv*: a scheme with values from either *posix_venv* or *nt_venv* depending + on the platform Python runs on. - *osx_framework_user*: scheme for macOS, when the *user* option is used. Each scheme is itself composed of a series of paths and each path has a unique @@ -187,7 +187,7 @@ identifier. Python currently uses eight paths: platform is used. If *vars* is provided, it must be a dictionary of variables that will update - the dictionary return by :func:`get_config_vars`. + the dictionary returned by :func:`get_config_vars`. If *expand* is set to ``False``, the path will not be expanded using the variables. From webhook-mailer at python.org Sun Aug 20 04:21:00 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Sun, 20 Aug 2023 08:21:00 -0000 Subject: [Python-checkins] [3.11] Fix misspellings in sysconfig docs (GH-108156) (#108158) Message-ID: https://github.com/python/cpython/commit/cdb5e9f76f6a3ecd0fbf1f49ea7de8befac1622d commit: cdb5e9f76f6a3ecd0fbf1f49ea7de8befac1622d branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: AlexWaygood date: 2023-08-20T08:20:57Z summary: [3.11] Fix misspellings in sysconfig docs (GH-108156) (#108158) Fix misspellings in sysconfig docs (GH-108156) (cherry picked from commit 1dc0c58d2b17819720d184ec0287a8a9b1dc347e) Co-authored-by: Rafael Fontenelle files: M Doc/library/sysconfig.rst diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index dfe6934e6da8c..17448f2a2abac 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -84,13 +84,13 @@ Python currently supports nine schemes: through Distutils and the *user* option is used. This scheme defines paths located under the user home directory. - *posix_venv*: scheme for :mod:`Python virtual environments ` on POSIX - platforms; by default it is the same as *posix_prefix* . + platforms; by default it is the same as *posix_prefix*. - *nt*: scheme for NT platforms like Windows. - *nt_user*: scheme for NT platforms, when the *user* option is used. - *nt_venv*: scheme for :mod:`Python virtual environments ` on NT - platforms; by default it is the same as *nt* . -- *venv*: a scheme with values from ether *posix_venv* or *nt_venv* depending - on the platform Python runs on + platforms; by default it is the same as *nt*. +- *venv*: a scheme with values from either *posix_venv* or *nt_venv* depending + on the platform Python runs on. - *osx_framework_user*: scheme for macOS, when the *user* option is used. Each scheme is itself composed of a series of paths and each path has a unique @@ -187,7 +187,7 @@ identifier. Python currently uses eight paths: platform is used. If *vars* is provided, it must be a dictionary of variables that will update - the dictionary return by :func:`get_config_vars`. + the dictionary returned by :func:`get_config_vars`. If *expand* is set to ``False``, the path will not be expanded using the variables. From webhook-mailer at python.org Sun Aug 20 05:00:46 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Sun, 20 Aug 2023 09:00:46 -0000 Subject: [Python-checkins] Fix patchcheck for `asyncio.tasks` (#108159) Message-ID: https://github.com/python/cpython/commit/b1e5d2c601bbd3d435b60deef4818f3622bdfca3 commit: b1e5d2c601bbd3d435b60deef4818f3622bdfca3 branch: main author: Kumar Aditya committer: kumaraditya303 date: 2023-08-20T09:00:42Z summary: Fix patchcheck for `asyncio.tasks` (#108159) files: M Lib/asyncio/tasks.py diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 75dd3cb6c2e98..edc64fda2a6ad 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -935,21 +935,21 @@ def callback(): def create_eager_task_factory(custom_task_constructor): """Create a function suitable for use as a task factory on an event-loop. - Example usage: + Example usage: - loop.set_task_factory( - asyncio.create_eager_task_factory(my_task_constructor)) + loop.set_task_factory( + asyncio.create_eager_task_factory(my_task_constructor)) - Now, tasks created will be started immediately (rather than being first - scheduled to an event loop). The constructor argument can be any callable - that returns a Task-compatible object and has a signature compatible - with `Task.__init__`; it must have the `eager_start` keyword argument. + Now, tasks created will be started immediately (rather than being first + scheduled to an event loop). The constructor argument can be any callable + that returns a Task-compatible object and has a signature compatible + with `Task.__init__`; it must have the `eager_start` keyword argument. - Most applications will use `Task` for `custom_task_constructor` and in + Most applications will use `Task` for `custom_task_constructor` and in this case there's no need to call `create_eager_task_factory()` directly. Instead the global `eager_task_factory` instance can be used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. - """ + """ def factory(loop, coro, *, name=None, context=None): return custom_task_constructor( From webhook-mailer at python.org Sun Aug 20 06:12:19 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Sun, 20 Aug 2023 10:12:19 -0000 Subject: [Python-checkins] [3.11] gh-107980: fix doc role for asyncio.timeouts (GH-108126) (#108153) Message-ID: https://github.com/python/cpython/commit/5d366f28ae5a3a6be9819280e4faf7e12e4407ca commit: 5d366f28ae5a3a6be9819280e4faf7e12e4407ca branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: AlexWaygood date: 2023-08-20T11:12:15+01:00 summary: [3.11] gh-107980: fix doc role for asyncio.timeouts (GH-108126) (#108153) gh-107980: fix doc role for asyncio.timeouts (GH-108126) (cherry picked from commit a47c13cae5b32e6f3d7532cc6dbb4e1ac31219de) Co-authored-by: Tin Tvrtkovi? files: M Doc/library/asyncio-task.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index ad5466b12b651..ef17f384c1bfe 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -578,9 +578,9 @@ Shielding From Cancellation Timeouts ======== -.. coroutinefunction:: timeout(delay) +.. function:: timeout(delay) - An :ref:`asynchronous context manager ` + Return an :ref:`asynchronous context manager ` that can be used to limit the amount of time spent waiting on something. @@ -671,7 +671,7 @@ Timeouts .. versionadded:: 3.11 -.. coroutinefunction:: timeout_at(when) +.. function:: timeout_at(when) Similar to :func:`asyncio.timeout`, except *when* is the absolute time to stop waiting, or ``None``. From webhook-mailer at python.org Sun Aug 20 06:33:18 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Sun, 20 Aug 2023 10:33:18 -0000 Subject: [Python-checkins] gh-107619: Extend functools LRU cache docs with generators and async functions (#107934) Message-ID: https://github.com/python/cpython/commit/1a713eac47b26899044752f02cbfcb4d628dda2a commit: 1a713eac47b26899044752f02cbfcb4d628dda2a branch: main author: Hadh?zy Tam?s <85063808+Hels15 at users.noreply.github.com> committer: kumaraditya303 date: 2023-08-20T10:33:15Z summary: gh-107619: Extend functools LRU cache docs with generators and async functions (#107934) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Kumar Aditya files: M Doc/library/functools.rst diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 40f43f8b3519c..f736eb0aca1ac 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -226,8 +226,9 @@ The :mod:`functools` module defines the following functions: In general, the LRU cache should only be used when you want to reuse previously computed values. Accordingly, it doesn't make sense to cache - functions with side-effects, functions that need to create distinct mutable - objects on each call, or impure functions such as time() or random(). + functions with side-effects, functions that need to create + distinct mutable objects on each call (such as generators and async functions), + or impure functions such as time() or random(). Example of an LRU cache for static web content:: From webhook-mailer at python.org Sun Aug 20 06:39:46 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Sun, 20 Aug 2023 10:39:46 -0000 Subject: [Python-checkins] [3.11] gh-107619: Extend functools LRU cache docs with generators and async functions (GH-107934) (#108162) Message-ID: https://github.com/python/cpython/commit/4c042a36ce26d6920abdc7e5a54f80d288b9cc80 commit: 4c042a36ce26d6920abdc7e5a54f80d288b9cc80 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: kumaraditya303 date: 2023-08-20T10:39:43Z summary: [3.11] gh-107619: Extend functools LRU cache docs with generators and async functions (GH-107934) (#108162) gh-107619: Extend functools LRU cache docs with generators and async functions (GH-107934) (cherry picked from commit 1a713eac47b26899044752f02cbfcb4d628dda2a) Co-authored-by: Hadh?zy Tam?s <85063808+Hels15 at users.noreply.github.com> Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Kumar Aditya files: M Doc/library/functools.rst diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 62fcddec09041..9dadbb69fc2f2 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -211,8 +211,9 @@ The :mod:`functools` module defines the following functions: In general, the LRU cache should only be used when you want to reuse previously computed values. Accordingly, it doesn't make sense to cache - functions with side-effects, functions that need to create distinct mutable - objects on each call, or impure functions such as time() or random(). + functions with side-effects, functions that need to create + distinct mutable objects on each call (such as generators and async functions), + or impure functions such as time() or random(). Example of an LRU cache for static web content:: From webhook-mailer at python.org Sun Aug 20 06:54:09 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Sun, 20 Aug 2023 10:54:09 -0000 Subject: [Python-checkins] gh-107659: Improve wording of the description of `ctypes.pointer` and `ctypes.POINTER` (#107769) Message-ID: https://github.com/python/cpython/commit/beffb30dc7a07044f4198245d049ddda1f4b24db commit: beffb30dc7a07044f4198245d049ddda1f4b24db branch: main author: Tomas R committer: kumaraditya303 date: 2023-08-20T10:54:06Z summary: gh-107659: Improve wording of the description of `ctypes.pointer` and `ctypes.POINTER` (#107769) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Kumar Aditya files: M Doc/library/ctypes.rst diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 474359a5cd98b..8af609d53ff4b 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -2029,17 +2029,17 @@ Utility functions specifying an address, or a ctypes instance. -.. function:: POINTER(type) +.. function:: POINTER(type, /) - This factory function creates and returns a new ctypes pointer type. Pointer - types are cached and reused internally, so calling this function repeatedly is - cheap. *type* must be a ctypes type. + Create and return a new ctypes pointer type. Pointer types are cached and + reused internally, so calling this function repeatedly is cheap. + *type* must be a ctypes type. -.. function:: pointer(obj) +.. function:: pointer(obj, /) - This function creates a new pointer instance, pointing to *obj*. The returned - object is of the type ``POINTER(type(obj))``. + Create a new pointer instance, pointing to *obj*. + The returned object is of the type ``POINTER(type(obj))``. Note: If you just want to pass a pointer to an object to a foreign function call, you should use ``byref(obj)`` which is much faster. From webhook-mailer at python.org Sun Aug 20 07:05:28 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Sun, 20 Aug 2023 11:05:28 -0000 Subject: [Python-checkins] [3.11] gh-107659: Improve wording of the description of `ctypes.pointer` and `ctypes.POINTER` (GH-107769) (#108164) Message-ID: https://github.com/python/cpython/commit/ba6370656f7bf2b2b5882aad7d3c060f9b117540 commit: ba6370656f7bf2b2b5882aad7d3c060f9b117540 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: kumaraditya303 date: 2023-08-20T11:05:24Z summary: [3.11] gh-107659: Improve wording of the description of `ctypes.pointer` and `ctypes.POINTER` (GH-107769) (#108164) gh-107659: Improve wording of the description of `ctypes.pointer` and `ctypes.POINTER` (GH-107769) (cherry picked from commit beffb30dc7a07044f4198245d049ddda1f4b24db) Co-authored-by: Tomas R Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Kumar Aditya files: M Doc/library/ctypes.rst diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index debc1c0adde84..e2dde683a1a5d 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1988,17 +1988,17 @@ Utility functions specifying an address, or a ctypes instance. -.. function:: POINTER(type) +.. function:: POINTER(type, /) - This factory function creates and returns a new ctypes pointer type. Pointer - types are cached and reused internally, so calling this function repeatedly is - cheap. *type* must be a ctypes type. + Create and return a new ctypes pointer type. Pointer types are cached and + reused internally, so calling this function repeatedly is cheap. + *type* must be a ctypes type. -.. function:: pointer(obj) +.. function:: pointer(obj, /) - This function creates a new pointer instance, pointing to *obj*. The returned - object is of the type ``POINTER(type(obj))``. + Create a new pointer instance, pointing to *obj*. + The returned object is of the type ``POINTER(type(obj))``. Note: If you just want to pass a pointer to an object to a foreign function call, you should use ``byref(obj)`` which is much faster. From webhook-mailer at python.org Sun Aug 20 07:09:37 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Sun, 20 Aug 2023 11:09:37 -0000 Subject: [Python-checkins] Docs: Fix Sphinx warnings in sys.rst (#108106) Message-ID: https://github.com/python/cpython/commit/29fa7afef94d74e18d97485c085d1ccf80c16ca3 commit: 29fa7afef94d74e18d97485c085d1ccf80c16ca3 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-20T13:09:33+02:00 summary: Docs: Fix Sphinx warnings in sys.rst (#108106) - Mark up named tuple attributes as attributes - Remove links for external functions - io.BufferedIOBase has no 'buffer' attribute; remove the link and mark up using :attr:`!buffer` - (Re)format some tables as bullet lists: - sys._emscripten_info - sys.hash_info - sys.int_info - sys.thread_info - In the paragraphs mentioning 'f_trace_lines' and 'f_trace_opcodes', add links to the frame objects reference. Co-authored-by: Alex Waygood Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/sys.rst M Doc/library/textwrap.rst M Doc/tools/.nitignore diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index a82e043f6209a..e99ab90bd18e4 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -333,23 +333,21 @@ always available. *wasm32-emscripten* platform. The named tuple is provisional and may change in the future. - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------------+ - | Attribute | Explanation | - +=============================+==============================================+ - | :const:`emscripten_version` | Emscripten version as tuple of ints | - | | (major, minor, micro), e.g. ``(3, 1, 8)``. | - +-----------------------------+----------------------------------------------+ - | :const:`runtime` | Runtime string, e.g. browser user agent, | - | | ``'Node.js v14.18.2'``, or ``'UNKNOWN'``. | - +-----------------------------+----------------------------------------------+ - | :const:`pthreads` | ``True`` if Python is compiled with | - | | Emscripten pthreads support. | - +-----------------------------+----------------------------------------------+ - | :const:`shared_memory` | ``True`` if Python is compiled with shared | - | | memory support. | - +-----------------------------+----------------------------------------------+ + .. attribute:: _emscripten_info.emscripten_version + + Emscripten version as tuple of ints (major, minor, micro), e.g. ``(3, 1, 8)``. + + .. attribute:: _emscripten_info.runtime + + Runtime string, e.g. browser user agent, ``'Node.js v14.18.2'``, or ``'UNKNOWN'``. + + .. attribute:: _emscripten_info.pthreads + + ``True`` if Python is compiled with Emscripten pthreads support. + + .. attribute:: _emscripten_info.shared_memory + + ``True`` if Python is compiled with shared memory support. .. availability:: Emscripten. @@ -515,28 +513,62 @@ always available. The :term:`named tuple` *flags* exposes the status of command line flags. The attributes are read only. - ============================== ============================================================================================================== - attribute flag - ============================== ============================================================================================================== - :const:`debug` :option:`-d` - :const:`inspect` :option:`-i` - :const:`interactive` :option:`-i` - :const:`isolated` :option:`-I` - :const:`optimize` :option:`-O` or :option:`-OO` - :const:`dont_write_bytecode` :option:`-B` - :const:`no_user_site` :option:`-s` - :const:`no_site` :option:`-S` - :const:`ignore_environment` :option:`-E` - :const:`verbose` :option:`-v` - :const:`bytes_warning` :option:`-b` - :const:`quiet` :option:`-q` - :const:`hash_randomization` :option:`-R` - :const:`dev_mode` :option:`-X dev <-X>` (:ref:`Python Development Mode `) - :const:`utf8_mode` :option:`-X utf8 <-X>` - :const:`safe_path` :option:`-P` - :const:`int_max_str_digits` :option:`-X int_max_str_digits <-X>` (:ref:`integer string conversion length limitation `) - :const:`warn_default_encoding` :option:`-X warn_default_encoding <-X>` - ============================== ============================================================================================================== + .. list-table:: + + * - .. attribute:: flags.debug + - :option:`-d` + + * - .. attribute:: flags.inspect + - :option:`-i` + + * - .. attribute:: flags.interactive + - :option:`-i` + + * - .. attribute:: flags.isolated + - :option:`-I` + + * - .. attribute:: flags.optimize + - :option:`-O` or :option:`-OO` + + * - .. attribute:: flags.dont_write_bytecode + - :option:`-B` + + * - .. attribute:: flags.no_user_site + - :option:`-s` + + * - .. attribute:: flags.no_site + - :option:`-S` + + * - .. attribute:: flags.ignore_environment + - :option:`-E` + + * - .. attribute:: flags.verbose + - :option:`-v` + + * - .. attribute:: flags.bytes_warning + - :option:`-b` + + * - .. attribute:: flags.quiet + - :option:`-q` + + * - .. attribute:: flags.hash_randomization + - :option:`-R` + + * - .. attribute:: flags.dev_mode + - :option:`-X dev <-X>` (:ref:`Python Development Mode `) + + * - .. attribute:: flags.utf8_mode + - :option:`-X utf8 <-X>` + + * - .. attribute:: flags.safe_path + - :option:`-P` + + * - .. attribute:: flags.int_max_str_digits + - :option:`-X int_max_str_digits <-X>` + (:ref:`integer string conversion length limitation `) + + * - .. attribute:: flags.warn_default_encoding + - :option:`-X warn_default_encoding <-X>` .. versionchanged:: 3.2 Added ``quiet`` attribute for the new :option:`-q` flag. @@ -923,8 +955,8 @@ always available. | | a domain controller. | +---------------------------------------+---------------------------------+ - This function wraps the Win32 :c:func:`GetVersionEx` function; see the - Microsoft documentation on :c:func:`OSVERSIONINFOEX` for more information + This function wraps the Win32 :c:func:`!GetVersionEx` function; see the + Microsoft documentation on :c:func:`!OSVERSIONINFOEX` for more information about these fields. *platform_version* returns the major version, minor version and @@ -982,28 +1014,37 @@ always available. implementation. For more details about hashing of numeric types, see :ref:`numeric-hash`. - +---------------------+--------------------------------------------------+ - | attribute | explanation | - +=====================+==================================================+ - | :const:`width` | width in bits used for hash values | - +---------------------+--------------------------------------------------+ - | :const:`modulus` | prime modulus P used for numeric hash scheme | - +---------------------+--------------------------------------------------+ - | :const:`inf` | hash value returned for a positive infinity | - +---------------------+--------------------------------------------------+ - | :const:`nan` | (this attribute is no longer used) | - +---------------------+--------------------------------------------------+ - | :const:`imag` | multiplier used for the imaginary part of a | - | | complex number | - +---------------------+--------------------------------------------------+ - | :const:`algorithm` | name of the algorithm for hashing of str, bytes, | - | | and memoryview | - +---------------------+--------------------------------------------------+ - | :const:`hash_bits` | internal output size of the hash algorithm | - +---------------------+--------------------------------------------------+ - | :const:`seed_bits` | size of the seed key of the hash algorithm | - +---------------------+--------------------------------------------------+ + .. attribute:: hash_info.width + + The width in bits used for hash values + + .. attribute:: hash_info.modulus + The prime modulus P used for numeric hash scheme + + .. attribute:: hash_info.inf + + The hash value returned for a positive infinity + + .. attribute:: hash_info.nan + + (This attribute is no longer used) + + .. attribute:: hash_info.imag + + The multiplier used for the imaginary part of a complex number + + .. attribute:: hash_info.algorithm + + The name of the algorithm for hashing of str, bytes, and memoryview + + .. attribute:: hash_info.hash_bits + + The internal output size of the hash algorithm + + .. attribute:: hash_info.seed_bits + + The size of the seed key of the hash algorithm .. versionadded:: 3.2 @@ -1081,32 +1122,31 @@ always available. A :term:`named tuple` that holds information about Python's internal representation of integers. The attributes are read only. - .. tabularcolumns:: |l|L| - - +----------------------------------------+-----------------------------------------------+ - | Attribute | Explanation | - +========================================+===============================================+ - | :const:`bits_per_digit` | number of bits held in each digit. Python | - | | integers are stored internally in base | - | | ``2**int_info.bits_per_digit`` | - +----------------------------------------+-----------------------------------------------+ - | :const:`sizeof_digit` | size in bytes of the C type used to | - | | represent a digit | - +----------------------------------------+-----------------------------------------------+ - | :const:`default_max_str_digits` | default value for | - | | :func:`sys.get_int_max_str_digits` when it | - | | is not otherwise explicitly configured. | - +----------------------------------------+-----------------------------------------------+ - | :const:`str_digits_check_threshold` | minimum non-zero value for | - | | :func:`sys.set_int_max_str_digits`, | - | | :envvar:`PYTHONINTMAXSTRDIGITS`, or | - | | :option:`-X int_max_str_digits <-X>`. | - +----------------------------------------+-----------------------------------------------+ + .. attribute:: int_info.bits_per_digit + + The number of bits held in each digit. + Python integers are stored internally in base ``2**int_info.bits_per_digit``. + + .. attribute:: int_info.sizeof_digit + + The size in bytes of the C type used to represent a digit. + + .. attribute:: int_info.default_max_str_digits + + The default value for :func:`sys.get_int_max_str_digits` + when it is not otherwise explicitly configured. + + .. attribute:: int_info.str_digits_check_threshold + + The minimum non-zero value for :func:`sys.set_int_max_str_digits`, + :envvar:`PYTHONINTMAXSTRDIGITS`, or :option:`-X int_max_str_digits <-X>`. .. versionadded:: 3.1 .. versionchanged:: 3.11 - Added ``default_max_str_digits`` and ``str_digits_check_threshold``. + + Added :attr:`~int_info.default_max_str_digits` and + :attr:`~int_info.str_digits_check_threshold`. .. data:: __interactivehook__ @@ -1533,7 +1573,7 @@ always available. :file:`Objects/lnotab_notes.txt` for a detailed explanation of how this works. Per-line events may be disabled for a frame by setting - :attr:`f_trace_lines` to :const:`False` on that frame. + :attr:`!f_trace_lines` to :const:`False` on that :ref:`frame `. ``'return'`` A function (or other code block) is about to return. The local trace @@ -1551,8 +1591,8 @@ always available. opcode details). The local trace function is called; *arg* is ``None``; the return value specifies the new local trace function. Per-opcode events are not emitted by default: they must be explicitly - requested by setting :attr:`f_trace_opcodes` to :const:`True` on the - frame. + requested by setting :attr:`!f_trace_opcodes` to :const:`True` on the + :ref:`frame `. Note that as an exception is propagated down the chain of callers, an ``'exception'`` event is generated at each level. @@ -1581,8 +1621,8 @@ always available. .. versionchanged:: 3.7 - ``'opcode'`` event type added; :attr:`f_trace_lines` and - :attr:`f_trace_opcodes` attributes added to frames + ``'opcode'`` event type added; :attr:`!f_trace_lines` and + :attr:`!f_trace_opcodes` attributes added to frames .. function:: set_asyncgen_hooks(firstiter, finalizer) @@ -1739,7 +1779,7 @@ always available. However, if you are writing a library (and do not control in which context its code will be executed), be aware that the standard streams may be replaced with file-like objects like :class:`io.StringIO` which - do not support the :attr:`~io.BufferedIOBase.buffer` attribute. + do not support the :attr:!buffer` attribute. .. data:: __stdin__ @@ -1787,29 +1827,28 @@ always available. A :term:`named tuple` holding information about the thread implementation. - .. tabularcolumns:: |l|p{0.7\linewidth}| - - +------------------+---------------------------------------------------------+ - | Attribute | Explanation | - +==================+=========================================================+ - | :const:`name` | Name of the thread implementation: | - | | | - | | * ``'nt'``: Windows threads | - | | * ``'pthread'``: POSIX threads | - | | * ``'pthread-stubs'``: stub POSIX threads | - | | (on WebAssembly platforms without threading support) | - | | * ``'solaris'``: Solaris threads | - +------------------+---------------------------------------------------------+ - | :const:`lock` | Name of the lock implementation: | - | | | - | | * ``'semaphore'``: a lock uses a semaphore | - | | * ``'mutex+cond'``: a lock uses a mutex | - | | and a condition variable | - | | * ``None`` if this information is unknown | - +------------------+---------------------------------------------------------+ - | :const:`version` | Name and version of the thread library. It is a string, | - | | or ``None`` if this information is unknown. | - +------------------+---------------------------------------------------------+ + .. attribute:: thread_info.name + + The name of the thread implementation: + + * ``"nt"``: Windows threads + * ``"pthread"``: POSIX threads + * ``"pthread-stubs"``: stub POSIX threads + (on WebAssembly platforms without threading support) + * ``"solaris"``: Solaris threads + + .. attribute:: thread_info.lock + + The name of the lock implementation: + + * ``"semaphore"``: a lock uses a semaphore + * ``"mutex+cond"``: a lock uses a mutex and a condition variable + * ``None`` if this information is unknown + + .. attribute:: thread_info.version + + The name and version of the thread library. + It is a string, or ``None`` if this information is unknown. .. versionadded:: 3.3 diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index a150eefbf932e..e2952ce3cc2ca 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -60,7 +60,7 @@ functions should be good enough; otherwise, you should use an instance of First the whitespace in *text* is collapsed (all whitespace is replaced by single spaces). If the result fits in the *width*, it is returned. Otherwise, enough words are dropped from the end so that the remaining words - plus the :attr:`.placeholder` fit within :attr:`.width`:: + plus the *placeholder* fit within *width*:: >>> textwrap.shorten("Hello world!", width=12) 'Hello world!' diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 654287cc50b55..622237e14318a 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -127,7 +127,6 @@ Doc/library/ssl.rst Doc/library/stdtypes.rst Doc/library/string.rst Doc/library/subprocess.rst -Doc/library/sys.rst Doc/library/sys_path_init.rst Doc/library/syslog.rst Doc/library/tarfile.rst From webhook-mailer at python.org Sun Aug 20 07:10:52 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Sun, 20 Aug 2023 11:10:52 -0000 Subject: [Python-checkins] Docs: Fix Sphinx warnings in license.rst (#108142) Message-ID: https://github.com/python/cpython/commit/4d4393139fae39db26dead33529b6ae0bafbfc58 commit: 4d4393139fae39db26dead33529b6ae0bafbfc58 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-20T13:10:48+02:00 summary: Docs: Fix Sphinx warnings in license.rst (#108142) - Fix links to stdlib modules - Silence links to external functions files: M Doc/license.rst M Doc/tools/.nitignore diff --git a/Doc/license.rst b/Doc/license.rst index be8efa7061137..82039fd9aaf9d 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -352,8 +352,8 @@ the verbatim comments from the original code:: Sockets ------- -The :mod:`socket` module uses the functions, :func:`getaddrinfo`, and -:func:`getnameinfo`, which are coded in separate source files from the WIDE +The :mod:`socket` module uses the functions, :c:func:`!getaddrinfo`, and +:c:func:`!getnameinfo`, which are coded in separate source files from the WIDE Project, https://www.wide.ad.jp/. :: Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project. @@ -387,7 +387,7 @@ Project, https://www.wide.ad.jp/. :: Asynchronous socket services ---------------------------- -The :mod:`test.support.asynchat` and :mod:`test.support.asyncore` +The :mod:`!test.support.asynchat` and :mod:`!test.support.asyncore` modules contain the following notice:: Copyright 1996 by Sam Rushing @@ -539,7 +539,7 @@ The :mod:`xmlrpc.client` module contains the following notice:: test_epoll ---------- -The :mod:`test_epoll` module contains the following notice:: +The :mod:`!test.test_epoll` module contains the following notice:: Copyright (c) 2001-2006 Twisted Matrix Laboratories. @@ -844,7 +844,7 @@ and later releases derived from that, the Apache License v2 applies:: expat ----- -The :mod:`pyexpat` extension is built using an included copy of the expat +The :mod:`pyexpat ` extension is built using an included copy of the expat sources unless the build is configured ``--with-system-expat``:: Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 622237e14318a..ebadd32759197 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -160,7 +160,6 @@ Doc/library/xml.sax.rst Doc/library/xmlrpc.client.rst Doc/library/xmlrpc.server.rst Doc/library/zlib.rst -Doc/license.rst Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst Doc/reference/expressions.rst From webhook-mailer at python.org Sun Aug 20 07:19:11 2023 From: webhook-mailer at python.org (vsajip) Date: Sun, 20 Aug 2023 11:19:11 -0000 Subject: [Python-checkins] Docs: Fix Sphinx warnings in logging.rst (GH-108139) Message-ID: https://github.com/python/cpython/commit/c735e79afb62324624864e1943f84825249f58ed commit: c735e79afb62324624864e1943f84825249f58ed branch: main author: Erlend E. Aasland committer: vsajip date: 2023-08-20T12:19:07+01:00 summary: Docs: Fix Sphinx warnings in logging.rst (GH-108139) Co-authored-by: Adam Turner <9087854+aa-turner at users.noreply.github.com> files: M Doc/library/logging.rst M Doc/tools/.nitignore diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 49e870e9e2479..97a0b82b78b1b 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -439,7 +439,7 @@ Handler Objects Handlers have the following attributes and methods. Note that :class:`Handler` is never instantiated directly; this class acts as a base for more useful -subclasses. However, the :meth:`__init__` method in subclasses needs to call +subclasses. However, the :meth:`!__init__` method in subclasses needs to call :meth:`Handler.__init__`. .. class:: Handler @@ -1019,30 +1019,34 @@ information into logging calls. For a usage example, see the section on 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the (possibly modified) versions of the arguments passed in. -In addition to the above, :class:`LoggerAdapter` supports the following -methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, -:meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, -:meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, -:meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and -:meth:`~Logger.hasHandlers`. These methods have the same signatures as their -counterparts in :class:`Logger`, so you can use the two types of instances -interchangeably. + In addition to the above, :class:`LoggerAdapter` supports the following + methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, + :meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, + :meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, + :meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and + :meth:`~Logger.hasHandlers`. These methods have the same signatures as their + counterparts in :class:`Logger`, so you can use the two types of instances + interchangeably. -.. versionchanged:: 3.2 - The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, - :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added - to :class:`LoggerAdapter`. These methods delegate to the underlying logger. + .. versionchanged:: 3.2 + + The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, + :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added + to :class:`LoggerAdapter`. These methods delegate to the underlying logger. + + .. versionchanged:: 3.6 -.. versionchanged:: 3.6 - Attribute :attr:`manager` and method :meth:`_log` were added, which - delegate to the underlying logger and allow adapters to be nested. + Attribute :attr:`!manager` and method :meth:`!_log` were added, which + delegate to the underlying logger and allow adapters to be nested. -.. versionchanged:: 3.13 - Remove the undocumented ``warn()`` method which was an alias to the - ``warning()`` method. + .. versionchanged:: 3.13 + + Remove the undocumented :meth:`!warn`` method which was an alias to the + :meth:`!warning` method. + + .. versionchanged:: 3.13 -.. versionchanged:: 3.13 - The *merge_extra* argument was added. + The *merge_extra* argument was added. Thread Safety @@ -1430,8 +1434,8 @@ functions. .. function:: setLoggerClass(klass) Tells the logging system to use the class *klass* when instantiating a logger. - The class should define :meth:`__init__` such that only a name argument is - required, and the :meth:`__init__` should call :meth:`Logger.__init__`. This + The class should define :meth:`!__init__` such that only a name argument is + required, and the :meth:`!__init__` should call :meth:`!Logger.__init__`. This function is typically called before any loggers are instantiated by applications which need to use custom logger behavior. After this call, as at any other time, do not instantiate loggers directly using the subclass: continue to use diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index ebadd32759197..f7e454af62c87 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -92,7 +92,6 @@ Doc/library/inspect.rst Doc/library/locale.rst Doc/library/logging.config.rst Doc/library/logging.handlers.rst -Doc/library/logging.rst Doc/library/lzma.rst Doc/library/mailbox.rst Doc/library/mmap.rst From webhook-mailer at python.org Sun Aug 20 09:48:49 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 20 Aug 2023 13:48:49 -0000 Subject: [Python-checkins] [3.12] Fix misspellings in sysconfig docs (GH-108156) (#108157) Message-ID: https://github.com/python/cpython/commit/a4a494aff481e210ce2658ecd9f32304c1e27f90 commit: a4a494aff481e210ce2658ecd9f32304c1e27f90 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-20T15:48:46+02:00 summary: [3.12] Fix misspellings in sysconfig docs (GH-108156) (#108157) Fix misspellings in sysconfig docs (GH-108156) (cherry picked from commit 1dc0c58d2b17819720d184ec0287a8a9b1dc347e) Co-authored-by: Rafael Fontenelle files: M Doc/library/sysconfig.rst diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index c805c50ffc689..26344ea4e7e2a 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -84,13 +84,13 @@ Python currently supports nine schemes: through Distutils and the *user* option is used. This scheme defines paths located under the user home directory. - *posix_venv*: scheme for :mod:`Python virtual environments ` on POSIX - platforms; by default it is the same as *posix_prefix* . + platforms; by default it is the same as *posix_prefix*. - *nt*: scheme for NT platforms like Windows. - *nt_user*: scheme for NT platforms, when the *user* option is used. - *nt_venv*: scheme for :mod:`Python virtual environments ` on NT - platforms; by default it is the same as *nt* . -- *venv*: a scheme with values from ether *posix_venv* or *nt_venv* depending - on the platform Python runs on + platforms; by default it is the same as *nt*. +- *venv*: a scheme with values from either *posix_venv* or *nt_venv* depending + on the platform Python runs on. - *osx_framework_user*: scheme for macOS, when the *user* option is used. Each scheme is itself composed of a series of paths and each path has a unique @@ -187,7 +187,7 @@ identifier. Python currently uses eight paths: platform is used. If *vars* is provided, it must be a dictionary of variables that will update - the dictionary return by :func:`get_config_vars`. + the dictionary returned by :func:`get_config_vars`. If *expand* is set to ``False``, the path will not be expanded using the variables. From webhook-mailer at python.org Sun Aug 20 09:49:06 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 20 Aug 2023 13:49:06 -0000 Subject: [Python-checkins] [3.12] Fix patchcheck for `asyncio.tasks` (GH-108159) (#108160) Message-ID: https://github.com/python/cpython/commit/1e46f1a9a438f11f7a69844c6701bca2016b1e8b commit: 1e46f1a9a438f11f7a69844c6701bca2016b1e8b branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-20T15:49:03+02:00 summary: [3.12] Fix patchcheck for `asyncio.tasks` (GH-108159) (#108160) Fix patchcheck for `asyncio.tasks` (GH-108159) (cherry picked from commit b1e5d2c601bbd3d435b60deef4818f3622bdfca3) Co-authored-by: Kumar Aditya files: M Lib/asyncio/tasks.py diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 8d5bde09ea9b5..152c9f8afcc06 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -944,21 +944,21 @@ def callback(): def create_eager_task_factory(custom_task_constructor): """Create a function suitable for use as a task factory on an event-loop. - Example usage: + Example usage: - loop.set_task_factory( - asyncio.create_eager_task_factory(my_task_constructor)) + loop.set_task_factory( + asyncio.create_eager_task_factory(my_task_constructor)) - Now, tasks created will be started immediately (rather than being first - scheduled to an event loop). The constructor argument can be any callable - that returns a Task-compatible object and has a signature compatible - with `Task.__init__`; it must have the `eager_start` keyword argument. + Now, tasks created will be started immediately (rather than being first + scheduled to an event loop). The constructor argument can be any callable + that returns a Task-compatible object and has a signature compatible + with `Task.__init__`; it must have the `eager_start` keyword argument. - Most applications will use `Task` for `custom_task_constructor` and in + Most applications will use `Task` for `custom_task_constructor` and in this case there's no need to call `create_eager_task_factory()` directly. Instead the global `eager_task_factory` instance can be used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. - """ + """ def factory(loop, coro, *, name=None, context=None): return custom_task_constructor( From webhook-mailer at python.org Sun Aug 20 09:49:35 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 20 Aug 2023 13:49:35 -0000 Subject: [Python-checkins] [3.12] gh-107619: Extend functools LRU cache docs with generators and async functions (GH-107934) (#108161) Message-ID: https://github.com/python/cpython/commit/fbe1cff015f0ae792005b443fe9fb4a5137ccc77 commit: fbe1cff015f0ae792005b443fe9fb4a5137ccc77 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-20T15:49:32+02:00 summary: [3.12] gh-107619: Extend functools LRU cache docs with generators and async functions (GH-107934) (#108161) gh-107619: Extend functools LRU cache docs with generators and async functions (GH-107934) (cherry picked from commit 1a713eac47b26899044752f02cbfcb4d628dda2a) Co-authored-by: Hadh?zy Tam?s <85063808+Hels15 at users.noreply.github.com> Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Kumar Aditya files: M Doc/library/functools.rst diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 40f43f8b3519c..f736eb0aca1ac 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -226,8 +226,9 @@ The :mod:`functools` module defines the following functions: In general, the LRU cache should only be used when you want to reuse previously computed values. Accordingly, it doesn't make sense to cache - functions with side-effects, functions that need to create distinct mutable - objects on each call, or impure functions such as time() or random(). + functions with side-effects, functions that need to create + distinct mutable objects on each call (such as generators and async functions), + or impure functions such as time() or random(). Example of an LRU cache for static web content:: From webhook-mailer at python.org Sun Aug 20 09:50:12 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 20 Aug 2023 13:50:12 -0000 Subject: [Python-checkins] [3.12] gh-107659: Improve wording of the description of `ctypes.pointer` and `ctypes.POINTER` (GH-107769) (#108163) Message-ID: https://github.com/python/cpython/commit/590bc6a226f6425970197ff5c3a57beeaa99cdca commit: 590bc6a226f6425970197ff5c3a57beeaa99cdca branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-20T15:50:09+02:00 summary: [3.12] gh-107659: Improve wording of the description of `ctypes.pointer` and `ctypes.POINTER` (GH-107769) (#108163) gh-107659: Improve wording of the description of `ctypes.pointer` and `ctypes.POINTER` (GH-107769) (cherry picked from commit beffb30dc7a07044f4198245d049ddda1f4b24db) Co-authored-by: Tomas R Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Kumar Aditya files: M Doc/library/ctypes.rst diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index cd5c41e9b59c9..19cee10711fb8 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -2029,17 +2029,17 @@ Utility functions specifying an address, or a ctypes instance. -.. function:: POINTER(type) +.. function:: POINTER(type, /) - This factory function creates and returns a new ctypes pointer type. Pointer - types are cached and reused internally, so calling this function repeatedly is - cheap. *type* must be a ctypes type. + Create and return a new ctypes pointer type. Pointer types are cached and + reused internally, so calling this function repeatedly is cheap. + *type* must be a ctypes type. -.. function:: pointer(obj) +.. function:: pointer(obj, /) - This function creates a new pointer instance, pointing to *obj*. The returned - object is of the type ``POINTER(type(obj))``. + Create a new pointer instance, pointing to *obj*. + The returned object is of the type ``POINTER(type(obj))``. Note: If you just want to pass a pointer to an object to a foreign function call, you should use ``byref(obj)`` which is much faster. From webhook-mailer at python.org Sun Aug 20 09:50:32 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 20 Aug 2023 13:50:32 -0000 Subject: [Python-checkins] [3.12] Docs: Fix Sphinx warnings in license.rst (GH-108142) (#108167) Message-ID: https://github.com/python/cpython/commit/ff125c4bd8694808892475ad6b63821fb7ea4338 commit: ff125c4bd8694808892475ad6b63821fb7ea4338 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-20T15:50:29+02:00 summary: [3.12] Docs: Fix Sphinx warnings in license.rst (GH-108142) (#108167) Docs: Fix Sphinx warnings in license.rst (GH-108142) - Fix links to stdlib modules - Silence links to external functions (cherry picked from commit 4d4393139fae39db26dead33529b6ae0bafbfc58) Co-authored-by: Erlend E. Aasland files: M Doc/license.rst M Doc/tools/.nitignore diff --git a/Doc/license.rst b/Doc/license.rst index 812a0adffe1a4..1b209922b0952 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -352,8 +352,8 @@ the verbatim comments from the original code:: Sockets ------- -The :mod:`socket` module uses the functions, :func:`getaddrinfo`, and -:func:`getnameinfo`, which are coded in separate source files from the WIDE +The :mod:`socket` module uses the functions, :c:func:`!getaddrinfo`, and +:c:func:`!getnameinfo`, which are coded in separate source files from the WIDE Project, https://www.wide.ad.jp/. :: Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project. @@ -387,7 +387,7 @@ Project, https://www.wide.ad.jp/. :: Asynchronous socket services ---------------------------- -The :mod:`test.support.asynchat` and :mod:`test.support.asyncore` +The :mod:`!test.support.asynchat` and :mod:`!test.support.asyncore` modules contain the following notice:: Copyright 1996 by Sam Rushing @@ -539,7 +539,7 @@ The :mod:`xmlrpc.client` module contains the following notice:: test_epoll ---------- -The :mod:`test_epoll` module contains the following notice:: +The :mod:`!test.test_epoll` module contains the following notice:: Copyright (c) 2001-2006 Twisted Matrix Laboratories. @@ -844,7 +844,7 @@ and later releases derived from that, the Apache License v2 applies:: expat ----- -The :mod:`pyexpat` extension is built using an included copy of the expat +The :mod:`pyexpat ` extension is built using an included copy of the expat sources unless the build is configured ``--with-system-expat``:: Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 88ac905467299..d55b611a05575 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -176,7 +176,6 @@ Doc/library/xml.sax.rst Doc/library/xmlrpc.client.rst Doc/library/xmlrpc.server.rst Doc/library/zlib.rst -Doc/license.rst Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst Doc/reference/expressions.rst From webhook-mailer at python.org Sun Aug 20 09:52:47 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Sun, 20 Aug 2023 13:52:47 -0000 Subject: [Python-checkins] [3.11] Docs: Fix Sphinx warnings in sys.rst (GH-108106) (#108166) Message-ID: https://github.com/python/cpython/commit/5af34d2a4e41c17765cc77e956b8ab8008db1b34 commit: 5af34d2a4e41c17765cc77e956b8ab8008db1b34 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: kumaraditya303 date: 2023-08-20T19:22:44+05:30 summary: [3.11] Docs: Fix Sphinx warnings in sys.rst (GH-108106) (#108166) Docs: Fix Sphinx warnings in sys.rst (GH-108106) - Mark up named tuple attributes as attributes - Remove links for external functions - io.BufferedIOBase has no 'buffer' attribute; remove the link and mark up using :attr:`!buffer` - (Re)format some tables as bullet lists: - sys._emscripten_info - sys.hash_info - sys.int_info - sys.thread_info - In the paragraphs mentioning 'f_trace_lines' and 'f_trace_opcodes', add links to the frame objects reference. (cherry picked from commit 29fa7afef94d74e18d97485c085d1ccf80c16ca3) Co-authored-by: Erlend E. Aasland Co-authored-by: Alex Waygood Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/sys.rst M Doc/library/textwrap.rst M Doc/tools/.nitignore diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 08bf2d13572f4..ecad0ed881da4 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -329,23 +329,21 @@ always available. *wasm32-emscripten* platform. The named tuple is provisional and may change in the future. - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------------+ - | Attribute | Explanation | - +=============================+==============================================+ - | :const:`emscripten_version` | Emscripten version as tuple of ints | - | | (major, minor, micro), e.g. ``(3, 1, 8)``. | - +-----------------------------+----------------------------------------------+ - | :const:`runtime` | Runtime string, e.g. browser user agent, | - | | ``'Node.js v14.18.2'``, or ``'UNKNOWN'``. | - +-----------------------------+----------------------------------------------+ - | :const:`pthreads` | ``True`` if Python is compiled with | - | | Emscripten pthreads support. | - +-----------------------------+----------------------------------------------+ - | :const:`shared_memory` | ``True`` if Python is compiled with shared | - | | memory support. | - +-----------------------------+----------------------------------------------+ + .. attribute:: _emscripten_info.emscripten_version + + Emscripten version as tuple of ints (major, minor, micro), e.g. ``(3, 1, 8)``. + + .. attribute:: _emscripten_info.runtime + + Runtime string, e.g. browser user agent, ``'Node.js v14.18.2'``, or ``'UNKNOWN'``. + + .. attribute:: _emscripten_info.pthreads + + ``True`` if Python is compiled with Emscripten pthreads support. + + .. attribute:: _emscripten_info.shared_memory + + ``True`` if Python is compiled with shared memory support. .. availability:: Emscripten. @@ -511,28 +509,62 @@ always available. The :term:`named tuple` *flags* exposes the status of command line flags. The attributes are read only. - ============================== ============================================================================================================== - attribute flag - ============================== ============================================================================================================== - :const:`debug` :option:`-d` - :const:`inspect` :option:`-i` - :const:`interactive` :option:`-i` - :const:`isolated` :option:`-I` - :const:`optimize` :option:`-O` or :option:`-OO` - :const:`dont_write_bytecode` :option:`-B` - :const:`no_user_site` :option:`-s` - :const:`no_site` :option:`-S` - :const:`ignore_environment` :option:`-E` - :const:`verbose` :option:`-v` - :const:`bytes_warning` :option:`-b` - :const:`quiet` :option:`-q` - :const:`hash_randomization` :option:`-R` - :const:`dev_mode` :option:`-X dev <-X>` (:ref:`Python Development Mode `) - :const:`utf8_mode` :option:`-X utf8 <-X>` - :const:`safe_path` :option:`-P` - :const:`int_max_str_digits` :option:`-X int_max_str_digits <-X>` (:ref:`integer string conversion length limitation `) - :const:`warn_default_encoding` :option:`-X warn_default_encoding <-X>` - ============================== ============================================================================================================== + .. list-table:: + + * - .. attribute:: flags.debug + - :option:`-d` + + * - .. attribute:: flags.inspect + - :option:`-i` + + * - .. attribute:: flags.interactive + - :option:`-i` + + * - .. attribute:: flags.isolated + - :option:`-I` + + * - .. attribute:: flags.optimize + - :option:`-O` or :option:`-OO` + + * - .. attribute:: flags.dont_write_bytecode + - :option:`-B` + + * - .. attribute:: flags.no_user_site + - :option:`-s` + + * - .. attribute:: flags.no_site + - :option:`-S` + + * - .. attribute:: flags.ignore_environment + - :option:`-E` + + * - .. attribute:: flags.verbose + - :option:`-v` + + * - .. attribute:: flags.bytes_warning + - :option:`-b` + + * - .. attribute:: flags.quiet + - :option:`-q` + + * - .. attribute:: flags.hash_randomization + - :option:`-R` + + * - .. attribute:: flags.dev_mode + - :option:`-X dev <-X>` (:ref:`Python Development Mode `) + + * - .. attribute:: flags.utf8_mode + - :option:`-X utf8 <-X>` + + * - .. attribute:: flags.safe_path + - :option:`-P` + + * - .. attribute:: flags.int_max_str_digits + - :option:`-X int_max_str_digits <-X>` + (:ref:`integer string conversion length limitation `) + + * - .. attribute:: flags.warn_default_encoding + - :option:`-X warn_default_encoding <-X>` .. versionchanged:: 3.2 Added ``quiet`` attribute for the new :option:`-q` flag. @@ -890,8 +922,8 @@ always available. | | a domain controller. | +---------------------------------------+---------------------------------+ - This function wraps the Win32 :c:func:`GetVersionEx` function; see the - Microsoft documentation on :c:func:`OSVERSIONINFOEX` for more information + This function wraps the Win32 :c:func:`!GetVersionEx` function; see the + Microsoft documentation on :c:func:`!OSVERSIONINFOEX` for more information about these fields. *platform_version* returns the major version, minor version and @@ -949,28 +981,37 @@ always available. implementation. For more details about hashing of numeric types, see :ref:`numeric-hash`. - +---------------------+--------------------------------------------------+ - | attribute | explanation | - +=====================+==================================================+ - | :const:`width` | width in bits used for hash values | - +---------------------+--------------------------------------------------+ - | :const:`modulus` | prime modulus P used for numeric hash scheme | - +---------------------+--------------------------------------------------+ - | :const:`inf` | hash value returned for a positive infinity | - +---------------------+--------------------------------------------------+ - | :const:`nan` | (this attribute is no longer used) | - +---------------------+--------------------------------------------------+ - | :const:`imag` | multiplier used for the imaginary part of a | - | | complex number | - +---------------------+--------------------------------------------------+ - | :const:`algorithm` | name of the algorithm for hashing of str, bytes, | - | | and memoryview | - +---------------------+--------------------------------------------------+ - | :const:`hash_bits` | internal output size of the hash algorithm | - +---------------------+--------------------------------------------------+ - | :const:`seed_bits` | size of the seed key of the hash algorithm | - +---------------------+--------------------------------------------------+ + .. attribute:: hash_info.width + + The width in bits used for hash values + + .. attribute:: hash_info.modulus + The prime modulus P used for numeric hash scheme + + .. attribute:: hash_info.inf + + The hash value returned for a positive infinity + + .. attribute:: hash_info.nan + + (This attribute is no longer used) + + .. attribute:: hash_info.imag + + The multiplier used for the imaginary part of a complex number + + .. attribute:: hash_info.algorithm + + The name of the algorithm for hashing of str, bytes, and memoryview + + .. attribute:: hash_info.hash_bits + + The internal output size of the hash algorithm + + .. attribute:: hash_info.seed_bits + + The size of the seed key of the hash algorithm .. versionadded:: 3.2 @@ -1048,32 +1089,31 @@ always available. A :term:`named tuple` that holds information about Python's internal representation of integers. The attributes are read only. - .. tabularcolumns:: |l|L| - - +----------------------------------------+-----------------------------------------------+ - | Attribute | Explanation | - +========================================+===============================================+ - | :const:`bits_per_digit` | number of bits held in each digit. Python | - | | integers are stored internally in base | - | | ``2**int_info.bits_per_digit`` | - +----------------------------------------+-----------------------------------------------+ - | :const:`sizeof_digit` | size in bytes of the C type used to | - | | represent a digit | - +----------------------------------------+-----------------------------------------------+ - | :const:`default_max_str_digits` | default value for | - | | :func:`sys.get_int_max_str_digits` when it | - | | is not otherwise explicitly configured. | - +----------------------------------------+-----------------------------------------------+ - | :const:`str_digits_check_threshold` | minimum non-zero value for | - | | :func:`sys.set_int_max_str_digits`, | - | | :envvar:`PYTHONINTMAXSTRDIGITS`, or | - | | :option:`-X int_max_str_digits <-X>`. | - +----------------------------------------+-----------------------------------------------+ + .. attribute:: int_info.bits_per_digit + + The number of bits held in each digit. + Python integers are stored internally in base ``2**int_info.bits_per_digit``. + + .. attribute:: int_info.sizeof_digit + + The size in bytes of the C type used to represent a digit. + + .. attribute:: int_info.default_max_str_digits + + The default value for :func:`sys.get_int_max_str_digits` + when it is not otherwise explicitly configured. + + .. attribute:: int_info.str_digits_check_threshold + + The minimum non-zero value for :func:`sys.set_int_max_str_digits`, + :envvar:`PYTHONINTMAXSTRDIGITS`, or :option:`-X int_max_str_digits <-X>`. .. versionadded:: 3.1 .. versionchanged:: 3.11 - Added ``default_max_str_digits`` and ``str_digits_check_threshold``. + + Added :attr:`~int_info.default_max_str_digits` and + :attr:`~int_info.str_digits_check_threshold`. .. data:: __interactivehook__ @@ -1501,7 +1541,7 @@ always available. :file:`Objects/lnotab_notes.txt` for a detailed explanation of how this works. Per-line events may be disabled for a frame by setting - :attr:`f_trace_lines` to :const:`False` on that frame. + :attr:`!f_trace_lines` to :const:`False` on that :ref:`frame `. ``'return'`` A function (or other code block) is about to return. The local trace @@ -1519,8 +1559,8 @@ always available. opcode details). The local trace function is called; *arg* is ``None``; the return value specifies the new local trace function. Per-opcode events are not emitted by default: they must be explicitly - requested by setting :attr:`f_trace_opcodes` to :const:`True` on the - frame. + requested by setting :attr:`!f_trace_opcodes` to :const:`True` on the + :ref:`frame `. Note that as an exception is propagated down the chain of callers, an ``'exception'`` event is generated at each level. @@ -1549,8 +1589,8 @@ always available. .. versionchanged:: 3.7 - ``'opcode'`` event type added; :attr:`f_trace_lines` and - :attr:`f_trace_opcodes` attributes added to frames + ``'opcode'`` event type added; :attr:`!f_trace_lines` and + :attr:`!f_trace_opcodes` attributes added to frames .. function:: set_asyncgen_hooks(firstiter, finalizer) @@ -1675,7 +1715,7 @@ always available. However, if you are writing a library (and do not control in which context its code will be executed), be aware that the standard streams may be replaced with file-like objects like :class:`io.StringIO` which - do not support the :attr:`~io.BufferedIOBase.buffer` attribute. + do not support the :attr:!buffer` attribute. .. data:: __stdin__ @@ -1723,29 +1763,28 @@ always available. A :term:`named tuple` holding information about the thread implementation. - .. tabularcolumns:: |l|p{0.7\linewidth}| - - +------------------+---------------------------------------------------------+ - | Attribute | Explanation | - +==================+=========================================================+ - | :const:`name` | Name of the thread implementation: | - | | | - | | * ``'nt'``: Windows threads | - | | * ``'pthread'``: POSIX threads | - | | * ``'pthread-stubs'``: stub POSIX threads | - | | (on WebAssembly platforms without threading support) | - | | * ``'solaris'``: Solaris threads | - +------------------+---------------------------------------------------------+ - | :const:`lock` | Name of the lock implementation: | - | | | - | | * ``'semaphore'``: a lock uses a semaphore | - | | * ``'mutex+cond'``: a lock uses a mutex | - | | and a condition variable | - | | * ``None`` if this information is unknown | - +------------------+---------------------------------------------------------+ - | :const:`version` | Name and version of the thread library. It is a string, | - | | or ``None`` if this information is unknown. | - +------------------+---------------------------------------------------------+ + .. attribute:: thread_info.name + + The name of the thread implementation: + + * ``"nt"``: Windows threads + * ``"pthread"``: POSIX threads + * ``"pthread-stubs"``: stub POSIX threads + (on WebAssembly platforms without threading support) + * ``"solaris"``: Solaris threads + + .. attribute:: thread_info.lock + + The name of the lock implementation: + + * ``"semaphore"``: a lock uses a semaphore + * ``"mutex+cond"``: a lock uses a mutex and a condition variable + * ``None`` if this information is unknown + + .. attribute:: thread_info.version + + The name and version of the thread library. + It is a string, or ``None`` if this information is unknown. .. versionadded:: 3.3 diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index a150eefbf932e..e2952ce3cc2ca 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -60,7 +60,7 @@ functions should be good enough; otherwise, you should use an instance of First the whitespace in *text* is collapsed (all whitespace is replaced by single spaces). If the result fits in the *width*, it is returned. Otherwise, enough words are dropped from the end so that the remaining words - plus the :attr:`.placeholder` fit within :attr:`.width`:: + plus the *placeholder* fit within *width*:: >>> textwrap.shorten("Hello world!", width=12) 'Hello world!' diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 407a4bd837c54..763f02935cd02 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -128,7 +128,6 @@ Doc/library/ssl.rst Doc/library/stdtypes.rst Doc/library/string.rst Doc/library/subprocess.rst -Doc/library/sys.rst Doc/library/sys_path_init.rst Doc/library/syslog.rst Doc/library/tarfile.rst From webhook-mailer at python.org Sun Aug 20 09:53:31 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Sun, 20 Aug 2023 13:53:31 -0000 Subject: [Python-checkins] Resolve reference warnings in faq/programming.rst (#108150) Message-ID: https://github.com/python/cpython/commit/a390ec20f5a85b9c16e8708f117667783d08863c commit: a390ec20f5a85b9c16e8708f117667783d08863c branch: main author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: kumaraditya303 date: 2023-08-20T19:23:28+05:30 summary: Resolve reference warnings in faq/programming.rst (#108150) files: M Doc/faq/programming.rst M Doc/tools/.nitignore diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 6e1812504a184..0a88c5f6384f2 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -454,7 +454,7 @@ There are two factors that produce this result: (the list), and both ``x`` and ``y`` refer to it. 2) Lists are :term:`mutable`, which means that you can change their content. -After the call to :meth:`~list.append`, the content of the mutable object has +After the call to :meth:`!append`, the content of the mutable object has changed from ``[]`` to ``[10]``. Since both the variables refer to the same object, using either name accesses the modified value ``[10]``. @@ -1397,7 +1397,7 @@ To see why this happens, you need to know that (a) if an object implements an :meth:`~object.__iadd__` magic method, it gets called when the ``+=`` augmented assignment is executed, and its return value is what gets used in the assignment statement; -and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`~list.extend` on the list +and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`!extend` on the list and returning the list. That's why we say that for lists, ``+=`` is a "shorthand" for :meth:`!list.extend`:: @@ -1903,7 +1903,7 @@ identity tests. This prevents the code from being confused by objects such as ``float('NaN')`` that are not equal to themselves. For example, here is the implementation of -:meth:`collections.abc.Sequence.__contains__`:: +:meth:`!collections.abc.Sequence.__contains__`:: def __contains__(self, value): for v in self: diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index f7e454af62c87..ac5b54b38f37c 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -27,7 +27,6 @@ Doc/extending/newtypes.rst Doc/faq/design.rst Doc/faq/gui.rst Doc/faq/library.rst -Doc/faq/programming.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst From webhook-mailer at python.org Sun Aug 20 10:01:28 2023 From: webhook-mailer at python.org (kumaraditya303) Date: Sun, 20 Aug 2023 14:01:28 -0000 Subject: [Python-checkins] [3.11] Resolve reference warnings in faq/programming.rst (GH-108150) (#108171) Message-ID: https://github.com/python/cpython/commit/bc055a21cc3f0ece881db8535bfc9457def1a368 commit: bc055a21cc3f0ece881db8535bfc9457def1a368 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: kumaraditya303 date: 2023-08-20T14:01:24Z summary: [3.11] Resolve reference warnings in faq/programming.rst (GH-108150) (#108171) Resolve reference warnings in faq/programming.rst (GH-108150) (cherry picked from commit a390ec20f5a85b9c16e8708f117667783d08863c) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/faq/programming.rst M Doc/tools/.nitignore diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index ab5618db84f77..f43f69b8a1ea9 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -454,7 +454,7 @@ There are two factors that produce this result: (the list), and both ``x`` and ``y`` refer to it. 2) Lists are :term:`mutable`, which means that you can change their content. -After the call to :meth:`~list.append`, the content of the mutable object has +After the call to :meth:`!append`, the content of the mutable object has changed from ``[]`` to ``[10]``. Since both the variables refer to the same object, using either name accesses the modified value ``[10]``. @@ -1397,7 +1397,7 @@ To see why this happens, you need to know that (a) if an object implements an :meth:`~object.__iadd__` magic method, it gets called when the ``+=`` augmented assignment is executed, and its return value is what gets used in the assignment statement; -and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`~list.extend` on the list +and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`!extend` on the list and returning the list. That's why we say that for lists, ``+=`` is a "shorthand" for :meth:`!list.extend`:: @@ -1903,7 +1903,7 @@ identity tests. This prevents the code from being confused by objects such as ``float('NaN')`` that are not equal to themselves. For example, here is the implementation of -:meth:`collections.abc.Sequence.__contains__`:: +:meth:`!collections.abc.Sequence.__contains__`:: def __contains__(self, value): for v in self: diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 763f02935cd02..2ce7274cb37a7 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -27,7 +27,6 @@ Doc/extending/newtypes.rst Doc/faq/design.rst Doc/faq/gui.rst Doc/faq/library.rst -Doc/faq/programming.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst From webhook-mailer at python.org Sun Aug 20 10:44:46 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 20 Aug 2023 14:44:46 -0000 Subject: [Python-checkins] [3.12] Resolve reference warnings in faq/programming.rst (GH-108150) (#108170) Message-ID: https://github.com/python/cpython/commit/7f5a741a28a7b87c9cdb1db87fcb3f0b9ed8ac2a commit: 7f5a741a28a7b87c9cdb1db87fcb3f0b9ed8ac2a branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-20T16:44:43+02:00 summary: [3.12] Resolve reference warnings in faq/programming.rst (GH-108150) (#108170) Resolve reference warnings in faq/programming.rst (GH-108150) (cherry picked from commit a390ec20f5a85b9c16e8708f117667783d08863c) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/faq/programming.rst M Doc/tools/.nitignore diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index ab5618db84f77..f43f69b8a1ea9 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -454,7 +454,7 @@ There are two factors that produce this result: (the list), and both ``x`` and ``y`` refer to it. 2) Lists are :term:`mutable`, which means that you can change their content. -After the call to :meth:`~list.append`, the content of the mutable object has +After the call to :meth:`!append`, the content of the mutable object has changed from ``[]`` to ``[10]``. Since both the variables refer to the same object, using either name accesses the modified value ``[10]``. @@ -1397,7 +1397,7 @@ To see why this happens, you need to know that (a) if an object implements an :meth:`~object.__iadd__` magic method, it gets called when the ``+=`` augmented assignment is executed, and its return value is what gets used in the assignment statement; -and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`~list.extend` on the list +and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`!extend` on the list and returning the list. That's why we say that for lists, ``+=`` is a "shorthand" for :meth:`!list.extend`:: @@ -1903,7 +1903,7 @@ identity tests. This prevents the code from being confused by objects such as ``float('NaN')`` that are not equal to themselves. For example, here is the implementation of -:meth:`collections.abc.Sequence.__contains__`:: +:meth:`!collections.abc.Sequence.__contains__`:: def __contains__(self, value): for v in self: diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index d55b611a05575..a0a121b4b9dbe 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -28,7 +28,6 @@ Doc/extending/newtypes.rst Doc/faq/design.rst Doc/faq/gui.rst Doc/faq/library.rst -Doc/faq/programming.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst From webhook-mailer at python.org Sun Aug 20 12:01:13 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Sun, 20 Aug 2023 16:01:13 -0000 Subject: [Python-checkins] Docs: document 'manager' and '_log' attrs of logging.Logging (#108145) Message-ID: https://github.com/python/cpython/commit/f904aa4e1f6943e5bd9a8a73cf762f063e6fa247 commit: f904aa4e1f6943e5bd9a8a73cf762f063e6fa247 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-20T16:01:10Z summary: Docs: document 'manager' and '_log' attrs of logging.Logging (#108145) Authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/logging.rst diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 97a0b82b78b1b..cf3dcb96a4c41 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1019,6 +1019,14 @@ information into logging calls. For a usage example, see the section on 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the (possibly modified) versions of the arguments passed in. + .. attribute:: manager + + Delegates to the underlying :attr:`!manager`` on *logger*. + + .. attribute:: _log + + Delegates to the underlying :meth:`!_log`` method on *logger*. + In addition to the above, :class:`LoggerAdapter` supports the following methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, :meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, From webhook-mailer at python.org Sun Aug 20 12:27:48 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Sun, 20 Aug 2023 16:27:48 -0000 Subject: [Python-checkins] [3.11] Docs: Fix Sphinx warnings in license.rst (#108142) (#108176) Message-ID: https://github.com/python/cpython/commit/797e3c9e4e4ace5c3693a1d4cfa37059d6e0e430 commit: 797e3c9e4e4ace5c3693a1d4cfa37059d6e0e430 branch: 3.11 author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-20T16:27:44Z summary: [3.11] Docs: Fix Sphinx warnings in license.rst (#108142) (#108176) (cherry picked by commit 4d4393139fae39db26dead33529b6ae0bafbfc58) - Fix links to stdlib modules - Silence links to external functions files: M Doc/license.rst M Doc/tools/.nitignore diff --git a/Doc/license.rst b/Doc/license.rst index 4e8a25ed5ecc1..6644df65170a0 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -351,8 +351,8 @@ the verbatim comments from the original code:: Sockets ------- -The :mod:`socket` module uses the functions, :func:`getaddrinfo`, and -:func:`getnameinfo`, which are coded in separate source files from the WIDE +The :mod:`socket` module uses the functions, :c:func:`!getaddrinfo`, and +:c:func:`!getnameinfo`, which are coded in separate source files from the WIDE Project, https://www.wide.ad.jp/. :: Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project. @@ -537,7 +537,7 @@ The :mod:`xmlrpc.client` module contains the following notice:: test_epoll ---------- -The :mod:`test_epoll` module contains the following notice:: +The :mod:`!test.test_epoll` module contains the following notice:: Copyright (c) 2001-2006 Twisted Matrix Laboratories. @@ -842,7 +842,7 @@ and later releases derived from that, the Apache License v2 applies:: expat ----- -The :mod:`pyexpat` extension is built using an included copy of the expat +The :mod:`pyexpat ` extension is built using an included copy of the expat sources unless the build is configured ``--with-system-expat``:: Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 2ce7274cb37a7..d80c8f1142a71 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -160,7 +160,6 @@ Doc/library/xml.sax.rst Doc/library/xmlrpc.client.rst Doc/library/xmlrpc.server.rst Doc/library/zlib.rst -Doc/license.rst Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst Doc/reference/expressions.rst From webhook-mailer at python.org Sun Aug 20 12:41:01 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Sun, 20 Aug 2023 16:41:01 -0000 Subject: [Python-checkins] [3.11] Docs: Fix Sphinx warnings in logging.rst (GH-108139) (#108175) Message-ID: https://github.com/python/cpython/commit/e5e87f230699f903cb286b2bb7a5529717dd432c commit: e5e87f230699f903cb286b2bb7a5529717dd432c branch: 3.11 author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-20T16:40:57Z summary: [3.11] Docs: Fix Sphinx warnings in logging.rst (GH-108139) (#108175) (cherry picked from commit c735e79afb62324624864e1943f84825249f58ed) Co-authored-by: Adam Turner <9087854+aa-turner at users.noreply.github.com> files: M Doc/library/logging.rst M Doc/tools/.nitignore diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 09dc887ee2415..f9658fa22d8b2 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -427,7 +427,7 @@ Handler Objects Handlers have the following attributes and methods. Note that :class:`Handler` is never instantiated directly; this class acts as a base for more useful -subclasses. However, the :meth:`__init__` method in subclasses needs to call +subclasses. However, the :meth:`!__init__` method in subclasses needs to call :meth:`Handler.__init__`. .. class:: Handler @@ -994,23 +994,25 @@ information into logging calls. For a usage example, see the section on 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the (possibly modified) versions of the arguments passed in. -In addition to the above, :class:`LoggerAdapter` supports the following -methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, -:meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, -:meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, -:meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and -:meth:`~Logger.hasHandlers`. These methods have the same signatures as their -counterparts in :class:`Logger`, so you can use the two types of instances -interchangeably. + In addition to the above, :class:`LoggerAdapter` supports the following + methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, + :meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, + :meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, + :meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and + :meth:`~Logger.hasHandlers`. These methods have the same signatures as their + counterparts in :class:`Logger`, so you can use the two types of instances + interchangeably. -.. versionchanged:: 3.2 - The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, - :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added - to :class:`LoggerAdapter`. These methods delegate to the underlying logger. + .. versionchanged:: 3.2 + + The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, + :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added + to :class:`LoggerAdapter`. These methods delegate to the underlying logger. + + .. versionchanged:: 3.6 -.. versionchanged:: 3.6 - Attribute :attr:`manager` and method :meth:`_log` were added, which - delegate to the underlying logger and allow adapters to be nested. + Attribute :attr:`!manager` and method :meth:`!_log` were added, which + delegate to the underlying logger and allow adapters to be nested. Thread Safety @@ -1381,8 +1383,8 @@ functions. .. function:: setLoggerClass(klass) Tells the logging system to use the class *klass* when instantiating a logger. - The class should define :meth:`__init__` such that only a name argument is - required, and the :meth:`__init__` should call :meth:`Logger.__init__`. This + The class should define :meth:`!__init__` such that only a name argument is + required, and the :meth:`!__init__` should call :meth:`!Logger.__init__`. This function is typically called before any loggers are instantiated by applications which need to use custom logger behavior. After this call, as at any other time, do not instantiate loggers directly using the subclass: continue to use diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index d80c8f1142a71..f36e2b8dc9854 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -92,7 +92,6 @@ Doc/library/inspect.rst Doc/library/locale.rst Doc/library/logging.config.rst Doc/library/logging.handlers.rst -Doc/library/logging.rst Doc/library/lzma.rst Doc/library/mailbox.rst Doc/library/mmap.rst From webhook-mailer at python.org Sun Aug 20 14:34:31 2023 From: webhook-mailer at python.org (hugovk) Date: Sun, 20 Aug 2023 18:34:31 -0000 Subject: [Python-checkins] Resolve reference warnings in faq/design.rst (#108148) Message-ID: https://github.com/python/cpython/commit/92815cc7cf3df8ab702c7cea4efaef349a4b0480 commit: 92815cc7cf3df8ab702c7cea4efaef349a4b0480 branch: main author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: hugovk date: 2023-08-20T12:34:27-06:00 summary: Resolve reference warnings in faq/design.rst (#108148) files: M Doc/faq/design.rst M Doc/tools/.nitignore diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 11d01374dc1e7..ae02c443e5938 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -584,9 +584,9 @@ exhaustive test suites that exercise every line of code in a module. An appropriate testing discipline can help build large complex applications in Python as well as having interface specifications would. In fact, it can be better because an interface specification cannot test certain properties of a -program. For example, the :meth:`list.append` method is expected to add new elements +program. For example, the :meth:`!list.append` method is expected to add new elements to the end of some internal list; an interface specification cannot test that -your :meth:`list.append` implementation will actually do this correctly, but it's +your :meth:`!list.append` implementation will actually do this correctly, but it's trivial to check this property in a test suite. Writing test suites is very helpful, and you might want to design your code to diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index ac5b54b38f37c..0262a0afe4546 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -24,7 +24,6 @@ Doc/c-api/typeobj.rst Doc/c-api/unicode.rst Doc/extending/extending.rst Doc/extending/newtypes.rst -Doc/faq/design.rst Doc/faq/gui.rst Doc/faq/library.rst Doc/glossary.rst From webhook-mailer at python.org Sun Aug 20 14:56:18 2023 From: webhook-mailer at python.org (hugovk) Date: Sun, 20 Aug 2023 18:56:18 -0000 Subject: [Python-checkins] [3.11] Resolve reference warnings in faq/design.rst (GH-108148) (#108181) Message-ID: https://github.com/python/cpython/commit/ff82da17b615c13cb4e46f251a5c0c6cdeedc633 commit: ff82da17b615c13cb4e46f251a5c0c6cdeedc633 branch: 3.11 author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: hugovk date: 2023-08-20T18:56:14Z summary: [3.11] Resolve reference warnings in faq/design.rst (GH-108148) (#108181) files: M Doc/faq/design.rst M Doc/tools/.nitignore diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 9dbfacd73cc6c..83c0152c85e84 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -581,9 +581,9 @@ exhaustive test suites that exercise every line of code in a module. An appropriate testing discipline can help build large complex applications in Python as well as having interface specifications would. In fact, it can be better because an interface specification cannot test certain properties of a -program. For example, the :meth:`append` method is expected to add new elements +program. For example, the :meth:`!list.append` method is expected to add new elements to the end of some internal list; an interface specification cannot test that -your :meth:`append` implementation will actually do this correctly, but it's +your :meth:`!list.append` implementation will actually do this correctly, but it's trivial to check this property in a test suite. Writing test suites is very helpful, and you might want to design your code to diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index f36e2b8dc9854..319c77776b7b8 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -24,7 +24,6 @@ Doc/c-api/typeobj.rst Doc/c-api/unicode.rst Doc/extending/extending.rst Doc/extending/newtypes.rst -Doc/faq/design.rst Doc/faq/gui.rst Doc/faq/library.rst Doc/glossary.rst From webhook-mailer at python.org Sun Aug 20 15:01:16 2023 From: webhook-mailer at python.org (hugovk) Date: Sun, 20 Aug 2023 19:01:16 -0000 Subject: [Python-checkins] Resolve reference warnings in faq/library.rst (#108149) Message-ID: https://github.com/python/cpython/commit/6323bc33ff9f445a947adf4af42b8be7e44c730c commit: 6323bc33ff9f445a947adf4af42b8be7e44c730c branch: main author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: hugovk date: 2023-08-20T19:01:13Z summary: Resolve reference warnings in faq/library.rst (#108149) Co-authored-by: Erlend E. Aasland Co-authored-by: Hugo van Kemenade files: M Doc/faq/library.rst M Doc/tools/.nitignore diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index 9e3727456bb96..476a43d9c288f 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -111,7 +111,7 @@ Is there an equivalent to C's onexit() in Python? ------------------------------------------------- The :mod:`atexit` module provides a register function that is similar to C's -:c:func:`onexit`. +:c:func:`!onexit`. Why don't my signal handlers work? @@ -397,7 +397,7 @@ These aren't:: D[x] = D[x] + 1 Operations that replace other objects may invoke those other objects' -:meth:`__del__` method when their reference count reaches zero, and that can +:meth:`~object.__del__` method when their reference count reaches zero, and that can affect things. This is especially true for the mass updates to dictionaries and lists. When in doubt, use a mutex! @@ -730,14 +730,17 @@ The :mod:`select` module is commonly used to help with asynchronous I/O on sockets. To prevent the TCP connect from blocking, you can set the socket to non-blocking -mode. Then when you do the :meth:`socket.connect`, you will either connect immediately +mode. Then when you do the :meth:`~socket.socket.connect`, +you will either connect immediately (unlikely) or get an exception that contains the error number as ``.errno``. ``errno.EINPROGRESS`` indicates that the connection is in progress, but hasn't finished yet. Different OSes will return different values, so you're going to have to check what's returned on your system. -You can use the :meth:`socket.connect_ex` method to avoid creating an exception. It will -just return the errno value. To poll, you can call :meth:`socket.connect_ex` again later +You can use the :meth:`~socket.socket.connect_ex` method +to avoid creating an exception. +It will just return the errno value. +To poll, you can call :meth:`~socket.socket.connect_ex` again later -- ``0`` or ``errno.EISCONN`` indicate that you're connected -- or you can pass this socket to :meth:`select.select` to check if it's writable. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 0262a0afe4546..9475e883ee5b5 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -25,7 +25,6 @@ Doc/c-api/unicode.rst Doc/extending/extending.rst Doc/extending/newtypes.rst Doc/faq/gui.rst -Doc/faq/library.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst From webhook-mailer at python.org Sun Aug 20 15:26:04 2023 From: webhook-mailer at python.org (hugovk) Date: Sun, 20 Aug 2023 19:26:04 -0000 Subject: [Python-checkins] [3.11] Resolve reference warnings in faq/library.rst (GH-108149) (#108183) Message-ID: https://github.com/python/cpython/commit/4be05aab33c4342d37c5b6b083975f08a16c8cc1 commit: 4be05aab33c4342d37c5b6b083975f08a16c8cc1 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: hugovk date: 2023-08-20T13:26:01-06:00 summary: [3.11] Resolve reference warnings in faq/library.rst (GH-108149) (#108183) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Erlend E. Aasland Co-authored-by: Hugo van Kemenade files: M Doc/faq/library.rst M Doc/tools/.nitignore diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index b43c7505c0401..c69910718f0c9 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -111,7 +111,7 @@ Is there an equivalent to C's onexit() in Python? ------------------------------------------------- The :mod:`atexit` module provides a register function that is similar to C's -:c:func:`onexit`. +:c:func:`!onexit`. Why don't my signal handlers work? @@ -397,7 +397,7 @@ These aren't:: D[x] = D[x] + 1 Operations that replace other objects may invoke those other objects' -:meth:`__del__` method when their reference count reaches zero, and that can +:meth:`~object.__del__` method when their reference count reaches zero, and that can affect things. This is especially true for the mass updates to dictionaries and lists. When in doubt, use a mutex! @@ -765,14 +765,17 @@ The :mod:`select` module is commonly used to help with asynchronous I/O on sockets. To prevent the TCP connect from blocking, you can set the socket to non-blocking -mode. Then when you do the :meth:`socket.connect`, you will either connect immediately +mode. Then when you do the :meth:`~socket.socket.connect`, +you will either connect immediately (unlikely) or get an exception that contains the error number as ``.errno``. ``errno.EINPROGRESS`` indicates that the connection is in progress, but hasn't finished yet. Different OSes will return different values, so you're going to have to check what's returned on your system. -You can use the :meth:`socket.connect_ex` method to avoid creating an exception. It will -just return the errno value. To poll, you can call :meth:`socket.connect_ex` again later +You can use the :meth:`~socket.socket.connect_ex` method +to avoid creating an exception. +It will just return the errno value. +To poll, you can call :meth:`~socket.socket.connect_ex` again later -- ``0`` or ``errno.EISCONN`` indicate that you're connected -- or you can pass this socket to :meth:`select.select` to check if it's writable. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 319c77776b7b8..8325f5b336413 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -25,7 +25,6 @@ Doc/c-api/unicode.rst Doc/extending/extending.rst Doc/extending/newtypes.rst Doc/faq/gui.rst -Doc/faq/library.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst From webhook-mailer at python.org Sun Aug 20 17:07:28 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 20 Aug 2023 21:07:28 -0000 Subject: [Python-checkins] [3.12] gh-107915: Handle errors in C API functions PyErr_Set*() and PyErr_Format() (GH-107918) (#108134) Message-ID: https://github.com/python/cpython/commit/97d67e9dabb3d66525a46dc4d802195ced24e6ef commit: 97d67e9dabb3d66525a46dc4d802195ced24e6ef branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-20T23:07:24+02:00 summary: [3.12] gh-107915: Handle errors in C API functions PyErr_Set*() and PyErr_Format() (GH-107918) (#108134) * gh-107915: Handle errors in C API functions PyErr_Set*() and PyErr_Format() (GH-107918) Such C API functions as PyErr_SetString(), PyErr_Format(), PyErr_SetFromErrnoWithFilename() and many others no longer crash or ignore errors if it failed to format the error message or decode the filename. Instead, they keep a corresponding error. (cherry picked from commit 633ea217a85f6b6ba5bdbc73094254d5811b3485) Co-authored-by: Serhiy Storchaka * Define PY_SSIZE_T_CLEAN. --------- Co-authored-by: Serhiy Storchaka files: A Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst M Lib/test/test_capi/test_exceptions.py M Modules/_testcapi/clinic/exceptions.c.h M Modules/_testcapi/exceptions.c M Python/errors.c diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index 118b575cba6df..b96cc7922a0ee 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -1,9 +1,12 @@ +import errno +import os import re import sys import unittest from test import support from test.support import import_helper +from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE from test.support.script_helper import assert_python_failure from test.support.testcase import ExceptionIsLikeMixin @@ -12,6 +15,8 @@ # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') +NULL = None + class Test_Exceptions(unittest.TestCase): def test_exception(self): @@ -189,6 +194,82 @@ def __repr__(self): self.assertEqual(exc.__notes__[0], 'Normalization failed: type=Broken args=') + def test_set_string(self): + """Test PyErr_SetString()""" + setstring = _testcapi.err_setstring + with self.assertRaises(ZeroDivisionError) as e: + setstring(ZeroDivisionError, b'error') + self.assertEqual(e.exception.args, ('error',)) + with self.assertRaises(ZeroDivisionError) as e: + setstring(ZeroDivisionError, '???????'.encode()) + self.assertEqual(e.exception.args, ('???????',)) + + with self.assertRaises(UnicodeDecodeError): + setstring(ZeroDivisionError, b'\xff') + self.assertRaises(SystemError, setstring, list, b'error') + # CRASHES setstring(ZeroDivisionError, NULL) + # CRASHES setstring(NULL, b'error') + + def test_format(self): + """Test PyErr_Format()""" + import_helper.import_module('ctypes') + from ctypes import pythonapi, py_object, c_char_p, c_int + name = "PyErr_Format" + PyErr_Format = getattr(pythonapi, name) + PyErr_Format.argtypes = (py_object, c_char_p,) + PyErr_Format.restype = py_object + with self.assertRaises(ZeroDivisionError) as e: + PyErr_Format(ZeroDivisionError, b'%s %d', b'error', c_int(42)) + self.assertEqual(e.exception.args, ('error 42',)) + with self.assertRaises(ZeroDivisionError) as e: + PyErr_Format(ZeroDivisionError, b'%s', '???????'.encode()) + self.assertEqual(e.exception.args, ('???????',)) + + with self.assertRaisesRegex(OverflowError, 'not in range'): + PyErr_Format(ZeroDivisionError, b'%c', c_int(-1)) + with self.assertRaisesRegex(ValueError, 'format string'): + PyErr_Format(ZeroDivisionError, b'\xff') + self.assertRaises(SystemError, PyErr_Format, list, b'error') + # CRASHES PyErr_Format(ZeroDivisionError, NULL) + # CRASHES PyErr_Format(py_object(), b'error') + + def test_setfromerrnowithfilename(self): + """Test PyErr_SetFromErrnoWithFilename()""" + setfromerrnowithfilename = _testcapi.err_setfromerrnowithfilename + ENOENT = errno.ENOENT + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, b'file') + self.assertEqual(e.exception.args, + (ENOENT, 'No such file or directory')) + self.assertEqual(e.exception.errno, ENOENT) + self.assertEqual(e.exception.filename, 'file') + + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, os.fsencode(TESTFN)) + self.assertEqual(e.exception.filename, TESTFN) + + if TESTFN_UNDECODABLE: + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, TESTFN_UNDECODABLE) + self.assertEqual(e.exception.filename, + os.fsdecode(TESTFN_UNDECODABLE)) + + with self.assertRaises(FileNotFoundError) as e: + setfromerrnowithfilename(ENOENT, OSError, NULL) + self.assertIsNone(e.exception.filename) + + with self.assertRaises(OSError) as e: + setfromerrnowithfilename(0, OSError, b'file') + self.assertEqual(e.exception.args, (0, 'Error')) + self.assertEqual(e.exception.errno, 0) + self.assertEqual(e.exception.filename, 'file') + + with self.assertRaises(ZeroDivisionError) as e: + setfromerrnowithfilename(ENOENT, ZeroDivisionError, b'file') + self.assertEqual(e.exception.args, + (ENOENT, 'No such file or directory', 'file')) + # CRASHES setfromerrnowithfilename(ENOENT, NULL, b'error') + class Test_PyUnstable_Exc_PrepReraiseStar(ExceptionIsLikeMixin, unittest.TestCase): diff --git a/Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst b/Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst new file mode 100644 index 0000000000000..58ee3f169a28c --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-13-12-33-00.gh-issue-107915.jQ0wOi.rst @@ -0,0 +1,4 @@ +Such C API functions as ``PyErr_SetString()``, ``PyErr_Format()``, +``PyErr_SetFromErrnoWithFilename()`` and many others no longer crash or +ignore errors if it failed to format the error message or decode the +filename. Instead, they keep a corresponding error. diff --git a/Modules/_testcapi/clinic/exceptions.c.h b/Modules/_testcapi/clinic/exceptions.c.h index 01730ffa2ed03..16954a5ebf3e5 100644 --- a/Modules/_testcapi/clinic/exceptions.c.h +++ b/Modules/_testcapi/clinic/exceptions.c.h @@ -215,6 +215,68 @@ _testcapi_exc_set_object_fetch(PyObject *module, PyObject *const *args, Py_ssize return return_value; } +PyDoc_STRVAR(_testcapi_err_setstring__doc__, +"err_setstring($module, exc, value, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_ERR_SETSTRING_METHODDEF \ + {"err_setstring", _PyCFunction_CAST(_testcapi_err_setstring), METH_FASTCALL, _testcapi_err_setstring__doc__}, + +static PyObject * +_testcapi_err_setstring_impl(PyObject *module, PyObject *exc, + const char *value, Py_ssize_t value_length); + +static PyObject * +_testcapi_err_setstring(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *exc; + const char *value; + Py_ssize_t value_length; + + if (!_PyArg_ParseStack(args, nargs, "Oz#:err_setstring", + &exc, &value, &value_length)) { + goto exit; + } + return_value = _testcapi_err_setstring_impl(module, exc, value, value_length); + +exit: + return return_value; +} + +PyDoc_STRVAR(_testcapi_err_setfromerrnowithfilename__doc__, +"err_setfromerrnowithfilename($module, error, exc, value, /)\n" +"--\n" +"\n"); + +#define _TESTCAPI_ERR_SETFROMERRNOWITHFILENAME_METHODDEF \ + {"err_setfromerrnowithfilename", _PyCFunction_CAST(_testcapi_err_setfromerrnowithfilename), METH_FASTCALL, _testcapi_err_setfromerrnowithfilename__doc__}, + +static PyObject * +_testcapi_err_setfromerrnowithfilename_impl(PyObject *module, int error, + PyObject *exc, const char *value, + Py_ssize_t value_length); + +static PyObject * +_testcapi_err_setfromerrnowithfilename(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int error; + PyObject *exc; + const char *value; + Py_ssize_t value_length; + + if (!_PyArg_ParseStack(args, nargs, "iOz#:err_setfromerrnowithfilename", + &error, &exc, &value, &value_length)) { + goto exit; + } + return_value = _testcapi_err_setfromerrnowithfilename_impl(module, error, exc, value, value_length); + +exit: + return return_value; +} + PyDoc_STRVAR(_testcapi_raise_exception__doc__, "raise_exception($module, exception, num_args, /)\n" "--\n" @@ -426,4 +488,4 @@ _testcapi_unstable_exc_prep_reraise_star(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=fd6aef54f195c77b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d574342d716e98b5 input=a9049054013a1b77]*/ diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index a627bf1717fe0..4ae9ba4044df8 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -1,6 +1,9 @@ +#define PY_SSIZE_T_CLEAN #include "parts.h" #include "clinic/exceptions.c.h" +#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); + /*[clinic input] module _testcapi [clinic start generated code]*/ @@ -129,6 +132,43 @@ _testcapi_exc_set_object_fetch_impl(PyObject *module, PyObject *exc, return value; } +/*[clinic input] +_testcapi.err_setstring + exc: object + value: str(zeroes=True, accept={robuffer, str, NoneType}) + / +[clinic start generated code]*/ + +static PyObject * +_testcapi_err_setstring_impl(PyObject *module, PyObject *exc, + const char *value, Py_ssize_t value_length) +/*[clinic end generated code: output=fba8705e5703dd3f input=e8a95fad66d9004b]*/ +{ + NULLABLE(exc); + PyErr_SetString(exc, value); + return NULL; +} + +/*[clinic input] +_testcapi.err_setfromerrnowithfilename + error: int + exc: object + value: str(zeroes=True, accept={robuffer, str, NoneType}) + / +[clinic start generated code]*/ + +static PyObject * +_testcapi_err_setfromerrnowithfilename_impl(PyObject *module, int error, + PyObject *exc, const char *value, + Py_ssize_t value_length) +/*[clinic end generated code: output=d02df5749a01850e input=ff7c384234bf097f]*/ +{ + NULLABLE(exc); + errno = error; + PyErr_SetFromErrnoWithFilename(exc, value); + return NULL; +} + /*[clinic input] _testcapi.raise_exception exception as exc: object @@ -338,6 +378,8 @@ static PyMethodDef test_methods[] = { _TESTCAPI_MAKE_EXCEPTION_WITH_DOC_METHODDEF _TESTCAPI_EXC_SET_OBJECT_METHODDEF _TESTCAPI_EXC_SET_OBJECT_FETCH_METHODDEF + _TESTCAPI_ERR_SETSTRING_METHODDEF + _TESTCAPI_ERR_SETFROMERRNOWITHFILENAME_METHODDEF _TESTCAPI_RAISE_EXCEPTION_METHODDEF _TESTCAPI_RAISE_MEMORYERROR_METHODDEF _TESTCAPI_SET_EXC_INFO_METHODDEF diff --git a/Python/errors.c b/Python/errors.c index a8000ac94918d..f1d3007b9fdf8 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -292,8 +292,10 @@ _PyErr_SetString(PyThreadState *tstate, PyObject *exception, const char *string) { PyObject *value = PyUnicode_FromString(string); - _PyErr_SetObject(tstate, exception, value); - Py_XDECREF(value); + if (value != NULL) { + _PyErr_SetObject(tstate, exception, value); + Py_DECREF(value); + } } void @@ -915,7 +917,13 @@ PyErr_SetFromErrnoWithFilenameObjects(PyObject *exc, PyObject *filenameObject, P PyObject * PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) { - PyObject *name = filename ? PyUnicode_DecodeFSDefault(filename) : NULL; + PyObject *name = NULL; + if (filename) { + name = PyUnicode_DecodeFSDefault(filename); + if (name == NULL) { + return NULL; + } + } PyObject *result = PyErr_SetFromErrnoWithFilenameObjects(exc, name, NULL); Py_XDECREF(name); return result; @@ -1012,7 +1020,13 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilename( int ierr, const char *filename) { - PyObject *name = filename ? PyUnicode_DecodeFSDefault(filename) : NULL; + PyObject *name = NULL; + if (filename) { + name = PyUnicode_DecodeFSDefault(filename); + if (name == NULL) { + return NULL; + } + } PyObject *ret = PyErr_SetExcFromWindowsErrWithFilenameObjects(exc, ierr, name, @@ -1036,7 +1050,13 @@ PyObject *PyErr_SetFromWindowsErrWithFilename( int ierr, const char *filename) { - PyObject *name = filename ? PyUnicode_DecodeFSDefault(filename) : NULL; + PyObject *name = NULL; + if (filename) { + name = PyUnicode_DecodeFSDefault(filename); + if (name == NULL) { + return NULL; + } + } PyObject *result = PyErr_SetExcFromWindowsErrWithFilenameObjects( PyExc_OSError, ierr, name, NULL); @@ -1161,9 +1181,10 @@ _PyErr_FormatV(PyThreadState *tstate, PyObject *exception, _PyErr_Clear(tstate); string = PyUnicode_FromFormatV(format, vargs); - - _PyErr_SetObject(tstate, exception, string); - Py_XDECREF(string); + if (string != NULL) { + _PyErr_SetObject(tstate, exception, string); + Py_DECREF(string); + } return NULL; } From webhook-mailer at python.org Sun Aug 20 17:08:20 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 20 Aug 2023 21:08:20 -0000 Subject: [Python-checkins] [3.12] Docs: Fix Sphinx warnings in logging.rst (GH-108139) (#108174) Message-ID: https://github.com/python/cpython/commit/fa6cd7f43f3f8ce9b31051b30438549b2c6bc255 commit: fa6cd7f43f3f8ce9b31051b30438549b2c6bc255 branch: 3.12 author: Erlend E. Aasland committer: Yhg1s date: 2023-08-20T23:08:16+02:00 summary: [3.12] Docs: Fix Sphinx warnings in logging.rst (GH-108139) (#108174) (cherry picked from commit c735e79afb62324624864e1943f84825249f58ed) Co-authored-by: Adam Turner <9087854+aa-turner at users.noreply.github.com> files: M Doc/library/logging.rst M Doc/tools/.nitignore diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index a92ef1787af74..edec0e0cdfa99 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -439,7 +439,7 @@ Handler Objects Handlers have the following attributes and methods. Note that :class:`Handler` is never instantiated directly; this class acts as a base for more useful -subclasses. However, the :meth:`__init__` method in subclasses needs to call +subclasses. However, the :meth:`!__init__` method in subclasses needs to call :meth:`Handler.__init__`. .. class:: Handler @@ -1015,23 +1015,25 @@ information into logging calls. For a usage example, see the section on 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the (possibly modified) versions of the arguments passed in. -In addition to the above, :class:`LoggerAdapter` supports the following -methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, -:meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, -:meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, -:meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and -:meth:`~Logger.hasHandlers`. These methods have the same signatures as their -counterparts in :class:`Logger`, so you can use the two types of instances -interchangeably. + In addition to the above, :class:`LoggerAdapter` supports the following + methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, + :meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, + :meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, + :meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and + :meth:`~Logger.hasHandlers`. These methods have the same signatures as their + counterparts in :class:`Logger`, so you can use the two types of instances + interchangeably. -.. versionchanged:: 3.2 - The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, - :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added - to :class:`LoggerAdapter`. These methods delegate to the underlying logger. + .. versionchanged:: 3.2 + + The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, + :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added + to :class:`LoggerAdapter`. These methods delegate to the underlying logger. + + .. versionchanged:: 3.6 -.. versionchanged:: 3.6 - Attribute :attr:`manager` and method :meth:`_log` were added, which - delegate to the underlying logger and allow adapters to be nested. + Attribute :attr:`!manager` and method :meth:`!_log` were added, which + delegate to the underlying logger and allow adapters to be nested. Thread Safety @@ -1415,8 +1417,8 @@ functions. .. function:: setLoggerClass(klass) Tells the logging system to use the class *klass* when instantiating a logger. - The class should define :meth:`__init__` such that only a name argument is - required, and the :meth:`__init__` should call :meth:`Logger.__init__`. This + The class should define :meth:`!__init__` such that only a name argument is + required, and the :meth:`!__init__` should call :meth:`!Logger.__init__`. This function is typically called before any loggers are instantiated by applications which need to use custom logger behavior. After this call, as at any other time, do not instantiate loggers directly using the subclass: continue to use diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index a0a121b4b9dbe..4dc8bd9ea7050 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -98,7 +98,6 @@ Doc/library/inspect.rst Doc/library/locale.rst Doc/library/logging.config.rst Doc/library/logging.handlers.rst -Doc/library/logging.rst Doc/library/lzma.rst Doc/library/mailbox.rst Doc/library/mmap.rst From webhook-mailer at python.org Sun Aug 20 17:08:43 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 20 Aug 2023 21:08:43 -0000 Subject: [Python-checkins] [3.12] Docs: Fix Sphinx warnings in sys.rst (#108106) (#108178) Message-ID: https://github.com/python/cpython/commit/399825ed26e0a1873082348f911f34106fc72973 commit: 399825ed26e0a1873082348f911f34106fc72973 branch: 3.12 author: Erlend E. Aasland committer: Yhg1s date: 2023-08-20T23:08:39+02:00 summary: [3.12] Docs: Fix Sphinx warnings in sys.rst (#108106) (#108178) (cherry picked from commit 29fa7afef94d74e18d97485c085d1ccf80c16ca3) - Mark up named tuple attributes as attributes - Remove links for external functions - io.BufferedIOBase has no 'buffer' attribute; remove the link and mark up using :attr:`!buffer` - (Re)format some tables as bullet lists: - sys._emscripten_info - sys.hash_info - sys.int_info - sys.thread_info - In the paragraphs mentioning 'f_trace_lines' and 'f_trace_opcodes', add links to the frame objects reference. Co-authored-by: Alex Waygood Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/library/sys.rst M Doc/library/textwrap.rst M Doc/tools/.nitignore diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 334a2b077dd6f..e7b34a83ebe9a 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -333,23 +333,21 @@ always available. *wasm32-emscripten* platform. The named tuple is provisional and may change in the future. - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------------+ - | Attribute | Explanation | - +=============================+==============================================+ - | :const:`emscripten_version` | Emscripten version as tuple of ints | - | | (major, minor, micro), e.g. ``(3, 1, 8)``. | - +-----------------------------+----------------------------------------------+ - | :const:`runtime` | Runtime string, e.g. browser user agent, | - | | ``'Node.js v14.18.2'``, or ``'UNKNOWN'``. | - +-----------------------------+----------------------------------------------+ - | :const:`pthreads` | ``True`` if Python is compiled with | - | | Emscripten pthreads support. | - +-----------------------------+----------------------------------------------+ - | :const:`shared_memory` | ``True`` if Python is compiled with shared | - | | memory support. | - +-----------------------------+----------------------------------------------+ + .. attribute:: _emscripten_info.emscripten_version + + Emscripten version as tuple of ints (major, minor, micro), e.g. ``(3, 1, 8)``. + + .. attribute:: _emscripten_info.runtime + + Runtime string, e.g. browser user agent, ``'Node.js v14.18.2'``, or ``'UNKNOWN'``. + + .. attribute:: _emscripten_info.pthreads + + ``True`` if Python is compiled with Emscripten pthreads support. + + .. attribute:: _emscripten_info.shared_memory + + ``True`` if Python is compiled with shared memory support. .. availability:: Emscripten. @@ -515,28 +513,62 @@ always available. The :term:`named tuple` *flags* exposes the status of command line flags. The attributes are read only. - ============================== ============================================================================================================== - attribute flag - ============================== ============================================================================================================== - :const:`debug` :option:`-d` - :const:`inspect` :option:`-i` - :const:`interactive` :option:`-i` - :const:`isolated` :option:`-I` - :const:`optimize` :option:`-O` or :option:`-OO` - :const:`dont_write_bytecode` :option:`-B` - :const:`no_user_site` :option:`-s` - :const:`no_site` :option:`-S` - :const:`ignore_environment` :option:`-E` - :const:`verbose` :option:`-v` - :const:`bytes_warning` :option:`-b` - :const:`quiet` :option:`-q` - :const:`hash_randomization` :option:`-R` - :const:`dev_mode` :option:`-X dev <-X>` (:ref:`Python Development Mode `) - :const:`utf8_mode` :option:`-X utf8 <-X>` - :const:`safe_path` :option:`-P` - :const:`int_max_str_digits` :option:`-X int_max_str_digits <-X>` (:ref:`integer string conversion length limitation `) - :const:`warn_default_encoding` :option:`-X warn_default_encoding <-X>` - ============================== ============================================================================================================== + .. list-table:: + + * - .. attribute:: flags.debug + - :option:`-d` + + * - .. attribute:: flags.inspect + - :option:`-i` + + * - .. attribute:: flags.interactive + - :option:`-i` + + * - .. attribute:: flags.isolated + - :option:`-I` + + * - .. attribute:: flags.optimize + - :option:`-O` or :option:`-OO` + + * - .. attribute:: flags.dont_write_bytecode + - :option:`-B` + + * - .. attribute:: flags.no_user_site + - :option:`-s` + + * - .. attribute:: flags.no_site + - :option:`-S` + + * - .. attribute:: flags.ignore_environment + - :option:`-E` + + * - .. attribute:: flags.verbose + - :option:`-v` + + * - .. attribute:: flags.bytes_warning + - :option:`-b` + + * - .. attribute:: flags.quiet + - :option:`-q` + + * - .. attribute:: flags.hash_randomization + - :option:`-R` + + * - .. attribute:: flags.dev_mode + - :option:`-X dev <-X>` (:ref:`Python Development Mode `) + + * - .. attribute:: flags.utf8_mode + - :option:`-X utf8 <-X>` + + * - .. attribute:: flags.safe_path + - :option:`-P` + + * - .. attribute:: flags.int_max_str_digits + - :option:`-X int_max_str_digits <-X>` + (:ref:`integer string conversion length limitation `) + + * - .. attribute:: flags.warn_default_encoding + - :option:`-X warn_default_encoding <-X>` .. versionchanged:: 3.2 Added ``quiet`` attribute for the new :option:`-q` flag. @@ -923,8 +955,8 @@ always available. | | a domain controller. | +---------------------------------------+---------------------------------+ - This function wraps the Win32 :c:func:`GetVersionEx` function; see the - Microsoft documentation on :c:func:`OSVERSIONINFOEX` for more information + This function wraps the Win32 :c:func:`!GetVersionEx` function; see the + Microsoft documentation on :c:func:`!OSVERSIONINFOEX` for more information about these fields. *platform_version* returns the major version, minor version and @@ -982,28 +1014,37 @@ always available. implementation. For more details about hashing of numeric types, see :ref:`numeric-hash`. - +---------------------+--------------------------------------------------+ - | attribute | explanation | - +=====================+==================================================+ - | :const:`width` | width in bits used for hash values | - +---------------------+--------------------------------------------------+ - | :const:`modulus` | prime modulus P used for numeric hash scheme | - +---------------------+--------------------------------------------------+ - | :const:`inf` | hash value returned for a positive infinity | - +---------------------+--------------------------------------------------+ - | :const:`nan` | (this attribute is no longer used) | - +---------------------+--------------------------------------------------+ - | :const:`imag` | multiplier used for the imaginary part of a | - | | complex number | - +---------------------+--------------------------------------------------+ - | :const:`algorithm` | name of the algorithm for hashing of str, bytes, | - | | and memoryview | - +---------------------+--------------------------------------------------+ - | :const:`hash_bits` | internal output size of the hash algorithm | - +---------------------+--------------------------------------------------+ - | :const:`seed_bits` | size of the seed key of the hash algorithm | - +---------------------+--------------------------------------------------+ + .. attribute:: hash_info.width + + The width in bits used for hash values + + .. attribute:: hash_info.modulus + The prime modulus P used for numeric hash scheme + + .. attribute:: hash_info.inf + + The hash value returned for a positive infinity + + .. attribute:: hash_info.nan + + (This attribute is no longer used) + + .. attribute:: hash_info.imag + + The multiplier used for the imaginary part of a complex number + + .. attribute:: hash_info.algorithm + + The name of the algorithm for hashing of str, bytes, and memoryview + + .. attribute:: hash_info.hash_bits + + The internal output size of the hash algorithm + + .. attribute:: hash_info.seed_bits + + The size of the seed key of the hash algorithm .. versionadded:: 3.2 @@ -1081,32 +1122,31 @@ always available. A :term:`named tuple` that holds information about Python's internal representation of integers. The attributes are read only. - .. tabularcolumns:: |l|L| - - +----------------------------------------+-----------------------------------------------+ - | Attribute | Explanation | - +========================================+===============================================+ - | :const:`bits_per_digit` | number of bits held in each digit. Python | - | | integers are stored internally in base | - | | ``2**int_info.bits_per_digit`` | - +----------------------------------------+-----------------------------------------------+ - | :const:`sizeof_digit` | size in bytes of the C type used to | - | | represent a digit | - +----------------------------------------+-----------------------------------------------+ - | :const:`default_max_str_digits` | default value for | - | | :func:`sys.get_int_max_str_digits` when it | - | | is not otherwise explicitly configured. | - +----------------------------------------+-----------------------------------------------+ - | :const:`str_digits_check_threshold` | minimum non-zero value for | - | | :func:`sys.set_int_max_str_digits`, | - | | :envvar:`PYTHONINTMAXSTRDIGITS`, or | - | | :option:`-X int_max_str_digits <-X>`. | - +----------------------------------------+-----------------------------------------------+ + .. attribute:: int_info.bits_per_digit + + The number of bits held in each digit. + Python integers are stored internally in base ``2**int_info.bits_per_digit``. + + .. attribute:: int_info.sizeof_digit + + The size in bytes of the C type used to represent a digit. + + .. attribute:: int_info.default_max_str_digits + + The default value for :func:`sys.get_int_max_str_digits` + when it is not otherwise explicitly configured. + + .. attribute:: int_info.str_digits_check_threshold + + The minimum non-zero value for :func:`sys.set_int_max_str_digits`, + :envvar:`PYTHONINTMAXSTRDIGITS`, or :option:`-X int_max_str_digits <-X>`. .. versionadded:: 3.1 .. versionchanged:: 3.11 - Added ``default_max_str_digits`` and ``str_digits_check_threshold``. + + Added :attr:`~int_info.default_max_str_digits` and + :attr:`~int_info.str_digits_check_threshold`. .. data:: __interactivehook__ @@ -1533,7 +1573,7 @@ always available. :file:`Objects/lnotab_notes.txt` for a detailed explanation of how this works. Per-line events may be disabled for a frame by setting - :attr:`f_trace_lines` to :const:`False` on that frame. + :attr:`!f_trace_lines` to :const:`False` on that :ref:`frame `. ``'return'`` A function (or other code block) is about to return. The local trace @@ -1551,8 +1591,8 @@ always available. opcode details). The local trace function is called; *arg* is ``None``; the return value specifies the new local trace function. Per-opcode events are not emitted by default: they must be explicitly - requested by setting :attr:`f_trace_opcodes` to :const:`True` on the - frame. + requested by setting :attr:`!f_trace_opcodes` to :const:`True` on the + :ref:`frame `. Note that as an exception is propagated down the chain of callers, an ``'exception'`` event is generated at each level. @@ -1581,8 +1621,8 @@ always available. .. versionchanged:: 3.7 - ``'opcode'`` event type added; :attr:`f_trace_lines` and - :attr:`f_trace_opcodes` attributes added to frames + ``'opcode'`` event type added; :attr:`!f_trace_lines` and + :attr:`!f_trace_opcodes` attributes added to frames .. function:: set_asyncgen_hooks(firstiter, finalizer) @@ -1739,7 +1779,7 @@ always available. However, if you are writing a library (and do not control in which context its code will be executed), be aware that the standard streams may be replaced with file-like objects like :class:`io.StringIO` which - do not support the :attr:`~io.BufferedIOBase.buffer` attribute. + do not support the :attr:!buffer` attribute. .. data:: __stdin__ @@ -1787,29 +1827,28 @@ always available. A :term:`named tuple` holding information about the thread implementation. - .. tabularcolumns:: |l|p{0.7\linewidth}| - - +------------------+---------------------------------------------------------+ - | Attribute | Explanation | - +==================+=========================================================+ - | :const:`name` | Name of the thread implementation: | - | | | - | | * ``'nt'``: Windows threads | - | | * ``'pthread'``: POSIX threads | - | | * ``'pthread-stubs'``: stub POSIX threads | - | | (on WebAssembly platforms without threading support) | - | | * ``'solaris'``: Solaris threads | - +------------------+---------------------------------------------------------+ - | :const:`lock` | Name of the lock implementation: | - | | | - | | * ``'semaphore'``: a lock uses a semaphore | - | | * ``'mutex+cond'``: a lock uses a mutex | - | | and a condition variable | - | | * ``None`` if this information is unknown | - +------------------+---------------------------------------------------------+ - | :const:`version` | Name and version of the thread library. It is a string, | - | | or ``None`` if this information is unknown. | - +------------------+---------------------------------------------------------+ + .. attribute:: thread_info.name + + The name of the thread implementation: + + * ``"nt"``: Windows threads + * ``"pthread"``: POSIX threads + * ``"pthread-stubs"``: stub POSIX threads + (on WebAssembly platforms without threading support) + * ``"solaris"``: Solaris threads + + .. attribute:: thread_info.lock + + The name of the lock implementation: + + * ``"semaphore"``: a lock uses a semaphore + * ``"mutex+cond"``: a lock uses a mutex and a condition variable + * ``None`` if this information is unknown + + .. attribute:: thread_info.version + + The name and version of the thread library. + It is a string, or ``None`` if this information is unknown. .. versionadded:: 3.3 diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index a150eefbf932e..e2952ce3cc2ca 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -60,7 +60,7 @@ functions should be good enough; otherwise, you should use an instance of First the whitespace in *text* is collapsed (all whitespace is replaced by single spaces). If the result fits in the *width*, it is returned. Otherwise, enough words are dropped from the end so that the remaining words - plus the :attr:`.placeholder` fit within :attr:`.width`:: + plus the *placeholder* fit within *width*:: >>> textwrap.shorten("Hello world!", width=12) 'Hello world!' diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 4dc8bd9ea7050..ba976bc50cf20 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -136,7 +136,6 @@ Doc/library/stdtypes.rst Doc/library/string.rst Doc/library/subprocess.rst Doc/library/sunau.rst -Doc/library/sys.rst Doc/library/sys_path_init.rst Doc/library/syslog.rst Doc/library/tarfile.rst From webhook-mailer at python.org Sun Aug 20 17:09:01 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 20 Aug 2023 21:09:01 -0000 Subject: [Python-checkins] [3.12] Resolve reference warnings in faq/design.rst (GH-108148) (#108180) Message-ID: https://github.com/python/cpython/commit/a6f15af866c4584b970c72b805a2a4a6f3f63b83 commit: a6f15af866c4584b970c72b805a2a4a6f3f63b83 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-20T23:08:57+02:00 summary: [3.12] Resolve reference warnings in faq/design.rst (GH-108148) (#108180) Resolve reference warnings in faq/design.rst (GH-108148) (cherry picked from commit 92815cc7cf3df8ab702c7cea4efaef349a4b0480) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: M Doc/faq/design.rst M Doc/tools/.nitignore diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 11d01374dc1e7..ae02c443e5938 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -584,9 +584,9 @@ exhaustive test suites that exercise every line of code in a module. An appropriate testing discipline can help build large complex applications in Python as well as having interface specifications would. In fact, it can be better because an interface specification cannot test certain properties of a -program. For example, the :meth:`list.append` method is expected to add new elements +program. For example, the :meth:`!list.append` method is expected to add new elements to the end of some internal list; an interface specification cannot test that -your :meth:`list.append` implementation will actually do this correctly, but it's +your :meth:`!list.append` implementation will actually do this correctly, but it's trivial to check this property in a test suite. Writing test suites is very helpful, and you might want to design your code to diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index ba976bc50cf20..5a28274cff6c4 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -25,7 +25,6 @@ Doc/c-api/typeobj.rst Doc/c-api/unicode.rst Doc/extending/extending.rst Doc/extending/newtypes.rst -Doc/faq/design.rst Doc/faq/gui.rst Doc/faq/library.rst Doc/glossary.rst From webhook-mailer at python.org Sun Aug 20 17:09:22 2023 From: webhook-mailer at python.org (Yhg1s) Date: Sun, 20 Aug 2023 21:09:22 -0000 Subject: [Python-checkins] [3.12] Resolve reference warnings in faq/library.rst (GH-108149) (#108182) Message-ID: https://github.com/python/cpython/commit/f2cc00527efeb69497e4304e826dd3a518005d1a commit: f2cc00527efeb69497e4304e826dd3a518005d1a branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-20T23:09:18+02:00 summary: [3.12] Resolve reference warnings in faq/library.rst (GH-108149) (#108182) Resolve reference warnings in faq/library.rst (GH-108149) (cherry picked from commit 6323bc33ff9f445a947adf4af42b8be7e44c730c) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Erlend E. Aasland Co-authored-by: Hugo van Kemenade files: M Doc/faq/library.rst M Doc/tools/.nitignore diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index b43c7505c0401..c69910718f0c9 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -111,7 +111,7 @@ Is there an equivalent to C's onexit() in Python? ------------------------------------------------- The :mod:`atexit` module provides a register function that is similar to C's -:c:func:`onexit`. +:c:func:`!onexit`. Why don't my signal handlers work? @@ -397,7 +397,7 @@ These aren't:: D[x] = D[x] + 1 Operations that replace other objects may invoke those other objects' -:meth:`__del__` method when their reference count reaches zero, and that can +:meth:`~object.__del__` method when their reference count reaches zero, and that can affect things. This is especially true for the mass updates to dictionaries and lists. When in doubt, use a mutex! @@ -765,14 +765,17 @@ The :mod:`select` module is commonly used to help with asynchronous I/O on sockets. To prevent the TCP connect from blocking, you can set the socket to non-blocking -mode. Then when you do the :meth:`socket.connect`, you will either connect immediately +mode. Then when you do the :meth:`~socket.socket.connect`, +you will either connect immediately (unlikely) or get an exception that contains the error number as ``.errno``. ``errno.EINPROGRESS`` indicates that the connection is in progress, but hasn't finished yet. Different OSes will return different values, so you're going to have to check what's returned on your system. -You can use the :meth:`socket.connect_ex` method to avoid creating an exception. It will -just return the errno value. To poll, you can call :meth:`socket.connect_ex` again later +You can use the :meth:`~socket.socket.connect_ex` method +to avoid creating an exception. +It will just return the errno value. +To poll, you can call :meth:`~socket.socket.connect_ex` again later -- ``0`` or ``errno.EISCONN`` indicate that you're connected -- or you can pass this socket to :meth:`select.select` to check if it's writable. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 5a28274cff6c4..b1894c4d3df37 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -26,7 +26,6 @@ Doc/c-api/unicode.rst Doc/extending/extending.rst Doc/extending/newtypes.rst Doc/faq/gui.rst -Doc/faq/library.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst From webhook-mailer at python.org Sun Aug 20 19:40:44 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Sun, 20 Aug 2023 23:40:44 -0000 Subject: [Python-checkins] gh-104504: cases generator: Add `--warn-unreachable` to the mypy config (#108112) Message-ID: https://github.com/python/cpython/commit/05ef4ca94c694bc50e6fd221fe648d05d227f3a4 commit: 05ef4ca94c694bc50e6fd221fe648d05d227f3a4 branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-21T00:40:41+01:00 summary: gh-104504: cases generator: Add `--warn-unreachable` to the mypy config (#108112) files: A Tools/cases_generator/_typing_backports.py M Tools/cases_generator/analysis.py M Tools/cases_generator/generate_cases.py M Tools/cases_generator/mypy.ini diff --git a/Tools/cases_generator/_typing_backports.py b/Tools/cases_generator/_typing_backports.py new file mode 100644 index 0000000000000..c2aa50804cefe --- /dev/null +++ b/Tools/cases_generator/_typing_backports.py @@ -0,0 +1,15 @@ +"""Backports from newer versions of the typing module. + +We backport these features here so that Python can still build +while using an older Python version for PYTHON_FOR_REGEN. +""" + +from typing import NoReturn + + +def assert_never(obj: NoReturn) -> NoReturn: + """Statically assert that a line of code is unreachable. + + Backport of typing.assert_never (introduced in Python 3.11). + """ + raise AssertionError(f"Expected code to be unreachable, but got: {obj}") diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 48f2db981c95b..72fb2d761dbfa 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -2,6 +2,7 @@ import sys import typing +from _typing_backports import assert_never from flags import InstructionFlags, variable_used from formatting import prettify_filename, UNUSED from instructions import ( @@ -172,7 +173,7 @@ def parse_file(self, filename: str, instrs_idx: dict[str, int]) -> None: self.pseudos[name] = thing self.everything.append(thing) case _: - typing.assert_never(thing) + assert_never(thing) if not psr.eof(): raise psr.make_syntax_error(f"Extra stuff at the end of {filename}") @@ -368,7 +369,7 @@ def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: # SAVE_IP in a macro is a no-op in Tier 1 flags.add(instr.instr_flags) case _: - typing.assert_never(component) + assert_never(component) format = "IB" if flags.HAS_ARG_FLAG else "IX" if offset: format += "C" + "0" * (offset - 1) @@ -409,5 +410,5 @@ def check_macro_components( case parsing.CacheEffect(): components.append(uop) case _: - typing.assert_never(uop) + assert_never(uop) return components diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index de31129ac0548..7b1880b98a8fe 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -13,6 +13,7 @@ from collections.abc import Iterator import stacking # Early import to avoid circular import +from _typing_backports import assert_never from analysis import Analyzer from formatting import Formatter, list_effect_size from flags import InstructionFlags, variable_used @@ -146,7 +147,7 @@ class Generator(Analyzer): def get_stack_effect_info( self, thing: parsing.InstDef | parsing.Macro | parsing.Pseudo - ) -> tuple[AnyInstruction | None, str | None, str | None]: + ) -> tuple[AnyInstruction | None, str, str]: def effect_str(effects: list[StackEffect]) -> str: n_effect, sym_effect = list_effect_size(effects) if sym_effect: @@ -154,8 +155,6 @@ def effect_str(effects: list[StackEffect]) -> str: return str(n_effect) instr: AnyInstruction | None - popped: str | None - pushed: str | None match thing: case parsing.InstDef(): if thing.kind != "op" or self.instrs[thing.name].is_viable_uop(): @@ -171,10 +170,9 @@ def effect_str(effects: list[StackEffect]) -> str: popped, pushed = stacking.get_stack_effect_info_for_macro(instr) case parsing.Pseudo(): instr = self.pseudo_instrs[thing.name] - popped = pushed = None # Calculate stack effect, and check that it's the the same # for all targets. - for target in self.pseudos[thing.name].targets: + for idx, target in enumerate(self.pseudos[thing.name].targets): target_instr = self.instrs.get(target) # Currently target is always an instr. This could change # in the future, e.g., if we have a pseudo targetting a @@ -182,14 +180,13 @@ def effect_str(effects: list[StackEffect]) -> str: assert target_instr target_popped = effect_str(target_instr.input_effects) target_pushed = effect_str(target_instr.output_effects) - if pushed is None: - assert popped is None + if idx == 0: popped, pushed = target_popped, target_pushed else: assert popped == target_popped assert pushed == target_pushed case _: - typing.assert_never(thing) + assert_never(thing) return instr, popped, pushed @contextlib.contextmanager @@ -209,7 +206,6 @@ def write_stack_effect_functions(self) -> None: continue instr, popped, pushed = self.get_stack_effect_info(thing) if instr is not None: - assert popped is not None and pushed is not None popped_data.append((instr, popped)) pushed_data.append((instr, pushed)) @@ -379,7 +375,6 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No # Compute the set of all instruction formats. all_formats: set[str] = set() for thing in self.everything: - format: str | None match thing: case OverriddenInstructionPlaceHolder(): continue @@ -388,17 +383,15 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No case parsing.Macro(): format = self.macro_instrs[thing.name].instr_fmt case parsing.Pseudo(): - format = None - for target in self.pseudos[thing.name].targets: + for idx, target in enumerate(self.pseudos[thing.name].targets): target_instr = self.instrs.get(target) assert target_instr - if format is None: + if idx == 0: format = target_instr.instr_fmt else: assert format == target_instr.instr_fmt - assert format is not None case _: - typing.assert_never(thing) + assert_never(thing) all_formats.add(format) # Turn it into a sorted list of enum values. @@ -488,7 +481,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.pseudo_instrs[thing.name] ) case _: - typing.assert_never(thing) + assert_never(thing) with self.metadata_item( "const struct opcode_macro_expansion " @@ -525,7 +518,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No case parsing.Pseudo(): pass case _: - typing.assert_never(thing) + assert_never(thing) with self.metadata_item( "const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]", "=", ";" @@ -774,7 +767,7 @@ def write_instructions( case parsing.Pseudo(): pass case _: - typing.assert_never(thing) + assert_never(thing) print( f"Wrote {n_instrs} instructions and {n_macros} macros " @@ -818,7 +811,7 @@ def write_executor_instructions( case parsing.Pseudo(): pass case _: - typing.assert_never(thing) + assert_never(thing) print( f"Wrote {n_instrs} instructions and {n_uops} ops to {executor_filename}", file=sys.stderr, @@ -850,7 +843,7 @@ def write_abstract_interpreter_instructions( case parsing.Pseudo(): pass case _: - typing.assert_never(thing) + assert_never(thing) print( f"Wrote some stuff to {abstract_interpreter_filename}", file=sys.stderr, diff --git a/Tools/cases_generator/mypy.ini b/Tools/cases_generator/mypy.ini index 7480841bf07ed..54ccb8c5c1a3a 100644 --- a/Tools/cases_generator/mypy.ini +++ b/Tools/cases_generator/mypy.ini @@ -2,13 +2,12 @@ files = Tools/cases_generator/ pretty = True +# Make sure Python can still be built +# using Python 3.10 for `PYTHON_FOR_REGEN`... python_version = 3.10 -# Be strict: +# ...And be strict: strict = True strict_concatenate = True enable_error_code = ignore-without-code,redundant-expr,truthy-bool - -# Don't enable this one yet; -# it has a lot of false positives on `cases_generator` -warn_unreachable = False +warn_unreachable = True From webhook-mailer at python.org Sun Aug 20 20:54:14 2023 From: webhook-mailer at python.org (JelleZijlstra) Date: Mon, 21 Aug 2023 00:54:14 -0000 Subject: [Python-checkins] gh-107526: Revert "gh-100357: Convert several functions in bltinsmodule to AC" (#107542) Message-ID: https://github.com/python/cpython/commit/db6dc6ce41f42ec2cb4fdfbc0a099114f42e888f commit: db6dc6ce41f42ec2cb4fdfbc0a099114f42e888f branch: main author: Nikita Sobolev committer: JelleZijlstra date: 2023-08-20T17:54:10-07:00 summary: gh-107526: Revert "gh-100357: Convert several functions in bltinsmodule to AC" (#107542) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-09-15-05-27.gh-issue-107526.PB32z-.rst M Python/bltinmodule.c M Python/clinic/bltinmodule.c.h diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-09-15-05-27.gh-issue-107526.PB32z-.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-09-15-05-27.gh-issue-107526.PB32z-.rst new file mode 100644 index 0000000000000..42ea09e78d41c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-09-15-05-27.gh-issue-107526.PB32z-.rst @@ -0,0 +1,2 @@ +Revert converting ``vars``, ``dir``, ``next``, ``getattr``, and ``iter`` to +argument clinic. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 9baf233614879..d06efcf6ba368 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -842,33 +842,31 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, return result; } -/*[clinic input] -dir as builtin_dir - - arg: object = NULL - / - -Show attributes of an object. - -If called without an argument, return the names in the current scope. -Else, return an alphabetized list of names comprising (some of) the attributes -of the given object, and of attributes reachable from it. -If the object supplies a method named __dir__, it will be used; otherwise -the default dir() logic is used and returns: - for a module object: the module's attributes. - for a class object: its attributes, and recursively the attributes - of its bases. - for any other object: its attributes, its class's attributes, and - recursively the attributes of its class's base classes. -[clinic start generated code]*/ - +/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ static PyObject * -builtin_dir_impl(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=24f2c7a52c1e3b08 input=ed6d6ccb13d52251]*/ +builtin_dir(PyObject *self, PyObject *args) { + PyObject *arg = NULL; + + if (!PyArg_UnpackTuple(args, "dir", 0, 1, &arg)) + return NULL; return PyObject_Dir(arg); } +PyDoc_STRVAR(dir_doc, +"dir([object]) -> list of strings\n" +"\n" +"If called without an argument, return the names in the current scope.\n" +"Else, return an alphabetized list of names comprising (some of) the attributes\n" +"of the given object, and of attributes reachable from it.\n" +"If the object supplies a method named __dir__, it will be used; otherwise\n" +"the default dir() logic is used and returns:\n" +" for a module object: the module's attributes.\n" +" for a class object: its attributes, and recursively the attributes\n" +" of its bases.\n" +" for any other object: its attributes, its class's attributes, and\n" +" recursively the attributes of its class's base classes."); + /*[clinic input] divmod as builtin_divmod @@ -1138,39 +1136,36 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, } -/*[clinic input] -getattr as builtin_getattr - - object: object - name: object - default: object = NULL - / - -Get a named attribute from an object. - -getattr(x, 'y') is equivalent to x.y -When a default argument is given, it is returned when the attribute doesn't -exist; without it, an exception is raised in that case. -[clinic start generated code]*/ - +/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ static PyObject * -builtin_getattr_impl(PyObject *module, PyObject *object, PyObject *name, - PyObject *default_value) -/*[clinic end generated code: output=74ad0e225e3f701c input=d7562cd4c3556171]*/ +builtin_getattr(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { - PyObject *result; + PyObject *v, *name, *result; + + if (!_PyArg_CheckPositional("getattr", nargs, 2, 3)) + return NULL; - if (default_value != NULL) { - if (PyObject_GetOptionalAttr(object, name, &result) == 0) { - return Py_NewRef(default_value); + v = args[0]; + name = args[1]; + if (nargs > 2) { + if (PyObject_GetOptionalAttr(v, name, &result) == 0) { + PyObject *dflt = args[2]; + return Py_NewRef(dflt); } } else { - result = PyObject_GetAttr(object, name); + result = PyObject_GetAttr(v, name); } return result; } +PyDoc_STRVAR(getattr_doc, +"getattr(object, name[, default]) -> value\n\ +\n\ +Get a named attribute from an object; getattr(x, 'y') is equivalent to x.y.\n\ +When a default argument is given, it is returned when the attribute doesn't\n\ +exist; without it, an exception is raised in that case."); + /*[clinic input] globals as builtin_globals @@ -1482,43 +1477,34 @@ PyTypeObject PyMap_Type = { }; -/*[clinic input] -next as builtin_next - - iterator: object - default: object = NULL - / - -Return the next item from the iterator. - -If default is given and the iterator is exhausted, -it is returned instead of raising StopIteration. -[clinic start generated code]*/ - +/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ static PyObject * -builtin_next_impl(PyObject *module, PyObject *iterator, - PyObject *default_value) -/*[clinic end generated code: output=a38a94eeb447fef9 input=180f9984f182020f]*/ +builtin_next(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { - PyObject *res; + PyObject *it, *res; + + if (!_PyArg_CheckPositional("next", nargs, 1, 2)) + return NULL; - if (!PyIter_Check(iterator)) { + it = args[0]; + if (!PyIter_Check(it)) { PyErr_Format(PyExc_TypeError, "'%.200s' object is not an iterator", - Py_TYPE(iterator)->tp_name); + Py_TYPE(it)->tp_name); return NULL; } - res = (*Py_TYPE(iterator)->tp_iternext)(iterator); + res = (*Py_TYPE(it)->tp_iternext)(it); if (res != NULL) { return res; - } else if (default_value != NULL) { + } else if (nargs > 1) { + PyObject *def = args[1]; if (PyErr_Occurred()) { if(!PyErr_ExceptionMatches(PyExc_StopIteration)) return NULL; PyErr_Clear(); } - return Py_NewRef(default_value); + return Py_NewRef(def); } else if (PyErr_Occurred()) { return NULL; } else { @@ -1527,6 +1513,12 @@ builtin_next_impl(PyObject *module, PyObject *iterator, } } +PyDoc_STRVAR(next_doc, +"next(iterator[, default])\n\ +\n\ +Return the next item from the iterator. If default is given and the iterator\n\ +is exhausted, it is returned instead of raising StopIteration."); + /*[clinic input] setattr as builtin_setattr @@ -1620,33 +1612,34 @@ builtin_hex(PyObject *module, PyObject *number) } -/*[clinic input] -iter as builtin_iter - - object: object - sentinel: object = NULL - / - -Get an iterator from an object. - -In the first form, the argument must supply its own iterator, or be a sequence. -In the second form, the callable is called until it returns the sentinel. -[clinic start generated code]*/ - +/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ static PyObject * -builtin_iter_impl(PyObject *module, PyObject *object, PyObject *sentinel) -/*[clinic end generated code: output=12cf64203c195a94 input=a5d64d9d81880ba6]*/ +builtin_iter(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { - if (sentinel == NULL) - return PyObject_GetIter(object); - if (!PyCallable_Check(object)) { + PyObject *v; + + if (!_PyArg_CheckPositional("iter", nargs, 1, 2)) + return NULL; + v = args[0]; + if (nargs == 1) + return PyObject_GetIter(v); + if (!PyCallable_Check(v)) { PyErr_SetString(PyExc_TypeError, - "iter(object, sentinel): object must be callable"); + "iter(v, w): v must be callable"); return NULL; } - return PyCallIter_New(object, sentinel); + PyObject *sentinel = args[1]; + return PyCallIter_New(v, sentinel); } +PyDoc_STRVAR(iter_doc, +"iter(iterable) -> iterator\n\ +iter(callable, sentinel) -> iterator\n\ +\n\ +Get an iterator from an object. In the first form, the argument must\n\ +supply its own iterator, or be a sequence.\n\ +In the second form, the callable is called until it returns the sentinel."); + /*[clinic input] aiter as builtin_aiter @@ -2444,29 +2437,20 @@ builtin_sorted(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject } -/*[clinic input] -vars as builtin_vars - - object: object = NULL - / - -Show vars. - -Without arguments, equivalent to locals(). -With an argument, equivalent to object.__dict__. -[clinic start generated code]*/ - +/* AC: cannot convert yet, as needs PEP 457 group support in inspect */ static PyObject * -builtin_vars_impl(PyObject *module, PyObject *object) -/*[clinic end generated code: output=840a7f64007a3e0a input=80cbdef9182c4ba3]*/ +builtin_vars(PyObject *self, PyObject *args) { + PyObject *v = NULL; PyObject *d; - if (object == NULL) { + if (!PyArg_UnpackTuple(args, "vars", 0, 1, &v)) + return NULL; + if (v == NULL) { d = _PyEval_GetFrameLocals(); } else { - if (PyObject_GetOptionalAttr(object, &_Py_ID(__dict__), &d) == 0) { + if (PyObject_GetOptionalAttr(v, &_Py_ID(__dict__), &d) == 0) { PyErr_SetString(PyExc_TypeError, "vars() argument must have __dict__ attribute"); } @@ -2474,6 +2458,12 @@ builtin_vars_impl(PyObject *module, PyObject *object) return d; } +PyDoc_STRVAR(vars_doc, +"vars([object]) -> dictionary\n\ +\n\ +Without arguments, equivalent to locals().\n\ +With an argument, equivalent to object.__dict__."); + /*[clinic input] sum as builtin_sum @@ -3022,12 +3012,12 @@ static PyMethodDef builtin_methods[] = { BUILTIN_CHR_METHODDEF BUILTIN_COMPILE_METHODDEF BUILTIN_DELATTR_METHODDEF - BUILTIN_DIR_METHODDEF + {"dir", builtin_dir, METH_VARARGS, dir_doc}, BUILTIN_DIVMOD_METHODDEF BUILTIN_EVAL_METHODDEF BUILTIN_EXEC_METHODDEF BUILTIN_FORMAT_METHODDEF - BUILTIN_GETATTR_METHODDEF + {"getattr", _PyCFunction_CAST(builtin_getattr), METH_FASTCALL, getattr_doc}, BUILTIN_GLOBALS_METHODDEF BUILTIN_HASATTR_METHODDEF BUILTIN_HASH_METHODDEF @@ -3036,13 +3026,13 @@ static PyMethodDef builtin_methods[] = { BUILTIN_INPUT_METHODDEF BUILTIN_ISINSTANCE_METHODDEF BUILTIN_ISSUBCLASS_METHODDEF - BUILTIN_ITER_METHODDEF + {"iter", _PyCFunction_CAST(builtin_iter), METH_FASTCALL, iter_doc}, BUILTIN_AITER_METHODDEF BUILTIN_LEN_METHODDEF BUILTIN_LOCALS_METHODDEF {"max", _PyCFunction_CAST(builtin_max), METH_VARARGS | METH_KEYWORDS, max_doc}, {"min", _PyCFunction_CAST(builtin_min), METH_VARARGS | METH_KEYWORDS, min_doc}, - BUILTIN_NEXT_METHODDEF + {"next", _PyCFunction_CAST(builtin_next), METH_FASTCALL, next_doc}, BUILTIN_ANEXT_METHODDEF BUILTIN_OCT_METHODDEF BUILTIN_ORD_METHODDEF @@ -3053,7 +3043,7 @@ static PyMethodDef builtin_methods[] = { BUILTIN_SETATTR_METHODDEF BUILTIN_SORTED_METHODDEF BUILTIN_SUM_METHODDEF - BUILTIN_VARS_METHODDEF + {"vars", builtin_vars, METH_VARARGS, vars_doc}, {NULL, NULL}, }; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index b0d05dde956ef..7540de6828043 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -383,49 +383,6 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -PyDoc_STRVAR(builtin_dir__doc__, -"dir($module, arg=, /)\n" -"--\n" -"\n" -"Show attributes of an object.\n" -"\n" -"If called without an argument, return the names in the current scope.\n" -"Else, return an alphabetized list of names comprising (some of) the attributes\n" -"of the given object, and of attributes reachable from it.\n" -"If the object supplies a method named __dir__, it will be used; otherwise\n" -"the default dir() logic is used and returns:\n" -" for a module object: the module\'s attributes.\n" -" for a class object: its attributes, and recursively the attributes\n" -" of its bases.\n" -" for any other object: its attributes, its class\'s attributes, and\n" -" recursively the attributes of its class\'s base classes."); - -#define BUILTIN_DIR_METHODDEF \ - {"dir", _PyCFunction_CAST(builtin_dir), METH_FASTCALL, builtin_dir__doc__}, - -static PyObject * -builtin_dir_impl(PyObject *module, PyObject *arg); - -static PyObject * -builtin_dir(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *arg = NULL; - - if (!_PyArg_CheckPositional("dir", nargs, 0, 1)) { - goto exit; - } - if (nargs < 1) { - goto skip_optional; - } - arg = args[0]; -skip_optional: - return_value = builtin_dir_impl(module, arg); - -exit: - return return_value; -} - PyDoc_STRVAR(builtin_divmod__doc__, "divmod($module, x, y, /)\n" "--\n" @@ -586,47 +543,6 @@ builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject return return_value; } -PyDoc_STRVAR(builtin_getattr__doc__, -"getattr($module, object, name, default=, /)\n" -"--\n" -"\n" -"Get a named attribute from an object.\n" -"\n" -"getattr(x, \'y\') is equivalent to x.y\n" -"When a default argument is given, it is returned when the attribute doesn\'t\n" -"exist; without it, an exception is raised in that case."); - -#define BUILTIN_GETATTR_METHODDEF \ - {"getattr", _PyCFunction_CAST(builtin_getattr), METH_FASTCALL, builtin_getattr__doc__}, - -static PyObject * -builtin_getattr_impl(PyObject *module, PyObject *object, PyObject *name, - PyObject *default_value); - -static PyObject * -builtin_getattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *object; - PyObject *name; - PyObject *default_value = NULL; - - if (!_PyArg_CheckPositional("getattr", nargs, 2, 3)) { - goto exit; - } - object = args[0]; - name = args[1]; - if (nargs < 3) { - goto skip_optional; - } - default_value = args[2]; -skip_optional: - return_value = builtin_getattr_impl(module, object, name, default_value); - -exit: - return return_value; -} - PyDoc_STRVAR(builtin_globals__doc__, "globals($module, /)\n" "--\n" @@ -692,44 +608,6 @@ PyDoc_STRVAR(builtin_id__doc__, #define BUILTIN_ID_METHODDEF \ {"id", (PyCFunction)builtin_id, METH_O, builtin_id__doc__}, -PyDoc_STRVAR(builtin_next__doc__, -"next($module, iterator, default=, /)\n" -"--\n" -"\n" -"Return the next item from the iterator.\n" -"\n" -"If default is given and the iterator is exhausted,\n" -"it is returned instead of raising StopIteration."); - -#define BUILTIN_NEXT_METHODDEF \ - {"next", _PyCFunction_CAST(builtin_next), METH_FASTCALL, builtin_next__doc__}, - -static PyObject * -builtin_next_impl(PyObject *module, PyObject *iterator, - PyObject *default_value); - -static PyObject * -builtin_next(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *iterator; - PyObject *default_value = NULL; - - if (!_PyArg_CheckPositional("next", nargs, 1, 2)) { - goto exit; - } - iterator = args[0]; - if (nargs < 2) { - goto skip_optional; - } - default_value = args[1]; -skip_optional: - return_value = builtin_next_impl(module, iterator, default_value); - -exit: - return return_value; -} - PyDoc_STRVAR(builtin_setattr__doc__, "setattr($module, obj, name, value, /)\n" "--\n" @@ -821,43 +699,6 @@ PyDoc_STRVAR(builtin_hex__doc__, #define BUILTIN_HEX_METHODDEF \ {"hex", (PyCFunction)builtin_hex, METH_O, builtin_hex__doc__}, -PyDoc_STRVAR(builtin_iter__doc__, -"iter($module, object, sentinel=, /)\n" -"--\n" -"\n" -"Get an iterator from an object.\n" -"\n" -"In the first form, the argument must supply its own iterator, or be a sequence.\n" -"In the second form, the callable is called until it returns the sentinel."); - -#define BUILTIN_ITER_METHODDEF \ - {"iter", _PyCFunction_CAST(builtin_iter), METH_FASTCALL, builtin_iter__doc__}, - -static PyObject * -builtin_iter_impl(PyObject *module, PyObject *object, PyObject *sentinel); - -static PyObject * -builtin_iter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *object; - PyObject *sentinel = NULL; - - if (!_PyArg_CheckPositional("iter", nargs, 1, 2)) { - goto exit; - } - object = args[0]; - if (nargs < 2) { - goto skip_optional; - } - sentinel = args[1]; -skip_optional: - return_value = builtin_iter_impl(module, object, sentinel); - -exit: - return return_value; -} - PyDoc_STRVAR(builtin_aiter__doc__, "aiter($module, async_iterable, /)\n" "--\n" @@ -1236,41 +1077,6 @@ builtin_round(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } -PyDoc_STRVAR(builtin_vars__doc__, -"vars($module, object=, /)\n" -"--\n" -"\n" -"Show vars.\n" -"\n" -"Without arguments, equivalent to locals().\n" -"With an argument, equivalent to object.__dict__."); - -#define BUILTIN_VARS_METHODDEF \ - {"vars", _PyCFunction_CAST(builtin_vars), METH_FASTCALL, builtin_vars__doc__}, - -static PyObject * -builtin_vars_impl(PyObject *module, PyObject *object); - -static PyObject * -builtin_vars(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *object = NULL; - - if (!_PyArg_CheckPositional("vars", nargs, 0, 1)) { - goto exit; - } - if (nargs < 1) { - goto skip_optional; - } - object = args[0]; -skip_optional: - return_value = builtin_vars_impl(module, object); - -exit: - return return_value; -} - PyDoc_STRVAR(builtin_sum__doc__, "sum($module, iterable, /, start=0)\n" "--\n" @@ -1406,4 +1212,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=ef2f16ece134d62d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=daeee81b018824f4 input=a9049054013a1b77]*/ From webhook-mailer at python.org Mon Aug 21 00:51:34 2023 From: webhook-mailer at python.org (corona10) Date: Mon, 21 Aug 2023 04:51:34 -0000 Subject: [Python-checkins] gh-107526: Fix test_module_level_callable_unrepresentable_default (gh-108187) Message-ID: https://github.com/python/cpython/commit/04f7875c4489bbe817e76f9f33773c8c21ba4ec2 commit: 04f7875c4489bbe817e76f9f33773c8c21ba4ec2 branch: main author: Dong-hee Na committer: corona10 date: 2023-08-21T04:51:31Z summary: gh-107526: Fix test_module_level_callable_unrepresentable_default (gh-108187) files: M Lib/test/test_pydoc.py diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index fe4e37d4858c8..9b5c11bf853fd 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -1232,7 +1232,7 @@ def test_bound_builtin_classmethod_o(self): def test_module_level_callable_unrepresentable_default(self): self.assertEqual(self._get_summary_line(getattr), - "getattr(object, name, default=, /)") + "getattr(...)") def test_builtin_staticmethod_unrepresentable_default(self): self.assertEqual(self._get_summary_line(str.maketrans), From webhook-mailer at python.org Mon Aug 21 01:50:12 2023 From: webhook-mailer at python.org (corona10) Date: Mon, 21 Aug 2023 05:50:12 -0000 Subject: [Python-checkins] gh-107265: Fix code_richcompare for ENTER_EXECUTOR case (gh-108165) Message-ID: https://github.com/python/cpython/commit/4fdf3fda0f970805b2404e71f466f430ab8cd9fd commit: 4fdf3fda0f970805b2404e71f466f430ab8cd9fd branch: main author: Dong-hee Na committer: corona10 date: 2023-08-21T05:50:09Z summary: gh-107265: Fix code_richcompare for ENTER_EXECUTOR case (gh-108165) files: M Lib/test/test_capi/test_misc.py M Objects/codeobject.c diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 18a0476122dab..ea0504333bab0 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2341,6 +2341,17 @@ def long_loop(): long_loop() self.assertEqual(opt.get_count(), 10) + def test_code_richcompare(self): + def testfunc(x): + i = 0 + while i < x: + i += 1 + + opt = _testinternalcapi.get_counter_optimizer() + with temporary_optimizer(opt): + testfunc(1000) + self.assertEqual(testfunc.__code__, testfunc.__code__.replace()) + def get_first_executor(func): code = func.__code__ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 4d6efe938f45d..c34905c319606 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1781,8 +1781,25 @@ code_richcompare(PyObject *self, PyObject *other, int op) for (int i = 0; i < Py_SIZE(co); i++) { _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; _Py_CODEUNIT cp_instr = _PyCode_CODE(cp)[i]; + + if (co_instr.op.code == ENTER_EXECUTOR) { + const int exec_index = co_instr.op.arg; + _PyExecutorObject *exec = co->co_executors->executors[exec_index]; + co_instr.op.code = exec->vm_data.opcode; + co_instr.op.arg = exec->vm_data.oparg; + } + assert(co_instr.op.code != ENTER_EXECUTOR); co_instr.op.code = _PyOpcode_Deopt[co_instr.op.code]; + + if (cp_instr.op.code == ENTER_EXECUTOR) { + const int exec_index = cp_instr.op.arg; + _PyExecutorObject *exec = cp->co_executors->executors[exec_index]; + cp_instr.op.code = exec->vm_data.opcode; + cp_instr.op.arg = exec->vm_data.oparg; + } + assert(cp_instr.op.code != ENTER_EXECUTOR); cp_instr.op.code = _PyOpcode_Deopt[cp_instr.op.code]; + eq = co_instr.cache == cp_instr.cache; if (!eq) { goto unequal; From webhook-mailer at python.org Mon Aug 21 03:37:55 2023 From: webhook-mailer at python.org (hugovk) Date: Mon, 21 Aug 2023 07:37:55 -0000 Subject: [Python-checkins] Resolve reference warnings in faq/gui.rst (#108147) Message-ID: https://github.com/python/cpython/commit/8f3d09bf5d16b508fece5420a22abe6f0c1f00b7 commit: 8f3d09bf5d16b508fece5420a22abe6f0c1f00b7 branch: main author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: hugovk date: 2023-08-21T07:37:52Z summary: Resolve reference warnings in faq/gui.rst (#108147) Co-authored-by: Hugo van Kemenade files: M Doc/faq/gui.rst M Doc/tools/.nitignore diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index 0a372342862d2..cfa60feceb31b 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -43,7 +43,7 @@ applications, the applications will not be truly stand-alone, as the application will still need the Tcl and Tk libraries. One solution is to ship the application with the Tcl and Tk libraries, and point -to them at run-time using the :envvar:`TCL_LIBRARY` and :envvar:`TK_LIBRARY` +to them at run-time using the :envvar:`!TCL_LIBRARY` and :envvar:`!TK_LIBRARY` environment variables. Various third-party freeze libraries such as py2exe and cx_Freeze have @@ -55,7 +55,7 @@ Can I have Tk events handled while waiting for I/O? On platforms other than Windows, yes, and you don't even need threads! But you'll have to restructure your I/O -code a bit. Tk has the equivalent of Xt's :c:func:`XtAddInput()` call, which allows you +code a bit. Tk has the equivalent of Xt's :c:func:`!XtAddInput` call, which allows you to register a callback function which will be called from the Tk mainloop when I/O is possible on a file descriptor. See :ref:`tkinter-file-handlers`. @@ -63,8 +63,9 @@ I/O is possible on a file descriptor. See :ref:`tkinter-file-handlers`. I can't get key bindings to work in Tkinter: why? ------------------------------------------------- -An often-heard complaint is that event handlers bound to events with the -:meth:`bind` method don't get handled even when the appropriate key is pressed. +An often-heard complaint is that event handlers :ref:`bound ` +to events with the :meth:`!bind` method +don't get handled even when the appropriate key is pressed. The most common cause is that the widget to which the binding applies doesn't have "keyboard focus". Check out the Tk documentation for the focus command. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 9475e883ee5b5..1b43a2e4270c9 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -24,7 +24,6 @@ Doc/c-api/typeobj.rst Doc/c-api/unicode.rst Doc/extending/extending.rst Doc/extending/newtypes.rst -Doc/faq/gui.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst From webhook-mailer at python.org Mon Aug 21 03:55:12 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 21 Aug 2023 07:55:12 -0000 Subject: [Python-checkins] gh-107895: Fix test_asyncio.test_runners when run it in CPython's "development mode" (GH-108168) Message-ID: https://github.com/python/cpython/commit/014a5b71e7538926ae1c03c8c5ea13c96e741be3 commit: 014a5b71e7538926ae1c03c8c5ea13c96e741be3 branch: main author: Joon Hwan ??? committer: serhiy-storchaka date: 2023-08-21T07:55:09Z summary: gh-107895: Fix test_asyncio.test_runners when run it in CPython's "development mode" (GH-108168) files: M Lib/test/test_asyncio/test_runners.py diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index b1eb6f492886b..1eb5641914f2a 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -101,11 +101,14 @@ async def main(expected): loop = asyncio.get_event_loop() self.assertIs(loop.get_debug(), expected) - asyncio.run(main(False)) + asyncio.run(main(False), debug=False) asyncio.run(main(True), debug=True) with mock.patch('asyncio.coroutines._is_debug_mode', lambda: True): asyncio.run(main(True)) asyncio.run(main(False), debug=False) + with mock.patch('asyncio.coroutines._is_debug_mode', lambda: False): + asyncio.run(main(True), debug=True) + asyncio.run(main(False)) def test_asyncio_run_from_running_loop(self): async def main(): From webhook-mailer at python.org Mon Aug 21 04:36:41 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 21 Aug 2023 08:36:41 -0000 Subject: [Python-checkins] [3.11] gh-107895: Fix test_asyncio.test_runners when run it in CPython's "development mode" (GH-108168) (GH-108197) Message-ID: https://github.com/python/cpython/commit/d4c66bd7328f6aadb584d54f81ae5281a66d08c4 commit: d4c66bd7328f6aadb584d54f81ae5281a66d08c4 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: serhiy-storchaka date: 2023-08-21T08:36:37Z summary: [3.11] gh-107895: Fix test_asyncio.test_runners when run it in CPython's "development mode" (GH-108168) (GH-108197) (cherry picked from commit 014a5b71e7538926ae1c03c8c5ea13c96e741be3) Co-authored-by: Joon Hwan ??? files: M Lib/test/test_asyncio/test_runners.py diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 4e3acb97c85d3..8a4aba6d470ba 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -102,11 +102,14 @@ async def main(expected): loop = asyncio.get_event_loop() self.assertIs(loop.get_debug(), expected) - asyncio.run(main(False)) + asyncio.run(main(False), debug=False) asyncio.run(main(True), debug=True) with mock.patch('asyncio.coroutines._is_debug_mode', lambda: True): asyncio.run(main(True)) asyncio.run(main(False), debug=False) + with mock.patch('asyncio.coroutines._is_debug_mode', lambda: False): + asyncio.run(main(True), debug=True) + asyncio.run(main(False)) def test_asyncio_run_from_running_loop(self): async def main(): From webhook-mailer at python.org Mon Aug 21 05:49:12 2023 From: webhook-mailer at python.org (ambv) Date: Mon, 21 Aug 2023 09:49:12 -0000 Subject: [Python-checkins] gh-105736: Sync pure python version of OrderedDict with the C version (#108098) Message-ID: https://github.com/python/cpython/commit/20cc90c0df3e368fe7cb63d958f0b17a78fa9d0a commit: 20cc90c0df3e368fe7cb63d958f0b17a78fa9d0a branch: main author: Raymond Hettinger committer: ambv date: 2023-08-21T11:49:08+02:00 summary: gh-105736: Sync pure python version of OrderedDict with the C version (#108098) files: A Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst M Lib/collections/__init__.py M Lib/test/test_ordered_dict.py diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 03ca2d7e18f6f..8652dc8a4ec45 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -95,17 +95,19 @@ class OrderedDict(dict): # Individual links are kept alive by the hard reference in self.__map. # Those hard references disappear when a key is deleted from an OrderedDict. + def __new__(cls, /, *args, **kwds): + "Create the ordered dict object and set up the underlying structures." + self = dict.__new__(cls) + self.__hardroot = _Link() + self.__root = root = _proxy(self.__hardroot) + root.prev = root.next = root + self.__map = {} + return self + def __init__(self, other=(), /, **kwds): '''Initialize an ordered dictionary. The signature is the same as regular dictionaries. Keyword argument order is preserved. ''' - try: - self.__root - except AttributeError: - self.__hardroot = _Link() - self.__root = root = _proxy(self.__hardroot) - root.prev = root.next = root - self.__map = {} self.__update(other, **kwds) def __setitem__(self, key, value, diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index decbcc2419c9f..4571b23dfe7c1 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -122,6 +122,17 @@ def items(self): self.OrderedDict(Spam()) self.assertEqual(calls, ['keys']) + def test_overridden_init(self): + # Sync-up pure Python OD class with C class where + # a consistent internal state is created in __new__ + # rather than __init__. + OrderedDict = self.OrderedDict + class ODNI(OrderedDict): + def __init__(*args, **kwargs): + pass + od = ODNI() + od['a'] = 1 # This used to fail because __init__ was bypassed + def test_fromkeys(self): OrderedDict = self.OrderedDict od = OrderedDict.fromkeys('abc') diff --git a/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst b/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst new file mode 100644 index 0000000000000..f051317d141ff --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst @@ -0,0 +1,3 @@ +Harmonized the pure Python version of OrderedDict with the C version. Now, +both versions set up their internal state in `__new__`. Formerly, the pure +Python version did the set up in `__init__`. From webhook-mailer at python.org Mon Aug 21 06:37:13 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 21 Aug 2023 10:37:13 -0000 Subject: [Python-checkins] [3.11] gh-105736: Sync pure python version of OrderedDict with the C version (GH-108098) (GH-108201) Message-ID: https://github.com/python/cpython/commit/a1d2e2c2ac93d125576592ce227ec999f5a3a8c5 commit: a1d2e2c2ac93d125576592ce227ec999f5a3a8c5 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: serhiy-storchaka date: 2023-08-21T10:37:09Z summary: [3.11] gh-105736: Sync pure python version of OrderedDict with the C version (GH-108098) (GH-108201) (cherry picked from commit 20cc90c0df3e368fe7cb63d958f0b17a78fa9d0a) Co-authored-by: Raymond Hettinger files: A Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst M Lib/collections/__init__.py M Lib/test/test_ordered_dict.py diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 7af8dcd526df8..69398ac116468 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -90,17 +90,19 @@ class OrderedDict(dict): # Individual links are kept alive by the hard reference in self.__map. # Those hard references disappear when a key is deleted from an OrderedDict. + def __new__(cls, /, *args, **kwds): + "Create the ordered dict object and set up the underlying structures." + self = dict.__new__(cls) + self.__hardroot = _Link() + self.__root = root = _proxy(self.__hardroot) + root.prev = root.next = root + self.__map = {} + return self + def __init__(self, other=(), /, **kwds): '''Initialize an ordered dictionary. The signature is the same as regular dictionaries. Keyword argument order is preserved. ''' - try: - self.__root - except AttributeError: - self.__hardroot = _Link() - self.__root = root = _proxy(self.__hardroot) - root.prev = root.next = root - self.__map = {} self.__update(other, **kwds) def __setitem__(self, key, value, diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 37447fd249b8c..dfd329aa0a9fa 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -122,6 +122,17 @@ def items(self): self.OrderedDict(Spam()) self.assertEqual(calls, ['keys']) + def test_overridden_init(self): + # Sync-up pure Python OD class with C class where + # a consistent internal state is created in __new__ + # rather than __init__. + OrderedDict = self.OrderedDict + class ODNI(OrderedDict): + def __init__(*args, **kwargs): + pass + od = ODNI() + od['a'] = 1 # This used to fail because __init__ was bypassed + def test_fromkeys(self): OrderedDict = self.OrderedDict od = OrderedDict.fromkeys('abc') diff --git a/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst b/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst new file mode 100644 index 0000000000000..7fed54a5184c0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst @@ -0,0 +1,3 @@ +Harmonized the pure Python version of OrderedDict with the C version. Now, +both versions set up their internal state in ``__new__``. Formerly, the pure +Python version did the set up in ``__init__``. From webhook-mailer at python.org Mon Aug 21 06:41:05 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 21 Aug 2023 10:41:05 -0000 Subject: [Python-checkins] Improve references in the tutorial (GH-108069) Message-ID: https://github.com/python/cpython/commit/622ddc41674c2566062af82f7b079aa01d2aae8c commit: 622ddc41674c2566062af82f7b079aa01d2aae8c branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-21T13:41:01+03:00 summary: Improve references in the tutorial (GH-108069) * Use full qualified names for references (even if they do not work now, they will work in future). * Silence references to examples. files: M Doc/tools/.nitignore M Doc/tutorial/classes.rst M Doc/tutorial/controlflow.rst M Doc/tutorial/datastructures.rst M Doc/tutorial/inputoutput.rst M Doc/tutorial/modules.rst diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 1b43a2e4270c9..4231fb8575891 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -164,9 +164,7 @@ Doc/tutorial/appendix.rst Doc/tutorial/classes.rst Doc/tutorial/controlflow.rst Doc/tutorial/datastructures.rst -Doc/tutorial/inputoutput.rst Doc/tutorial/introduction.rst -Doc/tutorial/modules.rst Doc/using/cmdline.rst Doc/using/configure.rst Doc/using/windows.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 06445e000c1ef..91a3b73d2b55a 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -91,7 +91,7 @@ Attributes may be read-only or writable. In the latter case, assignment to attributes is possible. Module attributes are writable: you can write ``modname.the_answer = 42``. Writable attributes may also be deleted with the :keyword:`del` statement. For example, ``del modname.the_answer`` will remove -the attribute :attr:`the_answer` from the object named by ``modname``. +the attribute :attr:`!the_answer` from the object named by ``modname``. Namespaces are created at different moments and have different lifetimes. The namespace containing the built-in names is created when the Python interpreter @@ -249,7 +249,7 @@ created. This is basically a wrapper around the contents of the namespace created by the class definition; we'll learn more about class objects in the next section. The original local scope (the one in effect just before the class definition was entered) is reinstated, and the class object is bound here to the -class name given in the class definition header (:class:`ClassName` in the +class name given in the class definition header (:class:`!ClassName` in the example). @@ -291,20 +291,20 @@ variable ``x``. The instantiation operation ("calling" a class object) creates an empty object. Many classes like to create objects with instances customized to a specific initial state. Therefore a class may define a special method named -:meth:`__init__`, like this:: +:meth:`~object.__init__`, like this:: def __init__(self): self.data = [] -When a class defines an :meth:`__init__` method, class instantiation -automatically invokes :meth:`__init__` for the newly created class instance. So +When a class defines an :meth:`~object.__init__` method, class instantiation +automatically invokes :meth:`!__init__` for the newly created class instance. So in this example, a new, initialized instance can be obtained by:: x = MyClass() -Of course, the :meth:`__init__` method may have arguments for greater +Of course, the :meth:`~object.__init__` method may have arguments for greater flexibility. In that case, arguments given to the class instantiation operator -are passed on to :meth:`__init__`. For example, :: +are passed on to :meth:`!__init__`. For example, :: >>> class Complex: ... def __init__(self, realpart, imagpart): @@ -328,7 +328,7 @@ attribute names: data attributes and methods. *data attributes* correspond to "instance variables" in Smalltalk, and to "data members" in C++. Data attributes need not be declared; like local variables, they spring into existence when they are first assigned to. For example, if -``x`` is the instance of :class:`MyClass` created above, the following piece of +``x`` is the instance of :class:`!MyClass` created above, the following piece of code will print the value ``16``, without leaving a trace:: x.counter = 1 @@ -363,7 +363,7 @@ Usually, a method is called right after it is bound:: x.f() -In the :class:`MyClass` example, this will return the string ``'hello world'``. +In the :class:`!MyClass` example, this will return the string ``'hello world'``. However, it is not necessary to call a method right away: ``x.f`` is a method object, and can be stored away and called at a later time. For example:: @@ -375,7 +375,7 @@ will continue to print ``hello world`` until the end of time. What exactly happens when a method is called? You may have noticed that ``x.f()`` was called without an argument above, even though the function -definition for :meth:`f` specified an argument. What happened to the argument? +definition for :meth:`!f` specified an argument. What happened to the argument? Surely Python raises an exception when a function that requires an argument is called without any --- even if the argument isn't actually used... @@ -532,9 +532,9 @@ variable in the class is also ok. For example:: h = g -Now ``f``, ``g`` and ``h`` are all attributes of class :class:`C` that refer to +Now ``f``, ``g`` and ``h`` are all attributes of class :class:`!C` that refer to function objects, and consequently they are all methods of instances of -:class:`C` --- ``h`` being exactly equivalent to ``g``. Note that this practice +:class:`!C` --- ``h`` being exactly equivalent to ``g``. Note that this practice usually only serves to confuse the reader of a program. Methods may call other methods by using method attributes of the ``self`` @@ -581,7 +581,7 @@ this:: . -The name :class:`BaseClassName` must be defined in a +The name :class:`!BaseClassName` must be defined in a namespace accessible from the scope containing the derived class definition. In place of a base class name, other arbitrary expressions are also allowed. This can be useful, for example, when the base @@ -645,9 +645,9 @@ multiple base classes looks like this:: For most purposes, in the simplest cases, you can think of the search for attributes inherited from a parent class as depth-first, left-to-right, not searching twice in the same class where there is an overlap in the hierarchy. -Thus, if an attribute is not found in :class:`DerivedClassName`, it is searched -for in :class:`Base1`, then (recursively) in the base classes of :class:`Base1`, -and if it was not found there, it was searched for in :class:`Base2`, and so on. +Thus, if an attribute is not found in :class:`!DerivedClassName`, it is searched +for in :class:`!Base1`, then (recursively) in the base classes of :class:`!Base1`, +and if it was not found there, it was searched for in :class:`!Base2`, and so on. In fact, it is slightly more complex than that; the method resolution order changes dynamically to support cooperative calls to :func:`super`. This @@ -760,7 +760,8 @@ is to use :mod:`dataclasses` for this purpose:: A piece of Python code that expects a particular abstract data type can often be passed a class that emulates the methods of that data type instead. For instance, if you have a function that formats some data from a file object, you -can define a class with methods :meth:`read` and :meth:`!readline` that get the +can define a class with methods :meth:`~io.TextIOBase.read` and +:meth:`~io.TextIOBase.readline` that get the data from a string buffer instead, and pass it as an argument. .. (Unfortunately, this technique has its limitations: a class can't define @@ -769,7 +770,7 @@ data from a string buffer instead, and pass it as an argument. not cause the interpreter to read further input from it.) Instance method objects have attributes, too: ``m.__self__`` is the instance -object with the method :meth:`m`, and ``m.__func__`` is the function object +object with the method :meth:`!m`, and ``m.__func__`` is the function object corresponding to the method. @@ -818,9 +819,9 @@ using the :func:`next` built-in function; this example shows how it all works:: StopIteration Having seen the mechanics behind the iterator protocol, it is easy to add -iterator behavior to your classes. Define an :meth:`__iter__` method which +iterator behavior to your classes. Define an :meth:`~container.__iter__` method which returns an object with a :meth:`~iterator.__next__` method. If the class -defines :meth:`__next__`, then :meth:`__iter__` can just return ``self``:: +defines :meth:`!__next__`, then :meth:`!__iter__` can just return ``self``:: class Reverse: """Iterator for looping over a sequence backwards.""" @@ -879,7 +880,7 @@ easy to create:: Anything that can be done with generators can also be done with class-based iterators as described in the previous section. What makes generators so -compact is that the :meth:`__iter__` and :meth:`~generator.__next__` methods +compact is that the :meth:`~iterator.__iter__` and :meth:`~generator.__next__` methods are created automatically. Another key feature is that the local variables and execution state are diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 138d87f892e89..4bcc3768111cc 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -534,7 +534,7 @@ This example, as usual, demonstrates some new Python features: Different types define different methods. Methods of different types may have the same name without causing ambiguity. (It is possible to define your own object types and methods, using *classes*, see :ref:`tut-classes`) - The method :meth:`append` shown in the example is defined for list objects; it + The method :meth:`~list.append` shown in the example is defined for list objects; it adds a new element at the end of the list. In this example it is equivalent to ``result = result + [a]``, but more efficient. diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index c8e89d9b79bdd..87614d082a1d4 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -143,8 +143,8 @@ Using Lists as Stacks The list methods make it very easy to use a list as a stack, where the last element added is the first element retrieved ("last-in, first-out"). To add an -item to the top of the stack, use :meth:`append`. To retrieve an item from the -top of the stack, use :meth:`pop` without an explicit index. For example:: +item to the top of the stack, use :meth:`~list.append`. To retrieve an item from the +top of the stack, use :meth:`~list.pop` without an explicit index. For example:: >>> stack = [3, 4, 5] >>> stack.append(6) @@ -341,7 +341,7 @@ The :keyword:`!del` statement ============================= There is a way to remove an item from a list given its index instead of its -value: the :keyword:`del` statement. This differs from the :meth:`pop` method +value: the :keyword:`del` statement. This differs from the :meth:`~list.pop` method which returns a value. The :keyword:`!del` statement can also be used to remove slices from a list or clear the entire list (which we did earlier by assignment of an empty list to the slice). For example:: @@ -501,8 +501,8 @@ any immutable type; strings and numbers can always be keys. Tuples can be used as keys if they contain only strings, numbers, or tuples; if a tuple contains any mutable object either directly or indirectly, it cannot be used as a key. You can't use lists as keys, since lists can be modified in place using index -assignments, slice assignments, or methods like :meth:`append` and -:meth:`extend`. +assignments, slice assignments, or methods like :meth:`~list.append` and +:meth:`~list.extend`. It is best to think of a dictionary as a set of *key: value* pairs, with the requirement that the keys are unique (within one dictionary). A pair of @@ -567,7 +567,7 @@ Looping Techniques ================== When looping through dictionaries, the key and corresponding value can be -retrieved at the same time using the :meth:`items` method. :: +retrieved at the same time using the :meth:`~dict.items` method. :: >>> knights = {'gallahad': 'the pure', 'robin': 'the brave'} >>> for k, v in knights.items(): diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index f5cdd84cbadef..fe9ca9ccb9c7e 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -15,7 +15,7 @@ Fancier Output Formatting ========================= So far we've encountered two ways of writing values: *expression statements* and -the :func:`print` function. (A third way is using the :meth:`write` method +the :func:`print` function. (A third way is using the :meth:`~io.TextIOBase.write` method of file objects; the standard output file can be referenced as ``sys.stdout``. See the Library Reference for more information on this.) @@ -456,8 +456,8 @@ to the very file end with ``seek(0, 2)``) and the only valid *offset* values are those returned from the ``f.tell()``, or zero. Any other *offset* value produces undefined behaviour. -File objects have some additional methods, such as :meth:`~file.isatty` and -:meth:`~file.truncate` which are less frequently used; consult the Library +File objects have some additional methods, such as :meth:`~io.IOBase.isatty` and +:meth:`~io.IOBase.truncate` which are less frequently used; consult the Library Reference for a complete guide to file objects. @@ -469,7 +469,7 @@ Saving structured data with :mod:`json` .. index:: pair: module; json Strings can easily be written to and read from a file. Numbers take a bit more -effort, since the :meth:`read` method only returns strings, which will have to +effort, since the :meth:`~io.TextIOBase.read` method only returns strings, which will have to be passed to a function like :func:`int`, which takes a string like ``'123'`` and returns its numeric value 123. When you want to save more complex data types like nested lists and dictionaries, parsing and serializing by hand diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 734dd1cfe6871..bf9e8e0b7b806 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -183,7 +183,7 @@ The Module Search Path .. index:: triple: module; search; path -When a module named :mod:`spam` is imported, the interpreter first searches for +When a module named :mod:`!spam` is imported, the interpreter first searches for a built-in module with that name. These module names are listed in :data:`sys.builtin_module_names`. If not found, it then searches for a file named :file:`spam.py` in a list of directories given by the variable @@ -389,7 +389,7 @@ Packages ======== Packages are a way of structuring Python's module namespace by using "dotted -module names". For example, the module name :mod:`A.B` designates a submodule +module names". For example, the module name :mod:`!A.B` designates a submodule named ``B`` in a package named ``A``. Just like the use of modules saves the authors of different modules from having to worry about each other's global variable names, the use of dotted module names saves the authors of multi-module @@ -448,7 +448,7 @@ example:: import sound.effects.echo -This loads the submodule :mod:`sound.effects.echo`. It must be referenced with +This loads the submodule :mod:`!sound.effects.echo`. It must be referenced with its full name. :: sound.effects.echo.echofilter(input, output, delay=0.7, atten=4) @@ -457,7 +457,7 @@ An alternative way of importing the submodule is:: from sound.effects import echo -This also loads the submodule :mod:`echo`, and makes it available without its +This also loads the submodule :mod:`!echo`, and makes it available without its package prefix, so it can be used as follows:: echo.echofilter(input, output, delay=0.7, atten=4) @@ -466,8 +466,8 @@ Yet another variation is to import the desired function or variable directly:: from sound.effects.echo import echofilter -Again, this loads the submodule :mod:`echo`, but this makes its function -:func:`echofilter` directly available:: +Again, this loads the submodule :mod:`!echo`, but this makes its function +:func:`!echofilter` directly available:: echofilter(input, output, delay=0.7, atten=4) @@ -510,7 +510,7 @@ code:: __all__ = ["echo", "surround", "reverse"] This would mean that ``from sound.effects import *`` would import the three -named submodules of the :mod:`sound.effects` package. +named submodules of the :mod:`!sound.effects` package. Be aware that submodules might become shadowed by locally defined names. For example, if you added a ``reverse`` function to the @@ -529,8 +529,8 @@ would only import the two submodules ``echo`` and ``surround``, but *not* the return msg[::-1] # in the case of a 'from sound.effects import *' If ``__all__`` is not defined, the statement ``from sound.effects import *`` -does *not* import all submodules from the package :mod:`sound.effects` into the -current namespace; it only ensures that the package :mod:`sound.effects` has +does *not* import all submodules from the package :mod:`!sound.effects` into the +current namespace; it only ensures that the package :mod:`!sound.effects` has been imported (possibly running any initialization code in :file:`__init__.py`) and then imports whatever names are defined in the package. This includes any names defined (and submodules explicitly loaded) by :file:`__init__.py`. It @@ -541,8 +541,8 @@ previous :keyword:`import` statements. Consider this code:: import sound.effects.surround from sound.effects import * -In this example, the :mod:`echo` and :mod:`surround` modules are imported in the -current namespace because they are defined in the :mod:`sound.effects` package +In this example, the :mod:`!echo` and :mod:`!surround` modules are imported in the +current namespace because they are defined in the :mod:`!sound.effects` package when the ``from...import`` statement is executed. (This also works when ``__all__`` is defined.) @@ -561,15 +561,15 @@ packages. Intra-package References ------------------------ -When packages are structured into subpackages (as with the :mod:`sound` package +When packages are structured into subpackages (as with the :mod:`!sound` package in the example), you can use absolute imports to refer to submodules of siblings -packages. For example, if the module :mod:`sound.filters.vocoder` needs to use -the :mod:`echo` module in the :mod:`sound.effects` package, it can use ``from +packages. For example, if the module :mod:`!sound.filters.vocoder` needs to use +the :mod:`!echo` module in the :mod:`!sound.effects` package, it can use ``from sound.effects import echo``. You can also write relative imports, with the ``from module import name`` form of import statement. These imports use leading dots to indicate the current and -parent packages involved in the relative import. From the :mod:`surround` +parent packages involved in the relative import. From the :mod:`!surround` module for example, you might use:: from . import echo From webhook-mailer at python.org Mon Aug 21 06:44:29 2023 From: webhook-mailer at python.org (vsajip) Date: Mon, 21 Aug 2023 10:44:29 -0000 Subject: [Python-checkins] [3.11] Docs: document 'manager' and '_log' attrs of logging.Logging (GH-108145) (GH-108189) Message-ID: https://github.com/python/cpython/commit/a372274a9b1d1d474831ccc6dd9d4de291acd37f commit: a372274a9b1d1d474831ccc6dd9d4de291acd37f branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: vsajip date: 2023-08-21T11:44:25+01:00 summary: [3.11] Docs: document 'manager' and '_log' attrs of logging.Logging (GH-108145) (GH-108189) (cherry picked from commit f904aa4e1f6943e5bd9a8a73cf762f063e6fa247) Authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Erlend E. Aasland files: M Doc/library/logging.rst diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index f9658fa22d8b2..1fd1d0966a612 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -994,6 +994,14 @@ information into logging calls. For a usage example, see the section on 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the (possibly modified) versions of the arguments passed in. + .. attribute:: manager + + Delegates to the underlying :attr:`!manager`` on *logger*. + + .. attribute:: _log + + Delegates to the underlying :meth:`!_log`` method on *logger*. + In addition to the above, :class:`LoggerAdapter` supports the following methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, :meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, From webhook-mailer at python.org Mon Aug 21 06:53:39 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 21 Aug 2023 10:53:39 -0000 Subject: [Python-checkins] [3.11] Improve references in the tutorial (GH-108069) (GH-108204) Message-ID: https://github.com/python/cpython/commit/be9965308022733bc7b0412a99da9fc80c6c7bef commit: be9965308022733bc7b0412a99da9fc80c6c7bef branch: 3.11 author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-21T10:53:36Z summary: [3.11] Improve references in the tutorial (GH-108069) (GH-108204) * Use full qualified names for references (even if they do not work now, they will work in future). * Silence references to examples. (cherry picked from commit 622ddc41674c2566062af82f7b079aa01d2aae8c) files: M Doc/tools/.nitignore M Doc/tutorial/classes.rst M Doc/tutorial/controlflow.rst M Doc/tutorial/datastructures.rst M Doc/tutorial/inputoutput.rst M Doc/tutorial/modules.rst diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 8325f5b336413..8c44199cb861d 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -166,9 +166,7 @@ Doc/tutorial/appendix.rst Doc/tutorial/classes.rst Doc/tutorial/controlflow.rst Doc/tutorial/datastructures.rst -Doc/tutorial/inputoutput.rst Doc/tutorial/introduction.rst -Doc/tutorial/modules.rst Doc/using/cmdline.rst Doc/using/configure.rst Doc/using/windows.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index db4ca92ae3422..e4002cfa35193 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -91,7 +91,7 @@ Attributes may be read-only or writable. In the latter case, assignment to attributes is possible. Module attributes are writable: you can write ``modname.the_answer = 42``. Writable attributes may also be deleted with the :keyword:`del` statement. For example, ``del modname.the_answer`` will remove -the attribute :attr:`the_answer` from the object named by ``modname``. +the attribute :attr:`!the_answer` from the object named by ``modname``. Namespaces are created at different moments and have different lifetimes. The namespace containing the built-in names is created when the Python interpreter @@ -249,7 +249,7 @@ created. This is basically a wrapper around the contents of the namespace created by the class definition; we'll learn more about class objects in the next section. The original local scope (the one in effect just before the class definition was entered) is reinstated, and the class object is bound here to the -class name given in the class definition header (:class:`ClassName` in the +class name given in the class definition header (:class:`!ClassName` in the example). @@ -291,20 +291,20 @@ variable ``x``. The instantiation operation ("calling" a class object) creates an empty object. Many classes like to create objects with instances customized to a specific initial state. Therefore a class may define a special method named -:meth:`__init__`, like this:: +:meth:`~object.__init__`, like this:: def __init__(self): self.data = [] -When a class defines an :meth:`__init__` method, class instantiation -automatically invokes :meth:`__init__` for the newly created class instance. So +When a class defines an :meth:`~object.__init__` method, class instantiation +automatically invokes :meth:`!__init__` for the newly created class instance. So in this example, a new, initialized instance can be obtained by:: x = MyClass() -Of course, the :meth:`__init__` method may have arguments for greater +Of course, the :meth:`~object.__init__` method may have arguments for greater flexibility. In that case, arguments given to the class instantiation operator -are passed on to :meth:`__init__`. For example, :: +are passed on to :meth:`!__init__`. For example, :: >>> class Complex: ... def __init__(self, realpart, imagpart): @@ -328,7 +328,7 @@ attribute names: data attributes and methods. *data attributes* correspond to "instance variables" in Smalltalk, and to "data members" in C++. Data attributes need not be declared; like local variables, they spring into existence when they are first assigned to. For example, if -``x`` is the instance of :class:`MyClass` created above, the following piece of +``x`` is the instance of :class:`!MyClass` created above, the following piece of code will print the value ``16``, without leaving a trace:: x.counter = 1 @@ -363,7 +363,7 @@ Usually, a method is called right after it is bound:: x.f() -In the :class:`MyClass` example, this will return the string ``'hello world'``. +In the :class:`!MyClass` example, this will return the string ``'hello world'``. However, it is not necessary to call a method right away: ``x.f`` is a method object, and can be stored away and called at a later time. For example:: @@ -375,7 +375,7 @@ will continue to print ``hello world`` until the end of time. What exactly happens when a method is called? You may have noticed that ``x.f()`` was called without an argument above, even though the function -definition for :meth:`f` specified an argument. What happened to the argument? +definition for :meth:`!f` specified an argument. What happened to the argument? Surely Python raises an exception when a function that requires an argument is called without any --- even if the argument isn't actually used... @@ -532,9 +532,9 @@ variable in the class is also ok. For example:: h = g -Now ``f``, ``g`` and ``h`` are all attributes of class :class:`C` that refer to +Now ``f``, ``g`` and ``h`` are all attributes of class :class:`!C` that refer to function objects, and consequently they are all methods of instances of -:class:`C` --- ``h`` being exactly equivalent to ``g``. Note that this practice +:class:`!C` --- ``h`` being exactly equivalent to ``g``. Note that this practice usually only serves to confuse the reader of a program. Methods may call other methods by using method attributes of the ``self`` @@ -581,7 +581,7 @@ this:: . -The name :class:`BaseClassName` must be defined in a scope containing the +The name :class:`!BaseClassName` must be defined in a scope containing the derived class definition. In place of a base class name, other arbitrary expressions are also allowed. This can be useful, for example, when the base class is defined in another module:: @@ -644,9 +644,9 @@ multiple base classes looks like this:: For most purposes, in the simplest cases, you can think of the search for attributes inherited from a parent class as depth-first, left-to-right, not searching twice in the same class where there is an overlap in the hierarchy. -Thus, if an attribute is not found in :class:`DerivedClassName`, it is searched -for in :class:`Base1`, then (recursively) in the base classes of :class:`Base1`, -and if it was not found there, it was searched for in :class:`Base2`, and so on. +Thus, if an attribute is not found in :class:`!DerivedClassName`, it is searched +for in :class:`!Base1`, then (recursively) in the base classes of :class:`!Base1`, +and if it was not found there, it was searched for in :class:`!Base2`, and so on. In fact, it is slightly more complex than that; the method resolution order changes dynamically to support cooperative calls to :func:`super`. This @@ -759,7 +759,8 @@ is to use :mod:`dataclasses` for this purpose:: A piece of Python code that expects a particular abstract data type can often be passed a class that emulates the methods of that data type instead. For instance, if you have a function that formats some data from a file object, you -can define a class with methods :meth:`read` and :meth:`!readline` that get the +can define a class with methods :meth:`~io.TextIOBase.read` and +:meth:`~io.TextIOBase.readline` that get the data from a string buffer instead, and pass it as an argument. .. (Unfortunately, this technique has its limitations: a class can't define @@ -768,7 +769,7 @@ data from a string buffer instead, and pass it as an argument. not cause the interpreter to read further input from it.) Instance method objects have attributes, too: ``m.__self__`` is the instance -object with the method :meth:`m`, and ``m.__func__`` is the function object +object with the method :meth:`!m`, and ``m.__func__`` is the function object corresponding to the method. @@ -817,9 +818,9 @@ using the :func:`next` built-in function; this example shows how it all works:: StopIteration Having seen the mechanics behind the iterator protocol, it is easy to add -iterator behavior to your classes. Define an :meth:`__iter__` method which +iterator behavior to your classes. Define an :meth:`~container.__iter__` method which returns an object with a :meth:`~iterator.__next__` method. If the class -defines :meth:`__next__`, then :meth:`__iter__` can just return ``self``:: +defines :meth:`!__next__`, then :meth:`!__iter__` can just return ``self``:: class Reverse: """Iterator for looping over a sequence backwards.""" @@ -878,7 +879,7 @@ easy to create:: Anything that can be done with generators can also be done with class-based iterators as described in the previous section. What makes generators so -compact is that the :meth:`__iter__` and :meth:`~generator.__next__` methods +compact is that the :meth:`~iterator.__iter__` and :meth:`~generator.__next__` methods are created automatically. Another key feature is that the local variables and execution state are diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 138d87f892e89..4bcc3768111cc 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -534,7 +534,7 @@ This example, as usual, demonstrates some new Python features: Different types define different methods. Methods of different types may have the same name without causing ambiguity. (It is possible to define your own object types and methods, using *classes*, see :ref:`tut-classes`) - The method :meth:`append` shown in the example is defined for list objects; it + The method :meth:`~list.append` shown in the example is defined for list objects; it adds a new element at the end of the list. In this example it is equivalent to ``result = result + [a]``, but more efficient. diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index c8e89d9b79bdd..87614d082a1d4 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -143,8 +143,8 @@ Using Lists as Stacks The list methods make it very easy to use a list as a stack, where the last element added is the first element retrieved ("last-in, first-out"). To add an -item to the top of the stack, use :meth:`append`. To retrieve an item from the -top of the stack, use :meth:`pop` without an explicit index. For example:: +item to the top of the stack, use :meth:`~list.append`. To retrieve an item from the +top of the stack, use :meth:`~list.pop` without an explicit index. For example:: >>> stack = [3, 4, 5] >>> stack.append(6) @@ -341,7 +341,7 @@ The :keyword:`!del` statement ============================= There is a way to remove an item from a list given its index instead of its -value: the :keyword:`del` statement. This differs from the :meth:`pop` method +value: the :keyword:`del` statement. This differs from the :meth:`~list.pop` method which returns a value. The :keyword:`!del` statement can also be used to remove slices from a list or clear the entire list (which we did earlier by assignment of an empty list to the slice). For example:: @@ -501,8 +501,8 @@ any immutable type; strings and numbers can always be keys. Tuples can be used as keys if they contain only strings, numbers, or tuples; if a tuple contains any mutable object either directly or indirectly, it cannot be used as a key. You can't use lists as keys, since lists can be modified in place using index -assignments, slice assignments, or methods like :meth:`append` and -:meth:`extend`. +assignments, slice assignments, or methods like :meth:`~list.append` and +:meth:`~list.extend`. It is best to think of a dictionary as a set of *key: value* pairs, with the requirement that the keys are unique (within one dictionary). A pair of @@ -567,7 +567,7 @@ Looping Techniques ================== When looping through dictionaries, the key and corresponding value can be -retrieved at the same time using the :meth:`items` method. :: +retrieved at the same time using the :meth:`~dict.items` method. :: >>> knights = {'gallahad': 'the pure', 'robin': 'the brave'} >>> for k, v in knights.items(): diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index f5cdd84cbadef..fe9ca9ccb9c7e 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -15,7 +15,7 @@ Fancier Output Formatting ========================= So far we've encountered two ways of writing values: *expression statements* and -the :func:`print` function. (A third way is using the :meth:`write` method +the :func:`print` function. (A third way is using the :meth:`~io.TextIOBase.write` method of file objects; the standard output file can be referenced as ``sys.stdout``. See the Library Reference for more information on this.) @@ -456,8 +456,8 @@ to the very file end with ``seek(0, 2)``) and the only valid *offset* values are those returned from the ``f.tell()``, or zero. Any other *offset* value produces undefined behaviour. -File objects have some additional methods, such as :meth:`~file.isatty` and -:meth:`~file.truncate` which are less frequently used; consult the Library +File objects have some additional methods, such as :meth:`~io.IOBase.isatty` and +:meth:`~io.IOBase.truncate` which are less frequently used; consult the Library Reference for a complete guide to file objects. @@ -469,7 +469,7 @@ Saving structured data with :mod:`json` .. index:: pair: module; json Strings can easily be written to and read from a file. Numbers take a bit more -effort, since the :meth:`read` method only returns strings, which will have to +effort, since the :meth:`~io.TextIOBase.read` method only returns strings, which will have to be passed to a function like :func:`int`, which takes a string like ``'123'`` and returns its numeric value 123. When you want to save more complex data types like nested lists and dictionaries, parsing and serializing by hand diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 734dd1cfe6871..bf9e8e0b7b806 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -183,7 +183,7 @@ The Module Search Path .. index:: triple: module; search; path -When a module named :mod:`spam` is imported, the interpreter first searches for +When a module named :mod:`!spam` is imported, the interpreter first searches for a built-in module with that name. These module names are listed in :data:`sys.builtin_module_names`. If not found, it then searches for a file named :file:`spam.py` in a list of directories given by the variable @@ -389,7 +389,7 @@ Packages ======== Packages are a way of structuring Python's module namespace by using "dotted -module names". For example, the module name :mod:`A.B` designates a submodule +module names". For example, the module name :mod:`!A.B` designates a submodule named ``B`` in a package named ``A``. Just like the use of modules saves the authors of different modules from having to worry about each other's global variable names, the use of dotted module names saves the authors of multi-module @@ -448,7 +448,7 @@ example:: import sound.effects.echo -This loads the submodule :mod:`sound.effects.echo`. It must be referenced with +This loads the submodule :mod:`!sound.effects.echo`. It must be referenced with its full name. :: sound.effects.echo.echofilter(input, output, delay=0.7, atten=4) @@ -457,7 +457,7 @@ An alternative way of importing the submodule is:: from sound.effects import echo -This also loads the submodule :mod:`echo`, and makes it available without its +This also loads the submodule :mod:`!echo`, and makes it available without its package prefix, so it can be used as follows:: echo.echofilter(input, output, delay=0.7, atten=4) @@ -466,8 +466,8 @@ Yet another variation is to import the desired function or variable directly:: from sound.effects.echo import echofilter -Again, this loads the submodule :mod:`echo`, but this makes its function -:func:`echofilter` directly available:: +Again, this loads the submodule :mod:`!echo`, but this makes its function +:func:`!echofilter` directly available:: echofilter(input, output, delay=0.7, atten=4) @@ -510,7 +510,7 @@ code:: __all__ = ["echo", "surround", "reverse"] This would mean that ``from sound.effects import *`` would import the three -named submodules of the :mod:`sound.effects` package. +named submodules of the :mod:`!sound.effects` package. Be aware that submodules might become shadowed by locally defined names. For example, if you added a ``reverse`` function to the @@ -529,8 +529,8 @@ would only import the two submodules ``echo`` and ``surround``, but *not* the return msg[::-1] # in the case of a 'from sound.effects import *' If ``__all__`` is not defined, the statement ``from sound.effects import *`` -does *not* import all submodules from the package :mod:`sound.effects` into the -current namespace; it only ensures that the package :mod:`sound.effects` has +does *not* import all submodules from the package :mod:`!sound.effects` into the +current namespace; it only ensures that the package :mod:`!sound.effects` has been imported (possibly running any initialization code in :file:`__init__.py`) and then imports whatever names are defined in the package. This includes any names defined (and submodules explicitly loaded) by :file:`__init__.py`. It @@ -541,8 +541,8 @@ previous :keyword:`import` statements. Consider this code:: import sound.effects.surround from sound.effects import * -In this example, the :mod:`echo` and :mod:`surround` modules are imported in the -current namespace because they are defined in the :mod:`sound.effects` package +In this example, the :mod:`!echo` and :mod:`!surround` modules are imported in the +current namespace because they are defined in the :mod:`!sound.effects` package when the ``from...import`` statement is executed. (This also works when ``__all__`` is defined.) @@ -561,15 +561,15 @@ packages. Intra-package References ------------------------ -When packages are structured into subpackages (as with the :mod:`sound` package +When packages are structured into subpackages (as with the :mod:`!sound` package in the example), you can use absolute imports to refer to submodules of siblings -packages. For example, if the module :mod:`sound.filters.vocoder` needs to use -the :mod:`echo` module in the :mod:`sound.effects` package, it can use ``from +packages. For example, if the module :mod:`!sound.filters.vocoder` needs to use +the :mod:`!echo` module in the :mod:`!sound.effects` package, it can use ``from sound.effects import echo``. You can also write relative imports, with the ``from module import name`` form of import statement. These imports use leading dots to indicate the current and -parent packages involved in the relative import. From the :mod:`surround` +parent packages involved in the relative import. From the :mod:`!surround` module for example, you might use:: from . import echo From webhook-mailer at python.org Mon Aug 21 06:56:49 2023 From: webhook-mailer at python.org (encukou) Date: Mon, 21 Aug 2023 10:56:49 -0000 Subject: [Python-checkins] gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) Message-ID: https://github.com/python/cpython/commit/acbd3f9c5c5f23e95267714e41236140d84fe962 commit: acbd3f9c5c5f23e95267714e41236140d84fe962 branch: main author: Petr Viktorin committer: encukou date: 2023-08-21T12:56:46+02:00 summary: gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) Co-authored-by: Victor Stinner Co-authored-by: Lum?r 'Frenzy' Balhar files: A Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst M Doc/library/tarfile.rst M Lib/tarfile.py M Lib/test/test_tarfile.py diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 00f3070324ec1..62d67bc577c7d 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -740,6 +740,11 @@ A ``TarInfo`` object has the following public data attributes: Name of the target file name, which is only present in :class:`TarInfo` objects of type :const:`LNKTYPE` and :const:`SYMTYPE`. + For symbolic links (``SYMTYPE``), the *linkname* is relative to the directory + that contains the link. + For hard links (``LNKTYPE``), the *linkname* is relative to the root of + the archive. + .. attribute:: TarInfo.uid :type: int diff --git a/Lib/tarfile.py b/Lib/tarfile.py index df4e41f7a0d23..1147d59d3d510 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -742,7 +742,7 @@ def __init__(self, tarinfo): class AbsoluteLinkError(FilterError): def __init__(self, tarinfo): self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a symlink to an absolute path') + super().__init__(f'{tarinfo.name!r} is a link to an absolute path') class LinkOutsideDestinationError(FilterError): def __init__(self, tarinfo, path): @@ -802,7 +802,14 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) - target_path = os.path.realpath(os.path.join(dest_path, member.linkname)) + if member.issym(): + target_path = os.path.join(dest_path, + os.path.dirname(name), + member.linkname) + else: + target_path = os.path.join(dest_path, + member.linkname) + target_path = os.path.realpath(target_path) if os.path.commonpath([target_path, dest_path]) != dest_path: raise LinkOutsideDestinationError(member, target_path) return new_attrs diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 2eda7fc4ceac7..2cdf2c1fc2f29 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3337,10 +3337,12 @@ def __exit__(self, *exc): self.bio = None def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, - mode=None, **kwargs): + mode=None, size=None, **kwargs): """Add a member to the test archive. Call within `with`.""" name = str(name) tarinfo = tarfile.TarInfo(name).replace(**kwargs) + if size is not None: + tarinfo.size = size if mode: tarinfo.mode = _filemode_to_int(mode) if symlink_to is not None: @@ -3416,7 +3418,8 @@ def check_context(self, tar, filter): raise self.raised_exception self.assertEqual(self.expected_paths, set()) - def expect_file(self, name, type=None, symlink_to=None, mode=None): + def expect_file(self, name, type=None, symlink_to=None, mode=None, + size=None): """Check a single file. See check_context.""" if self.raised_exception: raise self.raised_exception @@ -3445,6 +3448,8 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None): self.assertTrue(path.is_fifo()) else: raise NotImplementedError(type) + if size is not None: + self.assertEqual(path.stat().st_size, size) for parent in path.parents: self.expected_paths.discard(parent) @@ -3491,8 +3496,15 @@ def test_parent_symlink(self): # Test interplaying symlinks # Inspired by 'dirsymlink2a' in jwilk/traversal-archives with ArchiveMaker() as arc: + + # `current` links to `.` which is both: + # - the destination directory + # - `current` itself arc.add('current', symlink_to='.') + + # effectively points to ./../ arc.add('parent', symlink_to='current/..') + arc.add('parent/evil') if os_helper.can_symlink(): @@ -3534,9 +3546,46 @@ def test_parent_symlink(self): def test_parent_symlink2(self): # Test interplaying symlinks # Inspired by 'dirsymlink2b' in jwilk/traversal-archives + + # Posix and Windows have different pathname resolution: + # either symlink or a '..' component resolve first. + # Let's see which we are on. + if os_helper.can_symlink(): + testpath = os.path.join(TEMPDIR, 'resolution_test') + os.mkdir(testpath) + + # testpath/current links to `.` which is all of: + # - `testpath` + # - `testpath/current` + # - `testpath/current/current` + # - etc. + os.symlink('.', os.path.join(testpath, 'current')) + + # we'll test where `testpath/current/../file` ends up + with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): + pass + + if os.path.exists(os.path.join(testpath, 'file')): + # Windows collapses 'current\..' to '.' first, leaving + # 'testpath\file' + dotdot_resolves_early = True + elif os.path.exists(os.path.join(testpath, '..', 'file')): + # Posix resolves 'current' to '.' first, leaving + # 'testpath/../file' + dotdot_resolves_early = False + else: + raise AssertionError('Could not determine link resolution') + with ArchiveMaker() as arc: + + # `current` links to `.` which is both the destination directory + # and `current` itself arc.add('current', symlink_to='.') + + # `current/parent` is also available as `./parent`, + # and effectively points to `./../` arc.add('current/parent', symlink_to='..') + arc.add('parent/evil') with self.check_context(arc.open(), 'fully_trusted'): @@ -3550,6 +3599,7 @@ def test_parent_symlink2(self): with self.check_context(arc.open(), 'tar'): if os_helper.can_symlink(): + # Fail when extracting a file outside destination self.expect_exception( tarfile.OutsideDestinationError, "'parent/evil' would be extracted to " @@ -3560,10 +3610,24 @@ def test_parent_symlink2(self): self.expect_file('parent/evil') with self.check_context(arc.open(), 'data'): - self.expect_exception( - tarfile.LinkOutsideDestinationError, - """'current/parent' would link to ['"].*['"], """ - + "which is outside the destination") + if os_helper.can_symlink(): + if dotdot_resolves_early: + # Fail when extracting a file outside destination + self.expect_exception( + tarfile.OutsideDestinationError, + "'parent/evil' would be extracted to " + + """['"].*evil['"], which is outside """ + + "the destination") + else: + # Fail as soon as we have a symlink outside the destination + self.expect_exception( + tarfile.LinkOutsideDestinationError, + "'current/parent' would link to " + + """['"].*outerdir['"], which is outside """ + + "the destination") + else: + self.expect_file('current/') + self.expect_file('parent/evil') @symlink_test def test_absolute_symlink(self): @@ -3593,12 +3657,30 @@ def test_absolute_symlink(self): with self.check_context(arc.open(), 'data'): self.expect_exception( tarfile.AbsoluteLinkError, - "'parent' is a symlink to an absolute path") + "'parent' is a link to an absolute path") + + def test_absolute_hardlink(self): + # Test hardlink to an absolute path + # Inspired by 'dirsymlink' in https://github.com/jwilk/traversal-archives + with ArchiveMaker() as arc: + arc.add('parent', hardlink_to=self.outerdir / 'foo') + + with self.check_context(arc.open(), 'fully_trusted'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'tar'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.AbsoluteLinkError, + "'parent' is a link to an absolute path") @symlink_test def test_sly_relative0(self): # Inspired by 'relative0' in jwilk/traversal-archives with ArchiveMaker() as arc: + # points to `../../tmp/moo` arc.add('../moo', symlink_to='..//tmp/moo') try: @@ -3649,6 +3731,56 @@ def test_sly_relative2(self): + """['"].*moo['"], which is outside the """ + "destination") + @symlink_test + def test_deep_symlink(self): + # Test that symlinks and hardlinks inside a directory + # point to the correct file (`target` of size 3). + # If links aren't supported we get a copy of the file. + with ArchiveMaker() as arc: + arc.add('targetdir/target', size=3) + # a hardlink's linkname is relative to the archive + arc.add('linkdir/hardlink', hardlink_to=os.path.join( + 'targetdir', 'target')) + # a symlink's linkname is relative to the link's directory + arc.add('linkdir/symlink', symlink_to=os.path.join( + '..', 'targetdir', 'target')) + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + if os_helper.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='../targetdir/target') + else: + self.expect_file('linkdir/symlink', size=3) + + @symlink_test + def test_chains(self): + # Test chaining of symlinks/hardlinks. + # Symlinks are created before the files they point to. + with ArchiveMaker() as arc: + arc.add('linkdir/symlink', symlink_to='hardlink') + arc.add('symlink2', symlink_to=os.path.join( + 'linkdir', 'hardlink2')) + arc.add('targetdir/target', size=3) + arc.add('linkdir/hardlink', hardlink_to='targetdir/target') + arc.add('linkdir/hardlink2', hardlink_to='linkdir/symlink') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + self.expect_file('linkdir/hardlink2', size=3) + if os_helper.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='hardlink') + self.expect_file('symlink2', size=3, + symlink_to='linkdir/hardlink2') + else: + self.expect_file('linkdir/symlink', size=3) + self.expect_file('symlink2', size=3) + def test_modes(self): # Test how file modes are extracted # (Note that the modes are ignored on platforms without working chmod) diff --git a/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst new file mode 100644 index 0000000000000..32c1fb93f4ab2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst @@ -0,0 +1,3 @@ +:func:`tarfile.data_filter` now takes the location of symlinks into account +when determining their target, so it will no longer reject some valid +tarballs with ``LinkOutsideDestinationError``. From webhook-mailer at python.org Mon Aug 21 07:16:34 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 21 Aug 2023 11:16:34 -0000 Subject: [Python-checkins] gh-107916: Save the error code before decoding the filename in PyErr_SetFromErrnoWithFilename() etc (GH-107929) Message-ID: https://github.com/python/cpython/commit/80bdebdd8593f007a2232ec04a7729bba6ebf12c commit: 80bdebdd8593f007a2232ec04a7729bba6ebf12c branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-21T14:16:31+03:00 summary: gh-107916: Save the error code before decoding the filename in PyErr_SetFromErrnoWithFilename() etc (GH-107929) files: A Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst M Python/errors.c diff --git a/Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst b/Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst new file mode 100644 index 0000000000000..f1f16609b118b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst @@ -0,0 +1,4 @@ +C API functions :c:func:`PyErr_SetFromErrnoWithFilename`, +:c:func:`PyErr_SetExcFromWindowsErrWithFilename` and +:c:func:`PyErr_SetFromWindowsErrWithFilename` save now the error code before +calling :c:func:`PyUnicode_DecodeFSDefault`. diff --git a/Python/errors.c b/Python/errors.c index d9086c507251e..fb5b3ff2c7ba5 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -895,10 +895,12 @@ PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) { PyObject *name = NULL; if (filename) { + int i = errno; name = PyUnicode_DecodeFSDefault(filename); if (name == NULL) { return NULL; } + errno = i; } PyObject *result = PyErr_SetFromErrnoWithFilenameObjects(exc, name, NULL); Py_XDECREF(name); @@ -998,6 +1000,9 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilename( { PyObject *name = NULL; if (filename) { + if ((DWORD)ierr == 0) { + ierr = (int)GetLastError(); + } name = PyUnicode_DecodeFSDefault(filename); if (name == NULL) { return NULL; @@ -1028,6 +1033,9 @@ PyObject *PyErr_SetFromWindowsErrWithFilename( { PyObject *name = NULL; if (filename) { + if ((DWORD)ierr == 0) { + ierr = (int)GetLastError(); + } name = PyUnicode_DecodeFSDefault(filename); if (name == NULL) { return NULL; From webhook-mailer at python.org Mon Aug 21 07:16:57 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 21 Aug 2023 11:16:57 -0000 Subject: [Python-checkins] [3.12] Docs: document 'manager' and '_log' attrs of logging.Logging (GH-108145) (#108190) Message-ID: https://github.com/python/cpython/commit/9cd15cab772e9e6073b3074ab6ef85db83a0e40c commit: 9cd15cab772e9e6073b3074ab6ef85db83a0e40c branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-21T13:16:53+02:00 summary: [3.12] Docs: document 'manager' and '_log' attrs of logging.Logging (GH-108145) (#108190) Docs: document 'manager' and '_log' attrs of logging.Logging (GH-108145) (cherry picked from commit f904aa4e1f6943e5bd9a8a73cf762f063e6fa247) Authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Erlend E. Aasland files: M Doc/library/logging.rst diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index edec0e0cdfa99..d369c53f0f919 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1015,6 +1015,14 @@ information into logging calls. For a usage example, see the section on 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the (possibly modified) versions of the arguments passed in. + .. attribute:: manager + + Delegates to the underlying :attr:`!manager`` on *logger*. + + .. attribute:: _log + + Delegates to the underlying :meth:`!_log`` method on *logger*. + In addition to the above, :class:`LoggerAdapter` supports the following methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, :meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, From webhook-mailer at python.org Mon Aug 21 07:18:08 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 21 Aug 2023 11:18:08 -0000 Subject: [Python-checkins] [3.12] gh-107895: Fix test_asyncio.test_runners when run it in CPython's "development mode" (GH-108168) (#108196) Message-ID: https://github.com/python/cpython/commit/639ffd5aa0fcc1a66cb99ce2afa31a93e7e5f7b4 commit: 639ffd5aa0fcc1a66cb99ce2afa31a93e7e5f7b4 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-21T13:18:04+02:00 summary: [3.12] gh-107895: Fix test_asyncio.test_runners when run it in CPython's "development mode" (GH-108168) (#108196) gh-107895: Fix test_asyncio.test_runners when run it in CPython's "development mode" (GH-108168) (cherry picked from commit 014a5b71e7538926ae1c03c8c5ea13c96e741be3) Co-authored-by: Joon Hwan ??? files: M Lib/test/test_asyncio/test_runners.py diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 811cf8b72488b..0a1ad070c0c1f 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -101,11 +101,14 @@ async def main(expected): loop = asyncio.get_event_loop() self.assertIs(loop.get_debug(), expected) - asyncio.run(main(False)) + asyncio.run(main(False), debug=False) asyncio.run(main(True), debug=True) with mock.patch('asyncio.coroutines._is_debug_mode', lambda: True): asyncio.run(main(True)) asyncio.run(main(False), debug=False) + with mock.patch('asyncio.coroutines._is_debug_mode', lambda: False): + asyncio.run(main(True), debug=True) + asyncio.run(main(False)) def test_asyncio_run_from_running_loop(self): async def main(): From webhook-mailer at python.org Mon Aug 21 07:20:25 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 21 Aug 2023 11:20:25 -0000 Subject: [Python-checkins] [3.12] Improve references in the tutorial (GH-108069) (#108203) Message-ID: https://github.com/python/cpython/commit/f798a6360b038a3fc47509101eb77bcc1c9b6662 commit: f798a6360b038a3fc47509101eb77bcc1c9b6662 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-21T13:20:21+02:00 summary: [3.12] Improve references in the tutorial (GH-108069) (#108203) Improve references in the tutorial (GH-108069) * Use full qualified names for references (even if they do not work now, they will work in future). * Silence references to examples. (cherry picked from commit 622ddc41674c2566062af82f7b079aa01d2aae8c) Co-authored-by: Serhiy Storchaka files: M Doc/tools/.nitignore M Doc/tutorial/classes.rst M Doc/tutorial/controlflow.rst M Doc/tutorial/datastructures.rst M Doc/tutorial/inputoutput.rst M Doc/tutorial/modules.rst diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index b1894c4d3df37..ada46064e358e 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -180,9 +180,7 @@ Doc/tutorial/appendix.rst Doc/tutorial/classes.rst Doc/tutorial/controlflow.rst Doc/tutorial/datastructures.rst -Doc/tutorial/inputoutput.rst Doc/tutorial/introduction.rst -Doc/tutorial/modules.rst Doc/using/cmdline.rst Doc/using/configure.rst Doc/using/windows.rst diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 06445e000c1ef..91a3b73d2b55a 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -91,7 +91,7 @@ Attributes may be read-only or writable. In the latter case, assignment to attributes is possible. Module attributes are writable: you can write ``modname.the_answer = 42``. Writable attributes may also be deleted with the :keyword:`del` statement. For example, ``del modname.the_answer`` will remove -the attribute :attr:`the_answer` from the object named by ``modname``. +the attribute :attr:`!the_answer` from the object named by ``modname``. Namespaces are created at different moments and have different lifetimes. The namespace containing the built-in names is created when the Python interpreter @@ -249,7 +249,7 @@ created. This is basically a wrapper around the contents of the namespace created by the class definition; we'll learn more about class objects in the next section. The original local scope (the one in effect just before the class definition was entered) is reinstated, and the class object is bound here to the -class name given in the class definition header (:class:`ClassName` in the +class name given in the class definition header (:class:`!ClassName` in the example). @@ -291,20 +291,20 @@ variable ``x``. The instantiation operation ("calling" a class object) creates an empty object. Many classes like to create objects with instances customized to a specific initial state. Therefore a class may define a special method named -:meth:`__init__`, like this:: +:meth:`~object.__init__`, like this:: def __init__(self): self.data = [] -When a class defines an :meth:`__init__` method, class instantiation -automatically invokes :meth:`__init__` for the newly created class instance. So +When a class defines an :meth:`~object.__init__` method, class instantiation +automatically invokes :meth:`!__init__` for the newly created class instance. So in this example, a new, initialized instance can be obtained by:: x = MyClass() -Of course, the :meth:`__init__` method may have arguments for greater +Of course, the :meth:`~object.__init__` method may have arguments for greater flexibility. In that case, arguments given to the class instantiation operator -are passed on to :meth:`__init__`. For example, :: +are passed on to :meth:`!__init__`. For example, :: >>> class Complex: ... def __init__(self, realpart, imagpart): @@ -328,7 +328,7 @@ attribute names: data attributes and methods. *data attributes* correspond to "instance variables" in Smalltalk, and to "data members" in C++. Data attributes need not be declared; like local variables, they spring into existence when they are first assigned to. For example, if -``x`` is the instance of :class:`MyClass` created above, the following piece of +``x`` is the instance of :class:`!MyClass` created above, the following piece of code will print the value ``16``, without leaving a trace:: x.counter = 1 @@ -363,7 +363,7 @@ Usually, a method is called right after it is bound:: x.f() -In the :class:`MyClass` example, this will return the string ``'hello world'``. +In the :class:`!MyClass` example, this will return the string ``'hello world'``. However, it is not necessary to call a method right away: ``x.f`` is a method object, and can be stored away and called at a later time. For example:: @@ -375,7 +375,7 @@ will continue to print ``hello world`` until the end of time. What exactly happens when a method is called? You may have noticed that ``x.f()`` was called without an argument above, even though the function -definition for :meth:`f` specified an argument. What happened to the argument? +definition for :meth:`!f` specified an argument. What happened to the argument? Surely Python raises an exception when a function that requires an argument is called without any --- even if the argument isn't actually used... @@ -532,9 +532,9 @@ variable in the class is also ok. For example:: h = g -Now ``f``, ``g`` and ``h`` are all attributes of class :class:`C` that refer to +Now ``f``, ``g`` and ``h`` are all attributes of class :class:`!C` that refer to function objects, and consequently they are all methods of instances of -:class:`C` --- ``h`` being exactly equivalent to ``g``. Note that this practice +:class:`!C` --- ``h`` being exactly equivalent to ``g``. Note that this practice usually only serves to confuse the reader of a program. Methods may call other methods by using method attributes of the ``self`` @@ -581,7 +581,7 @@ this:: . -The name :class:`BaseClassName` must be defined in a +The name :class:`!BaseClassName` must be defined in a namespace accessible from the scope containing the derived class definition. In place of a base class name, other arbitrary expressions are also allowed. This can be useful, for example, when the base @@ -645,9 +645,9 @@ multiple base classes looks like this:: For most purposes, in the simplest cases, you can think of the search for attributes inherited from a parent class as depth-first, left-to-right, not searching twice in the same class where there is an overlap in the hierarchy. -Thus, if an attribute is not found in :class:`DerivedClassName`, it is searched -for in :class:`Base1`, then (recursively) in the base classes of :class:`Base1`, -and if it was not found there, it was searched for in :class:`Base2`, and so on. +Thus, if an attribute is not found in :class:`!DerivedClassName`, it is searched +for in :class:`!Base1`, then (recursively) in the base classes of :class:`!Base1`, +and if it was not found there, it was searched for in :class:`!Base2`, and so on. In fact, it is slightly more complex than that; the method resolution order changes dynamically to support cooperative calls to :func:`super`. This @@ -760,7 +760,8 @@ is to use :mod:`dataclasses` for this purpose:: A piece of Python code that expects a particular abstract data type can often be passed a class that emulates the methods of that data type instead. For instance, if you have a function that formats some data from a file object, you -can define a class with methods :meth:`read` and :meth:`!readline` that get the +can define a class with methods :meth:`~io.TextIOBase.read` and +:meth:`~io.TextIOBase.readline` that get the data from a string buffer instead, and pass it as an argument. .. (Unfortunately, this technique has its limitations: a class can't define @@ -769,7 +770,7 @@ data from a string buffer instead, and pass it as an argument. not cause the interpreter to read further input from it.) Instance method objects have attributes, too: ``m.__self__`` is the instance -object with the method :meth:`m`, and ``m.__func__`` is the function object +object with the method :meth:`!m`, and ``m.__func__`` is the function object corresponding to the method. @@ -818,9 +819,9 @@ using the :func:`next` built-in function; this example shows how it all works:: StopIteration Having seen the mechanics behind the iterator protocol, it is easy to add -iterator behavior to your classes. Define an :meth:`__iter__` method which +iterator behavior to your classes. Define an :meth:`~container.__iter__` method which returns an object with a :meth:`~iterator.__next__` method. If the class -defines :meth:`__next__`, then :meth:`__iter__` can just return ``self``:: +defines :meth:`!__next__`, then :meth:`!__iter__` can just return ``self``:: class Reverse: """Iterator for looping over a sequence backwards.""" @@ -879,7 +880,7 @@ easy to create:: Anything that can be done with generators can also be done with class-based iterators as described in the previous section. What makes generators so -compact is that the :meth:`__iter__` and :meth:`~generator.__next__` methods +compact is that the :meth:`~iterator.__iter__` and :meth:`~generator.__next__` methods are created automatically. Another key feature is that the local variables and execution state are diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 138d87f892e89..4bcc3768111cc 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -534,7 +534,7 @@ This example, as usual, demonstrates some new Python features: Different types define different methods. Methods of different types may have the same name without causing ambiguity. (It is possible to define your own object types and methods, using *classes*, see :ref:`tut-classes`) - The method :meth:`append` shown in the example is defined for list objects; it + The method :meth:`~list.append` shown in the example is defined for list objects; it adds a new element at the end of the list. In this example it is equivalent to ``result = result + [a]``, but more efficient. diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index c8e89d9b79bdd..87614d082a1d4 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -143,8 +143,8 @@ Using Lists as Stacks The list methods make it very easy to use a list as a stack, where the last element added is the first element retrieved ("last-in, first-out"). To add an -item to the top of the stack, use :meth:`append`. To retrieve an item from the -top of the stack, use :meth:`pop` without an explicit index. For example:: +item to the top of the stack, use :meth:`~list.append`. To retrieve an item from the +top of the stack, use :meth:`~list.pop` without an explicit index. For example:: >>> stack = [3, 4, 5] >>> stack.append(6) @@ -341,7 +341,7 @@ The :keyword:`!del` statement ============================= There is a way to remove an item from a list given its index instead of its -value: the :keyword:`del` statement. This differs from the :meth:`pop` method +value: the :keyword:`del` statement. This differs from the :meth:`~list.pop` method which returns a value. The :keyword:`!del` statement can also be used to remove slices from a list or clear the entire list (which we did earlier by assignment of an empty list to the slice). For example:: @@ -501,8 +501,8 @@ any immutable type; strings and numbers can always be keys. Tuples can be used as keys if they contain only strings, numbers, or tuples; if a tuple contains any mutable object either directly or indirectly, it cannot be used as a key. You can't use lists as keys, since lists can be modified in place using index -assignments, slice assignments, or methods like :meth:`append` and -:meth:`extend`. +assignments, slice assignments, or methods like :meth:`~list.append` and +:meth:`~list.extend`. It is best to think of a dictionary as a set of *key: value* pairs, with the requirement that the keys are unique (within one dictionary). A pair of @@ -567,7 +567,7 @@ Looping Techniques ================== When looping through dictionaries, the key and corresponding value can be -retrieved at the same time using the :meth:`items` method. :: +retrieved at the same time using the :meth:`~dict.items` method. :: >>> knights = {'gallahad': 'the pure', 'robin': 'the brave'} >>> for k, v in knights.items(): diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index f5cdd84cbadef..fe9ca9ccb9c7e 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -15,7 +15,7 @@ Fancier Output Formatting ========================= So far we've encountered two ways of writing values: *expression statements* and -the :func:`print` function. (A third way is using the :meth:`write` method +the :func:`print` function. (A third way is using the :meth:`~io.TextIOBase.write` method of file objects; the standard output file can be referenced as ``sys.stdout``. See the Library Reference for more information on this.) @@ -456,8 +456,8 @@ to the very file end with ``seek(0, 2)``) and the only valid *offset* values are those returned from the ``f.tell()``, or zero. Any other *offset* value produces undefined behaviour. -File objects have some additional methods, such as :meth:`~file.isatty` and -:meth:`~file.truncate` which are less frequently used; consult the Library +File objects have some additional methods, such as :meth:`~io.IOBase.isatty` and +:meth:`~io.IOBase.truncate` which are less frequently used; consult the Library Reference for a complete guide to file objects. @@ -469,7 +469,7 @@ Saving structured data with :mod:`json` .. index:: pair: module; json Strings can easily be written to and read from a file. Numbers take a bit more -effort, since the :meth:`read` method only returns strings, which will have to +effort, since the :meth:`~io.TextIOBase.read` method only returns strings, which will have to be passed to a function like :func:`int`, which takes a string like ``'123'`` and returns its numeric value 123. When you want to save more complex data types like nested lists and dictionaries, parsing and serializing by hand diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 734dd1cfe6871..bf9e8e0b7b806 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -183,7 +183,7 @@ The Module Search Path .. index:: triple: module; search; path -When a module named :mod:`spam` is imported, the interpreter first searches for +When a module named :mod:`!spam` is imported, the interpreter first searches for a built-in module with that name. These module names are listed in :data:`sys.builtin_module_names`. If not found, it then searches for a file named :file:`spam.py` in a list of directories given by the variable @@ -389,7 +389,7 @@ Packages ======== Packages are a way of structuring Python's module namespace by using "dotted -module names". For example, the module name :mod:`A.B` designates a submodule +module names". For example, the module name :mod:`!A.B` designates a submodule named ``B`` in a package named ``A``. Just like the use of modules saves the authors of different modules from having to worry about each other's global variable names, the use of dotted module names saves the authors of multi-module @@ -448,7 +448,7 @@ example:: import sound.effects.echo -This loads the submodule :mod:`sound.effects.echo`. It must be referenced with +This loads the submodule :mod:`!sound.effects.echo`. It must be referenced with its full name. :: sound.effects.echo.echofilter(input, output, delay=0.7, atten=4) @@ -457,7 +457,7 @@ An alternative way of importing the submodule is:: from sound.effects import echo -This also loads the submodule :mod:`echo`, and makes it available without its +This also loads the submodule :mod:`!echo`, and makes it available without its package prefix, so it can be used as follows:: echo.echofilter(input, output, delay=0.7, atten=4) @@ -466,8 +466,8 @@ Yet another variation is to import the desired function or variable directly:: from sound.effects.echo import echofilter -Again, this loads the submodule :mod:`echo`, but this makes its function -:func:`echofilter` directly available:: +Again, this loads the submodule :mod:`!echo`, but this makes its function +:func:`!echofilter` directly available:: echofilter(input, output, delay=0.7, atten=4) @@ -510,7 +510,7 @@ code:: __all__ = ["echo", "surround", "reverse"] This would mean that ``from sound.effects import *`` would import the three -named submodules of the :mod:`sound.effects` package. +named submodules of the :mod:`!sound.effects` package. Be aware that submodules might become shadowed by locally defined names. For example, if you added a ``reverse`` function to the @@ -529,8 +529,8 @@ would only import the two submodules ``echo`` and ``surround``, but *not* the return msg[::-1] # in the case of a 'from sound.effects import *' If ``__all__`` is not defined, the statement ``from sound.effects import *`` -does *not* import all submodules from the package :mod:`sound.effects` into the -current namespace; it only ensures that the package :mod:`sound.effects` has +does *not* import all submodules from the package :mod:`!sound.effects` into the +current namespace; it only ensures that the package :mod:`!sound.effects` has been imported (possibly running any initialization code in :file:`__init__.py`) and then imports whatever names are defined in the package. This includes any names defined (and submodules explicitly loaded) by :file:`__init__.py`. It @@ -541,8 +541,8 @@ previous :keyword:`import` statements. Consider this code:: import sound.effects.surround from sound.effects import * -In this example, the :mod:`echo` and :mod:`surround` modules are imported in the -current namespace because they are defined in the :mod:`sound.effects` package +In this example, the :mod:`!echo` and :mod:`!surround` modules are imported in the +current namespace because they are defined in the :mod:`!sound.effects` package when the ``from...import`` statement is executed. (This also works when ``__all__`` is defined.) @@ -561,15 +561,15 @@ packages. Intra-package References ------------------------ -When packages are structured into subpackages (as with the :mod:`sound` package +When packages are structured into subpackages (as with the :mod:`!sound` package in the example), you can use absolute imports to refer to submodules of siblings -packages. For example, if the module :mod:`sound.filters.vocoder` needs to use -the :mod:`echo` module in the :mod:`sound.effects` package, it can use ``from +packages. For example, if the module :mod:`!sound.filters.vocoder` needs to use +the :mod:`!echo` module in the :mod:`!sound.effects` package, it can use ``from sound.effects import echo``. You can also write relative imports, with the ``from module import name`` form of import statement. These imports use leading dots to indicate the current and -parent packages involved in the relative import. From the :mod:`surround` +parent packages involved in the relative import. From the :mod:`!surround` module for example, you might use:: from . import echo From webhook-mailer at python.org Mon Aug 21 07:39:10 2023 From: webhook-mailer at python.org (encukou) Date: Mon, 21 Aug 2023 11:39:10 -0000 Subject: [Python-checkins] gh-107396: tarfiles: set self.exception before _init_read_gz() (GH-107485) Message-ID: https://github.com/python/cpython/commit/37135d25e269ede92bc7da363bebfa574782e59a commit: 37135d25e269ede92bc7da363bebfa574782e59a branch: main author: balmeida-nokia <83089745+balmeida-nokia at users.noreply.github.com> committer: encukou date: 2023-08-21T11:39:06Z summary: gh-107396: tarfiles: set self.exception before _init_read_gz() (GH-107485) In the stack call of: _init_read_gz() ``` _read, tarfile.py:548 read, tarfile.py:526 _init_read_gz, tarfile.py:491 ``` a try;except exists that uses `self.exception`, so it needs to be set before calling _init_read_gz(). files: A Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst M Lib/tarfile.py M Lib/test/test_tarfile.py diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 1147d59d3d510..a835d00c90c92 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -372,8 +372,8 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, self.zlib = zlib self.crc = zlib.crc32(b"") if mode == "r": - self._init_read_gz() self.exception = zlib.error + self._init_read_gz() else: self._init_write_gz(compresslevel) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 2cdf2c1fc2f29..0d6ca4315cfda 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -938,6 +938,23 @@ class LzmaDetectReadTest(LzmaTest, DetectReadTest): pass +class GzipBrokenHeaderCorrectException(GzipTest, unittest.TestCase): + """ + See: https://github.com/python/cpython/issues/107396 + """ + def runTest(self): + f = io.BytesIO( + b'\x1f\x8b' # header + b'\x08' # compression method + b'\x04' # flags + b'\0\0\0\0\0\0' # timestamp, compression data, OS ID + b'\0\x01' # size + b'\0\0\0\0\0' # corrupt data (zeros) + ) + with self.assertRaises(tarfile.ReadError): + tarfile.open(fileobj=f, mode='r|gz') + + class MemberReadTest(ReadTest, unittest.TestCase): def _test_member(self, tarinfo, chksum=None, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst b/Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst new file mode 100644 index 0000000000000..73bff4bdbe024 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst @@ -0,0 +1 @@ +tarfiles; Fixed use before assignment of self.exception for gzip decompression From webhook-mailer at python.org Mon Aug 21 07:53:08 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 21 Aug 2023 11:53:08 -0000 Subject: [Python-checkins] [3.11] gh-107916: Save the error code before decoding the filename in PyErr_SetFromErrnoWithFilename() etc (GH-107929) (GH-108206) Message-ID: https://github.com/python/cpython/commit/ed67e60f48d4e2d881eab0ab12e31e259b126f28 commit: ed67e60f48d4e2d881eab0ab12e31e259b126f28 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: serhiy-storchaka date: 2023-08-21T11:53:05Z summary: [3.11] gh-107916: Save the error code before decoding the filename in PyErr_SetFromErrnoWithFilename() etc (GH-107929) (GH-108206) (cherry picked from commit 80bdebdd8593f007a2232ec04a7729bba6ebf12c) Co-authored-by: Serhiy Storchaka files: A Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst M Python/errors.c diff --git a/Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst b/Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst new file mode 100644 index 0000000000000..f1f16609b118b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst @@ -0,0 +1,4 @@ +C API functions :c:func:`PyErr_SetFromErrnoWithFilename`, +:c:func:`PyErr_SetExcFromWindowsErrWithFilename` and +:c:func:`PyErr_SetFromWindowsErrWithFilename` save now the error code before +calling :c:func:`PyUnicode_DecodeFSDefault`. diff --git a/Python/errors.c b/Python/errors.c index d693f3a24126a..8e150c3d008b7 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -856,10 +856,12 @@ PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) { PyObject *name = NULL; if (filename) { + int i = errno; name = PyUnicode_DecodeFSDefault(filename); if (name == NULL) { return NULL; } + errno = i; } PyObject *result = PyErr_SetFromErrnoWithFilenameObjects(exc, name, NULL); Py_XDECREF(name); @@ -959,6 +961,9 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilename( { PyObject *name = NULL; if (filename) { + if ((DWORD)ierr == 0) { + ierr = (int)GetLastError(); + } name = PyUnicode_DecodeFSDefault(filename); if (name == NULL) { return NULL; @@ -989,6 +994,9 @@ PyObject *PyErr_SetFromWindowsErrWithFilename( { PyObject *name = NULL; if (filename) { + if ((DWORD)ierr == 0) { + ierr = (int)GetLastError(); + } name = PyUnicode_DecodeFSDefault(filename); if (name == NULL) { return NULL; From webhook-mailer at python.org Mon Aug 21 08:31:15 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 21 Aug 2023 12:31:15 -0000 Subject: [Python-checkins] [3.12] gh-105736: Sync pure python version of OrderedDict with the C version (GH-108098) (#108200) Message-ID: https://github.com/python/cpython/commit/1ce8b92ce92e9a44ac804ea4e4c61d08d7e89a2e commit: 1ce8b92ce92e9a44ac804ea4e4c61d08d7e89a2e branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-21T14:31:11+02:00 summary: [3.12] gh-105736: Sync pure python version of OrderedDict with the C version (GH-108098) (#108200) gh-105736: Sync pure python version of OrderedDict with the C version (GH-108098) (cherry picked from commit 20cc90c0df3e368fe7cb63d958f0b17a78fa9d0a) Co-authored-by: Raymond Hettinger files: A Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst M Lib/collections/__init__.py M Lib/test/test_ordered_dict.py diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 03ca2d7e18f6f..8652dc8a4ec45 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -95,17 +95,19 @@ class OrderedDict(dict): # Individual links are kept alive by the hard reference in self.__map. # Those hard references disappear when a key is deleted from an OrderedDict. + def __new__(cls, /, *args, **kwds): + "Create the ordered dict object and set up the underlying structures." + self = dict.__new__(cls) + self.__hardroot = _Link() + self.__root = root = _proxy(self.__hardroot) + root.prev = root.next = root + self.__map = {} + return self + def __init__(self, other=(), /, **kwds): '''Initialize an ordered dictionary. The signature is the same as regular dictionaries. Keyword argument order is preserved. ''' - try: - self.__root - except AttributeError: - self.__hardroot = _Link() - self.__root = root = _proxy(self.__hardroot) - root.prev = root.next = root - self.__map = {} self.__update(other, **kwds) def __setitem__(self, key, value, diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index decbcc2419c9f..4571b23dfe7c1 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -122,6 +122,17 @@ def items(self): self.OrderedDict(Spam()) self.assertEqual(calls, ['keys']) + def test_overridden_init(self): + # Sync-up pure Python OD class with C class where + # a consistent internal state is created in __new__ + # rather than __init__. + OrderedDict = self.OrderedDict + class ODNI(OrderedDict): + def __init__(*args, **kwargs): + pass + od = ODNI() + od['a'] = 1 # This used to fail because __init__ was bypassed + def test_fromkeys(self): OrderedDict = self.OrderedDict od = OrderedDict.fromkeys('abc') diff --git a/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst b/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst new file mode 100644 index 0000000000000..f051317d141ff --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst @@ -0,0 +1,3 @@ +Harmonized the pure Python version of OrderedDict with the C version. Now, +both versions set up their internal state in `__new__`. Formerly, the pure +Python version did the set up in `__init__`. From webhook-mailer at python.org Mon Aug 21 08:33:49 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 21 Aug 2023 12:33:49 -0000 Subject: [Python-checkins] [3.12] gh-107916: Save the error code before decoding the filename in PyErr_SetFromErrnoWithFilename() etc (GH-107929) (#108205) Message-ID: https://github.com/python/cpython/commit/238c8d236bca2fab43453d09af69eba2bab1fcb0 commit: 238c8d236bca2fab43453d09af69eba2bab1fcb0 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-21T14:33:46+02:00 summary: [3.12] gh-107916: Save the error code before decoding the filename in PyErr_SetFromErrnoWithFilename() etc (GH-107929) (#108205) gh-107916: Save the error code before decoding the filename in PyErr_SetFromErrnoWithFilename() etc (GH-107929) (cherry picked from commit 80bdebdd8593f007a2232ec04a7729bba6ebf12c) Co-authored-by: Serhiy Storchaka files: A Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst M Python/errors.c diff --git a/Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst b/Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst new file mode 100644 index 0000000000000..f1f16609b118b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-08-14-10-59-03.gh-issue-107916.KH4Muo.rst @@ -0,0 +1,4 @@ +C API functions :c:func:`PyErr_SetFromErrnoWithFilename`, +:c:func:`PyErr_SetExcFromWindowsErrWithFilename` and +:c:func:`PyErr_SetFromWindowsErrWithFilename` save now the error code before +calling :c:func:`PyUnicode_DecodeFSDefault`. diff --git a/Python/errors.c b/Python/errors.c index f1d3007b9fdf8..6c46d1f213665 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -919,10 +919,12 @@ PyErr_SetFromErrnoWithFilename(PyObject *exc, const char *filename) { PyObject *name = NULL; if (filename) { + int i = errno; name = PyUnicode_DecodeFSDefault(filename); if (name == NULL) { return NULL; } + errno = i; } PyObject *result = PyErr_SetFromErrnoWithFilenameObjects(exc, name, NULL); Py_XDECREF(name); @@ -1022,6 +1024,9 @@ PyObject *PyErr_SetExcFromWindowsErrWithFilename( { PyObject *name = NULL; if (filename) { + if ((DWORD)ierr == 0) { + ierr = (int)GetLastError(); + } name = PyUnicode_DecodeFSDefault(filename); if (name == NULL) { return NULL; @@ -1052,6 +1057,9 @@ PyObject *PyErr_SetFromWindowsErrWithFilename( { PyObject *name = NULL; if (filename) { + if ((DWORD)ierr == 0) { + ierr = (int)GetLastError(); + } name = PyUnicode_DecodeFSDefault(filename); if (name == NULL) { return NULL; From webhook-mailer at python.org Mon Aug 21 08:35:22 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 21 Aug 2023 12:35:22 -0000 Subject: [Python-checkins] [3.12] gh-107396: tarfiles: set self.exception before _init_read_gz() (GH-107485) (#108207) Message-ID: https://github.com/python/cpython/commit/e1b069fe06d953500c2c906d26c2a06eddff5523 commit: e1b069fe06d953500c2c906d26c2a06eddff5523 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-21T14:35:18+02:00 summary: [3.12] gh-107396: tarfiles: set self.exception before _init_read_gz() (GH-107485) (#108207) gh-107396: tarfiles: set self.exception before _init_read_gz() (GH-107485) In the stack call of: _init_read_gz() ``` _read, tarfile.py:548 read, tarfile.py:526 _init_read_gz, tarfile.py:491 ``` a try;except exists that uses `self.exception`, so it needs to be set before calling _init_read_gz(). (cherry picked from commit 37135d25e269ede92bc7da363bebfa574782e59a) Co-authored-by: balmeida-nokia <83089745+balmeida-nokia at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst M Lib/tarfile.py M Lib/test/test_tarfile.py diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 7781a430839ea..50212d3f7d879 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -372,8 +372,8 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, self.zlib = zlib self.crc = zlib.crc32(b"") if mode == "r": - self._init_read_gz() self.exception = zlib.error + self._init_read_gz() else: self._init_write_gz(compresslevel) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index e8d322d20a5a8..4077a758e3cf1 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -915,6 +915,23 @@ class LzmaDetectReadTest(LzmaTest, DetectReadTest): pass +class GzipBrokenHeaderCorrectException(GzipTest, unittest.TestCase): + """ + See: https://github.com/python/cpython/issues/107396 + """ + def runTest(self): + f = io.BytesIO( + b'\x1f\x8b' # header + b'\x08' # compression method + b'\x04' # flags + b'\0\0\0\0\0\0' # timestamp, compression data, OS ID + b'\0\x01' # size + b'\0\0\0\0\0' # corrupt data (zeros) + ) + with self.assertRaises(tarfile.ReadError): + tarfile.open(fileobj=f, mode='r|gz') + + class MemberReadTest(ReadTest, unittest.TestCase): def _test_member(self, tarinfo, chksum=None, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst b/Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst new file mode 100644 index 0000000000000..73bff4bdbe024 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst @@ -0,0 +1 @@ +tarfiles; Fixed use before assignment of self.exception for gzip decompression From webhook-mailer at python.org Mon Aug 21 09:13:59 2023 From: webhook-mailer at python.org (hugovk) Date: Mon, 21 Aug 2023 13:13:59 -0000 Subject: [Python-checkins] Run sphinx-lint on Misc/NEWS.d/next/ (#108212) Message-ID: https://github.com/python/cpython/commit/71962e5237bb7e12d8fc1447b8ed8ba082e23f77 commit: 71962e5237bb7e12d8fc1447b8ed8ba082e23f77 branch: main author: Hugo van Kemenade committer: hugovk date: 2023-08-21T13:13:55Z summary: Run sphinx-lint on Misc/NEWS.d/next/ (#108212) Co-authored-by: Alex Waygood files: M .pre-commit-config.yaml M Misc/NEWS.d/next/Build/2023-07-28-18-17-33.gh-issue-106881.U3Ezdq.rst M Misc/NEWS.d/next/Core and Builtins/2023-06-24-10-34-27.gh-issue-105775.OqjoGV.rst M Misc/NEWS.d/next/Core and Builtins/2023-07-04-04-50-14.gh-issue-100288.yNQ1ez.rst M Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-18-04.gh-issue-106078.WEy2Yn.rst M Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-47-29.gh-issue-104432.oGHF-z.rst M Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst M Misc/NEWS.d/next/Library/2023-06-25-12-28-55.gh-issue-106075.W7tMRb.rst M Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst M Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst M Misc/NEWS.d/next/Library/2023-07-16-10-40-34.gh-issue-106789.NvyE3C.rst M Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst M Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst M Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 85a6de4abe014..451cbe8bc8482 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,5 +14,5 @@ repos: hooks: - id: sphinx-lint args: [--enable=default-role] - files: ^Doc/ + files: ^Doc/|^Misc/NEWS.d/next/ types: [rst] diff --git a/Misc/NEWS.d/next/Build/2023-07-28-18-17-33.gh-issue-106881.U3Ezdq.rst b/Misc/NEWS.d/next/Build/2023-07-28-18-17-33.gh-issue-106881.U3Ezdq.rst index 40b2609e95c7e..7febf99c48a79 100644 --- a/Misc/NEWS.d/next/Build/2023-07-28-18-17-33.gh-issue-106881.U3Ezdq.rst +++ b/Misc/NEWS.d/next/Build/2023-07-28-18-17-33.gh-issue-106881.U3Ezdq.rst @@ -1 +1 @@ -Check for `linux/limits.h` before including it in `Modules/posixmodule.c`. +Check for ``linux/limits.h`` before including it in ``Modules/posixmodule.c``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-24-10-34-27.gh-issue-105775.OqjoGV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-24-10-34-27.gh-issue-105775.OqjoGV.rst index 27d0e9929794f..57a30f601379c 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-06-24-10-34-27.gh-issue-105775.OqjoGV.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-24-10-34-27.gh-issue-105775.OqjoGV.rst @@ -1 +1 @@ -:opcode:`LOAD_CLOSURE` is now a pseudo-op. \ No newline at end of file +:opcode:`LOAD_CLOSURE` is now a pseudo-op. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-04-04-50-14.gh-issue-100288.yNQ1ez.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-04-04-50-14.gh-issue-100288.yNQ1ez.rst index faf43bd0c2f48..0d074ffa9942d 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-04-04-50-14.gh-issue-100288.yNQ1ez.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-04-04-50-14.gh-issue-100288.yNQ1ez.rst @@ -1,3 +1,2 @@ Specialize :opcode:`LOAD_ATTR` for non-descriptors on the class. Adds -:opcode:`LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES` and :opcode -`LOAD_ATTR_NONDESCRIPTOR_NO_DICT`. +:opcode:`LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES` and :opcode:`LOAD_ATTR_NONDESCRIPTOR_NO_DICT`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-18-04.gh-issue-106078.WEy2Yn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-18-04.gh-issue-106078.WEy2Yn.rst index f5a0e539e5d05..60d3304a3fab9 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-18-04.gh-issue-106078.WEy2Yn.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-18-04.gh-issue-106078.WEy2Yn.rst @@ -1 +1 @@ -Isolate :mod:`!_decimal` (apply :pep:`687`). Patch by Charlie Zhao. \ No newline at end of file +Isolate :mod:`!_decimal` (apply :pep:`687`). Patch by Charlie Zhao. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-47-29.gh-issue-104432.oGHF-z.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-47-29.gh-issue-104432.oGHF-z.rst index e47927b4e1188..a9ab5cd43f0ff 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-47-29.gh-issue-104432.oGHF-z.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-27-11-47-29.gh-issue-104432.oGHF-z.rst @@ -1,4 +1,4 @@ Fix potential unaligned memory access on C APIs involving returned sequences -of `char *` pointers within the :mod:`grp` and :mod:`socket` modules. These +of ``char *`` pointers within the :mod:`grp` and :mod:`socket` modules. These were revealed using a ``-fsaniziter=alignment`` build on ARM macOS. Patch by Christopher Chavez. diff --git a/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst b/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst index a1a4cf6d63725..94d7cc9deadbb 100644 --- a/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst +++ b/Misc/NEWS.d/next/Library/2020-05-03-00-33-15.bpo-18319.faPTlx.rst @@ -1,2 +1,2 @@ -Ensure `gettext(msg)` retrieve translations even if a plural form exists. In -other words: `gettext(msg) == ngettext(msg, '', 1)`. +Ensure ``gettext(msg)`` retrieve translations even if a plural form exists. In +other words: ``gettext(msg) == ngettext(msg, '', 1)``. diff --git a/Misc/NEWS.d/next/Library/2023-06-25-12-28-55.gh-issue-106075.W7tMRb.rst b/Misc/NEWS.d/next/Library/2023-06-25-12-28-55.gh-issue-106075.W7tMRb.rst index d2687154a5859..0c24ff70334e1 100644 --- a/Misc/NEWS.d/next/Library/2023-06-25-12-28-55.gh-issue-106075.W7tMRb.rst +++ b/Misc/NEWS.d/next/Library/2023-06-25-12-28-55.gh-issue-106075.W7tMRb.rst @@ -1 +1 @@ -Added `asyncio.taskgroups.__all__` to `asyncio.__all__` for export in star imports. +Added ``asyncio.taskgroups.__all__`` to ``asyncio.__all__`` for export in star imports. diff --git a/Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst b/Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst index 23763818d84ba..d86a6bfdabbfb 100644 --- a/Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst +++ b/Misc/NEWS.d/next/Library/2023-06-30-16-42-44.gh-issue-106263.tk-t93.rst @@ -1,2 +1,2 @@ Fix crash when calling ``repr`` with a manually constructed SignalDict object. -Patch by Charlie Zhao. \ No newline at end of file +Patch by Charlie Zhao. diff --git a/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst b/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst index bbc53d76decbc..bcda64153413d 100644 --- a/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst +++ b/Misc/NEWS.d/next/Library/2023-07-14-16-54-13.gh-issue-106752.BT1Yxw.rst @@ -1,4 +1,4 @@ -Fixed several bugs in zipfile.Path, including: in ``Path.match`, Windows +Fixed several bugs in zipfile.Path, including: in :meth:`zipfile.Path.match`, Windows separators are no longer honored (and never were meant to be); Fixed ``name``/``suffix``/``suffixes``/``stem`` operations when no filename is present and the Path is not at the root of the zipfile; Reworked glob for diff --git a/Misc/NEWS.d/next/Library/2023-07-16-10-40-34.gh-issue-106789.NvyE3C.rst b/Misc/NEWS.d/next/Library/2023-07-16-10-40-34.gh-issue-106789.NvyE3C.rst index 532f8059740da..b9cf35adf9552 100644 --- a/Misc/NEWS.d/next/Library/2023-07-16-10-40-34.gh-issue-106789.NvyE3C.rst +++ b/Misc/NEWS.d/next/Library/2023-07-16-10-40-34.gh-issue-106789.NvyE3C.rst @@ -1 +1 @@ -Remove import of :mod:``pprint`` from :mod:``sysconfig``. +Remove import of :mod:`pprint` from :mod:`sysconfig`. diff --git a/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst b/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst index 9d5ba1a2d7ccb..3c29ddaa634ca 100644 --- a/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst +++ b/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst @@ -1,2 +1,2 @@ Shelves opened with :func:`shelve.open` have a much faster :meth:`clear` -method. Patch by James Cave. \ No newline at end of file +method. Patch by James Cave. diff --git a/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst b/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst index cd2a5d0d5324a..deea9af01c4eb 100644 --- a/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst +++ b/Misc/NEWS.d/next/Library/2023-08-07-14-12-07.gh-issue-107715.238r2f.rst @@ -1 +1,2 @@ -Fix `doctest.DocTestFinder.find` in presence of class names with special characters. Patch by Gertjan van Zwieten. +Fix :meth:`doctest.DocTestFinder.find` in presence of class names with special +characters. Patch by Gertjan van Zwieten. diff --git a/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst b/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst index f051317d141ff..1d959a3b22284 100644 --- a/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst +++ b/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst @@ -1,3 +1,3 @@ -Harmonized the pure Python version of OrderedDict with the C version. Now, -both versions set up their internal state in `__new__`. Formerly, the pure -Python version did the set up in `__init__`. +Harmonized the pure Python version of :class:`~collections.OrderedDict` with the C version. Now, +both versions set up their internal state in ``__new__``. Formerly, the pure +Python version did the set up in ``__init__``. From webhook-mailer at python.org Mon Aug 21 09:31:44 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 21 Aug 2023 13:31:44 -0000 Subject: [Python-checkins] [3.12] gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) (#108211) Message-ID: https://github.com/python/cpython/commit/a909ec399d977f5fdfef13c6ad55bf07ae2b8275 commit: a909ec399d977f5fdfef13c6ad55bf07ae2b8275 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-21T15:31:40+02:00 summary: [3.12] gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) (#108211) gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) (cherry picked from commit acbd3f9c5c5f23e95267714e41236140d84fe962) Co-authored-by: Petr Viktorin Co-authored-by: Victor Stinner Co-authored-by: Lum?r 'Frenzy' Balhar files: A Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst M Doc/library/tarfile.rst M Lib/tarfile.py M Lib/test/test_tarfile.py diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 7a5bdaaf065fe..574ea337e71b7 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -735,6 +735,11 @@ A ``TarInfo`` object has the following public data attributes: Name of the target file name, which is only present in :class:`TarInfo` objects of type :const:`LNKTYPE` and :const:`SYMTYPE`. + For symbolic links (``SYMTYPE``), the *linkname* is relative to the directory + that contains the link. + For hard links (``LNKTYPE``), the *linkname* is relative to the root of + the archive. + .. attribute:: TarInfo.uid :type: int diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 50212d3f7d879..02f5e3b66c076 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -742,7 +742,7 @@ def __init__(self, tarinfo): class AbsoluteLinkError(FilterError): def __init__(self, tarinfo): self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a symlink to an absolute path') + super().__init__(f'{tarinfo.name!r} is a link to an absolute path') class LinkOutsideDestinationError(FilterError): def __init__(self, tarinfo, path): @@ -802,7 +802,14 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) - target_path = os.path.realpath(os.path.join(dest_path, member.linkname)) + if member.issym(): + target_path = os.path.join(dest_path, + os.path.dirname(name), + member.linkname) + else: + target_path = os.path.join(dest_path, + member.linkname) + target_path = os.path.realpath(target_path) if os.path.commonpath([target_path, dest_path]) != dest_path: raise LinkOutsideDestinationError(member, target_path) return new_attrs diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 4077a758e3cf1..013c62630bc44 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3331,10 +3331,12 @@ def __exit__(self, *exc): self.bio = None def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, - mode=None, **kwargs): + mode=None, size=None, **kwargs): """Add a member to the test archive. Call within `with`.""" name = str(name) tarinfo = tarfile.TarInfo(name).replace(**kwargs) + if size is not None: + tarinfo.size = size if mode: tarinfo.mode = _filemode_to_int(mode) if symlink_to is not None: @@ -3410,7 +3412,8 @@ def check_context(self, tar, filter): raise self.raised_exception self.assertEqual(self.expected_paths, set()) - def expect_file(self, name, type=None, symlink_to=None, mode=None): + def expect_file(self, name, type=None, symlink_to=None, mode=None, + size=None): """Check a single file. See check_context.""" if self.raised_exception: raise self.raised_exception @@ -3439,6 +3442,8 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None): self.assertTrue(path.is_fifo()) else: raise NotImplementedError(type) + if size is not None: + self.assertEqual(path.stat().st_size, size) for parent in path.parents: self.expected_paths.discard(parent) @@ -3485,8 +3490,15 @@ def test_parent_symlink(self): # Test interplaying symlinks # Inspired by 'dirsymlink2a' in jwilk/traversal-archives with ArchiveMaker() as arc: + + # `current` links to `.` which is both: + # - the destination directory + # - `current` itself arc.add('current', symlink_to='.') + + # effectively points to ./../ arc.add('parent', symlink_to='current/..') + arc.add('parent/evil') if os_helper.can_symlink(): @@ -3528,9 +3540,46 @@ def test_parent_symlink(self): def test_parent_symlink2(self): # Test interplaying symlinks # Inspired by 'dirsymlink2b' in jwilk/traversal-archives + + # Posix and Windows have different pathname resolution: + # either symlink or a '..' component resolve first. + # Let's see which we are on. + if os_helper.can_symlink(): + testpath = os.path.join(TEMPDIR, 'resolution_test') + os.mkdir(testpath) + + # testpath/current links to `.` which is all of: + # - `testpath` + # - `testpath/current` + # - `testpath/current/current` + # - etc. + os.symlink('.', os.path.join(testpath, 'current')) + + # we'll test where `testpath/current/../file` ends up + with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): + pass + + if os.path.exists(os.path.join(testpath, 'file')): + # Windows collapses 'current\..' to '.' first, leaving + # 'testpath\file' + dotdot_resolves_early = True + elif os.path.exists(os.path.join(testpath, '..', 'file')): + # Posix resolves 'current' to '.' first, leaving + # 'testpath/../file' + dotdot_resolves_early = False + else: + raise AssertionError('Could not determine link resolution') + with ArchiveMaker() as arc: + + # `current` links to `.` which is both the destination directory + # and `current` itself arc.add('current', symlink_to='.') + + # `current/parent` is also available as `./parent`, + # and effectively points to `./../` arc.add('current/parent', symlink_to='..') + arc.add('parent/evil') with self.check_context(arc.open(), 'fully_trusted'): @@ -3544,6 +3593,7 @@ def test_parent_symlink2(self): with self.check_context(arc.open(), 'tar'): if os_helper.can_symlink(): + # Fail when extracting a file outside destination self.expect_exception( tarfile.OutsideDestinationError, "'parent/evil' would be extracted to " @@ -3554,10 +3604,24 @@ def test_parent_symlink2(self): self.expect_file('parent/evil') with self.check_context(arc.open(), 'data'): - self.expect_exception( - tarfile.LinkOutsideDestinationError, - """'current/parent' would link to ['"].*['"], """ - + "which is outside the destination") + if os_helper.can_symlink(): + if dotdot_resolves_early: + # Fail when extracting a file outside destination + self.expect_exception( + tarfile.OutsideDestinationError, + "'parent/evil' would be extracted to " + + """['"].*evil['"], which is outside """ + + "the destination") + else: + # Fail as soon as we have a symlink outside the destination + self.expect_exception( + tarfile.LinkOutsideDestinationError, + "'current/parent' would link to " + + """['"].*outerdir['"], which is outside """ + + "the destination") + else: + self.expect_file('current/') + self.expect_file('parent/evil') @symlink_test def test_absolute_symlink(self): @@ -3587,12 +3651,30 @@ def test_absolute_symlink(self): with self.check_context(arc.open(), 'data'): self.expect_exception( tarfile.AbsoluteLinkError, - "'parent' is a symlink to an absolute path") + "'parent' is a link to an absolute path") + + def test_absolute_hardlink(self): + # Test hardlink to an absolute path + # Inspired by 'dirsymlink' in https://github.com/jwilk/traversal-archives + with ArchiveMaker() as arc: + arc.add('parent', hardlink_to=self.outerdir / 'foo') + + with self.check_context(arc.open(), 'fully_trusted'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'tar'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.AbsoluteLinkError, + "'parent' is a link to an absolute path") @symlink_test def test_sly_relative0(self): # Inspired by 'relative0' in jwilk/traversal-archives with ArchiveMaker() as arc: + # points to `../../tmp/moo` arc.add('../moo', symlink_to='..//tmp/moo') try: @@ -3643,6 +3725,56 @@ def test_sly_relative2(self): + """['"].*moo['"], which is outside the """ + "destination") + @symlink_test + def test_deep_symlink(self): + # Test that symlinks and hardlinks inside a directory + # point to the correct file (`target` of size 3). + # If links aren't supported we get a copy of the file. + with ArchiveMaker() as arc: + arc.add('targetdir/target', size=3) + # a hardlink's linkname is relative to the archive + arc.add('linkdir/hardlink', hardlink_to=os.path.join( + 'targetdir', 'target')) + # a symlink's linkname is relative to the link's directory + arc.add('linkdir/symlink', symlink_to=os.path.join( + '..', 'targetdir', 'target')) + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + if os_helper.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='../targetdir/target') + else: + self.expect_file('linkdir/symlink', size=3) + + @symlink_test + def test_chains(self): + # Test chaining of symlinks/hardlinks. + # Symlinks are created before the files they point to. + with ArchiveMaker() as arc: + arc.add('linkdir/symlink', symlink_to='hardlink') + arc.add('symlink2', symlink_to=os.path.join( + 'linkdir', 'hardlink2')) + arc.add('targetdir/target', size=3) + arc.add('linkdir/hardlink', hardlink_to='targetdir/target') + arc.add('linkdir/hardlink2', hardlink_to='linkdir/symlink') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + self.expect_file('linkdir/hardlink2', size=3) + if os_helper.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='hardlink') + self.expect_file('symlink2', size=3, + symlink_to='linkdir/hardlink2') + else: + self.expect_file('linkdir/symlink', size=3) + self.expect_file('symlink2', size=3) + def test_modes(self): # Test how file modes are extracted # (Note that the modes are ignored on platforms without working chmod) diff --git a/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst new file mode 100644 index 0000000000000..32c1fb93f4ab2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst @@ -0,0 +1,3 @@ +:func:`tarfile.data_filter` now takes the location of symlinks into account +when determining their target, so it will no longer reject some valid +tarballs with ``LinkOutsideDestinationError``. From webhook-mailer at python.org Mon Aug 21 09:32:36 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 21 Aug 2023 13:32:36 -0000 Subject: [Python-checkins] [3.12] Run sphinx-lint on Misc/NEWS.d/next/ (GH-108212) (#108213) Message-ID: https://github.com/python/cpython/commit/90a22eae452e2f50b7be9db81003e3e849480215 commit: 90a22eae452e2f50b7be9db81003e3e849480215 branch: 3.12 author: Hugo van Kemenade committer: Yhg1s date: 2023-08-21T15:32:33+02:00 summary: [3.12] Run sphinx-lint on Misc/NEWS.d/next/ (GH-108212) (#108213) Run sphinx-lint on Misc/NEWS.d/next/ files: M .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 85a6de4abe014..451cbe8bc8482 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,5 +14,5 @@ repos: hooks: - id: sphinx-lint args: [--enable=default-role] - files: ^Doc/ + files: ^Doc/|^Misc/NEWS.d/next/ types: [rst] From webhook-mailer at python.org Mon Aug 21 09:52:41 2023 From: webhook-mailer at python.org (JelleZijlstra) Date: Mon, 21 Aug 2023 13:52:41 -0000 Subject: [Python-checkins] gh-107905: Test raising `__value__` for `TypeAliasType` (#107997) Message-ID: https://github.com/python/cpython/commit/13104f3b7412dce9bf7cfd09bf2d6dad1f3cc2ed commit: 13104f3b7412dce9bf7cfd09bf2d6dad1f3cc2ed branch: main author: Nikita Sobolev committer: JelleZijlstra date: 2023-08-21T13:52:37Z summary: gh-107905: Test raising `__value__` for `TypeAliasType` (#107997) files: M Lib/test/test_type_aliases.py diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py index 0ce97f57de686..8f0a998e1f3dc 100644 --- a/Lib/test/test_type_aliases.py +++ b/Lib/test/test_type_aliases.py @@ -168,6 +168,24 @@ def test_recursive_repr(self): self.assertEqual(repr(GenericRecursive[GenericRecursive[int]]), "GenericRecursive[GenericRecursive[int]]") + def test_raising(self): + type MissingName = list[_My_X] + with self.assertRaisesRegex( + NameError, + "cannot access free variable '_My_X' where it is not associated with a value", + ): + MissingName.__value__ + _My_X = int + self.assertEqual(MissingName.__value__, list[int]) + del _My_X + # Cache should still work: + self.assertEqual(MissingName.__value__, list[int]) + + # Explicit exception: + type ExprException = 1 / 0 + with self.assertRaises(ZeroDivisionError): + ExprException.__value__ + class TypeAliasConstructorTest(unittest.TestCase): def test_basic(self): From webhook-mailer at python.org Mon Aug 21 10:00:03 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 21 Aug 2023 14:00:03 -0000 Subject: [Python-checkins] gh-95065, gh-107704: Argument Clinic: support multiple '/ [from ...]' and '* [from ...]' markers (GH-108132) Message-ID: https://github.com/python/cpython/commit/60942cccb18cfd43240c1a1eb5deab7b31fcb81c commit: 60942cccb18cfd43240c1a1eb5deab7b31fcb81c branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-21T13:59:58Z summary: gh-95065, gh-107704: Argument Clinic: support multiple '/ [from ...]' and '* [from ...]' markers (GH-108132) files: M Include/internal/pycore_global_objects_fini_generated.h M Include/internal/pycore_global_strings.h M Include/internal/pycore_runtime_init_generated.h M Include/internal/pycore_unicodeobject_generated.h M Lib/test/test_clinic.py M Modules/_testclinic.c M Modules/clinic/_testclinic_depr.c.h M Tools/clinic/clinic.py diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index ee9010583ff8b..2f930babf6aad 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -923,6 +923,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extend)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extra_tokens)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(f)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(facility)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(false)); @@ -954,6 +955,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(func)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(future)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(g)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(genexpr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get)); @@ -967,6 +969,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(globals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groupindex)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groups)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(h)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(handle)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hash_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(header)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index b081c0e023fa4..5a0cd1a02ba56 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -412,6 +412,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(exp) STRUCT_FOR_ID(extend) STRUCT_FOR_ID(extra_tokens) + STRUCT_FOR_ID(f) STRUCT_FOR_ID(facility) STRUCT_FOR_ID(factory) STRUCT_FOR_ID(false) @@ -443,6 +444,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(fset) STRUCT_FOR_ID(func) STRUCT_FOR_ID(future) + STRUCT_FOR_ID(g) STRUCT_FOR_ID(generation) STRUCT_FOR_ID(genexpr) STRUCT_FOR_ID(get) @@ -456,6 +458,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(globals) STRUCT_FOR_ID(groupindex) STRUCT_FOR_ID(groups) + STRUCT_FOR_ID(h) STRUCT_FOR_ID(handle) STRUCT_FOR_ID(hash_name) STRUCT_FOR_ID(header) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 8c9c7f753d857..8c0fcdb1bdae1 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -918,6 +918,7 @@ extern "C" { INIT_ID(exp), \ INIT_ID(extend), \ INIT_ID(extra_tokens), \ + INIT_ID(f), \ INIT_ID(facility), \ INIT_ID(factory), \ INIT_ID(false), \ @@ -949,6 +950,7 @@ extern "C" { INIT_ID(fset), \ INIT_ID(func), \ INIT_ID(future), \ + INIT_ID(g), \ INIT_ID(generation), \ INIT_ID(genexpr), \ INIT_ID(get), \ @@ -962,6 +964,7 @@ extern "C" { INIT_ID(globals), \ INIT_ID(groupindex), \ INIT_ID(groups), \ + INIT_ID(h), \ INIT_ID(handle), \ INIT_ID(hash_name), \ INIT_ID(header), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 59f40075f9398..841eb78701127 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1077,6 +1077,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(extra_tokens); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(f); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(facility); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1170,6 +1173,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(future); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(g); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(generation); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1209,6 +1215,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(groups); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(h); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(handle); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 934c1e3ffccca..310615083778f 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1751,7 +1751,7 @@ def test_depr_star_must_come_before_star(self): * [from 3.14] Docstring. """ - err = "Function 'bar': '* [from ...]' must come before '*'" + err = "Function 'bar': '* [from ...]' must precede '*'" self.expect_failure(block, err, lineno=4) def test_depr_star_duplicate(self): @@ -1765,7 +1765,7 @@ def test_depr_star_duplicate(self): c: int Docstring. """ - err = "Function 'bar' uses '* [from ...]' more than once." + err = "Function 'bar' uses '* [from 3.14]' more than once." self.expect_failure(block, err, lineno=5) def test_depr_star_duplicate2(self): @@ -1779,7 +1779,7 @@ def test_depr_star_duplicate2(self): c: int Docstring. """ - err = "Function 'bar' uses '* [from ...]' more than once." + err = "Function 'bar': '* [from 3.15]' must precede '* [from 3.14]'" self.expect_failure(block, err, lineno=5) def test_depr_slash_duplicate(self): @@ -1793,7 +1793,7 @@ def test_depr_slash_duplicate(self): c: int Docstring. """ - err = "Function 'bar' uses '/ [from ...]' more than once." + err = "Function 'bar' uses '/ [from 3.14]' more than once." self.expect_failure(block, err, lineno=5) def test_depr_slash_duplicate2(self): @@ -1801,13 +1801,13 @@ def test_depr_slash_duplicate2(self): module foo foo.bar a: int - / [from 3.14] - b: int / [from 3.15] + b: int + / [from 3.14] c: int Docstring. """ - err = "Function 'bar' uses '/ [from ...]' more than once." + err = "Function 'bar': '/ [from 3.14]' must precede '/ [from 3.15]'" self.expect_failure(block, err, lineno=5) def test_single_slash(self): @@ -2724,7 +2724,15 @@ class ClinicFunctionalTest(unittest.TestCase): locals().update((name, getattr(ac_tester, name)) for name in dir(ac_tester) if name.startswith('test_')) - def check_depr_star(self, pnames, fn, *args, name=None, **kwds): + def check_depr(self, regex, fn, /, *args, **kwds): + with self.assertWarnsRegex(DeprecationWarning, regex) as cm: + # Record the line number, so we're sure we've got the correct stack + # level on the deprecation warning. + _, lineno = fn(*args, **kwds), sys._getframe().f_lineno + self.assertEqual(cm.filename, __file__) + self.assertEqual(cm.lineno, lineno) + + def check_depr_star(self, pnames, fn, /, *args, name=None, **kwds): if name is None: name = fn.__qualname__ if isinstance(fn, type): @@ -2734,12 +2742,7 @@ def check_depr_star(self, pnames, fn, *args, name=None, **kwds): fr"{re.escape(name)}\(\) is deprecated. Parameters? {pnames} will " fr"become( a)? keyword-only parameters? in Python 3\.14" ) - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - # Record the line number, so we're sure we've got the correct stack - # level on the deprecation warning. - _, lineno = fn(*args, **kwds), sys._getframe().f_lineno - self.assertEqual(cm.filename, __file__) - self.assertEqual(cm.lineno, lineno) + self.check_depr(regex, fn, *args, **kwds) def check_depr_kwd(self, pnames, fn, *args, name=None, **kwds): if name is None: @@ -2749,15 +2752,10 @@ def check_depr_kwd(self, pnames, fn, *args, name=None, **kwds): pl = 's' if ' ' in pnames else '' regex = ( fr"Passing keyword argument{pl} {pnames} to " - fr"{re.escape(name)}\(\) is deprecated. Corresponding parameter{pl} " + fr"{re.escape(name)}\(\) is deprecated. Parameter{pl} {pnames} " fr"will become positional-only in Python 3\.14." ) - with self.assertWarnsRegex(DeprecationWarning, regex) as cm: - # Record the line number, so we're sure we've got the correct stack - # level on the deprecation warning. - _, lineno = fn(*args, **kwds), sys._getframe().f_lineno - self.assertEqual(cm.filename, __file__) - self.assertEqual(cm.lineno, lineno) + self.check_depr(regex, fn, *args, **kwds) def test_objects_converter(self): with self.assertRaises(TypeError): @@ -3368,6 +3366,24 @@ def test_depr_star_noinline(self): check("a", "b", c="c") self.assertRaises(TypeError, fn, "a", "b", "c", "d") + def test_depr_star_multi(self): + fn = ac_tester.depr_star_multi + self.assertRaises(TypeError, fn, "a") + fn("a", b="b", c="c", d="d", e="e", f="f", g="g", h="h") + errmsg = ( + "Passing more than 1 positional argument to depr_star_multi() is deprecated. " + "Parameter 'b' will become a keyword-only parameter in Python 3.16. " + "Parameters 'c' and 'd' will become keyword-only parameters in Python 3.15. " + "Parameters 'e', 'f' and 'g' will become keyword-only parameters in Python 3.14.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", c="c", d="d", e="e", f="f", g="g", h="h") + check("a", "b", "c", d="d", e="e", f="f", g="g", h="h") + check("a", "b", "c", "d", e="e", f="f", g="g", h="h") + check("a", "b", "c", "d", "e", f="f", g="g", h="h") + check("a", "b", "c", "d", "e", "f", g="g", h="h") + check("a", "b", "c", "d", "e", "f", "g", h="h") + self.assertRaises(TypeError, fn, "a", "b", "c", "d", "e", "f", "g", "h") + def test_depr_kwd_required_1(self): fn = ac_tester.depr_kwd_required_1 fn("a", "b") @@ -3452,6 +3468,44 @@ def test_depr_kwd_noinline(self): self.assertRaises(TypeError, fn, "a", c="c") self.assertRaises(TypeError, fn, a="a", b="b", c="c") + def test_depr_kwd_multi(self): + fn = ac_tester.depr_kwd_multi + fn("a", "b", "c", "d", "e", "f", "g", h="h") + errmsg = ( + "Passing keyword arguments 'b', 'c', 'd', 'e', 'f' and 'g' to depr_kwd_multi() is deprecated. " + "Parameter 'b' will become positional-only in Python 3.14. " + "Parameters 'c' and 'd' will become positional-only in Python 3.15. " + "Parameters 'e', 'f' and 'g' will become positional-only in Python 3.16.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", "c", "d", "e", "f", g="g", h="h") + check("a", "b", "c", "d", "e", f="f", g="g", h="h") + check("a", "b", "c", "d", e="e", f="f", g="g", h="h") + check("a", "b", "c", d="d", e="e", f="f", g="g", h="h") + check("a", "b", c="c", d="d", e="e", f="f", g="g", h="h") + check("a", b="b", c="c", d="d", e="e", f="f", g="g", h="h") + self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g", h="h") + + def test_depr_multi(self): + fn = ac_tester.depr_multi + self.assertRaises(TypeError, fn, "a", "b", "c", "d", "e", "f", "g") + errmsg = ( + "Passing more than 4 positional arguments to depr_multi() is deprecated. " + "Parameter 'e' will become a keyword-only parameter in Python 3.15. " + "Parameter 'f' will become a keyword-only parameter in Python 3.14.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", "c", "d", "e", "f", g="g") + check("a", "b", "c", "d", "e", f="f", g="g") + fn("a", "b", "c", "d", e="e", f="f", g="g") + fn("a", "b", "c", d="d", e="e", f="f", g="g") + errmsg = ( + "Passing keyword arguments 'b' and 'c' to depr_multi() is deprecated. " + "Parameter 'b' will become positional-only in Python 3.14. " + "Parameter 'c' will become positional-only in Python 3.15.") + check = partial(self.check_depr, re.escape(errmsg), fn) + check("a", "b", c="c", d="d", e="e", f="f", g="g") + check("a", b="b", c="c", d="d", e="e", f="f", g="g") + self.assertRaises(TypeError, fn, a="a", b="b", c="c", d="d", e="e", f="f", g="g") + class PermutationTests(unittest.TestCase): """Test permutation support functions.""" diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index efec04d99029b..2e0535d52641e 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1580,6 +1580,32 @@ depr_star_noinline_impl(PyObject *module, PyObject *a, PyObject *b, } +/*[clinic input] +depr_star_multi + a: object + * [from 3.16] + b: object + * [from 3.15] + c: object + d: object + * [from 3.14] + e: object + f: object + g: object + * + h: object +[clinic start generated code]*/ + +static PyObject * +depr_star_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyObject *d, PyObject *e, PyObject *f, PyObject *g, + PyObject *h) +/*[clinic end generated code: output=77681653f4202068 input=3ebd05d888a957ea]*/ +{ + Py_RETURN_NONE; +} + + /*[clinic input] depr_kwd_required_1 a: object @@ -1702,6 +1728,59 @@ depr_kwd_noinline_impl(PyObject *module, PyObject *a, PyObject *b, Py_RETURN_NONE; } + +/*[clinic input] +depr_kwd_multi + a: object + / + b: object + / [from 3.14] + c: object + d: object + / [from 3.15] + e: object + f: object + g: object + / [from 3.16] + h: object +[clinic start generated code]*/ + +static PyObject * +depr_kwd_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyObject *d, PyObject *e, PyObject *f, PyObject *g, + PyObject *h) +/*[clinic end generated code: output=ddfbde80fe1942e1 input=7a074e621c79efd7]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +depr_multi + a: object + / + b: object + / [from 3.14] + c: object + / [from 3.15] + d: object + * [from 3.15] + e: object + * [from 3.14] + f: object + * + g: object +[clinic start generated code]*/ + +static PyObject * +depr_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyObject *d, PyObject *e, PyObject *f, PyObject *g) +/*[clinic end generated code: output=f81c92852ca2d4ee input=5b847c5e44bedd02]*/ +{ + Py_RETURN_NONE; +} + + // Reset PY_VERSION_HEX #undef PY_VERSION_HEX #define PY_VERSION_HEX _SAVED_PY_VERSION @@ -1779,6 +1858,7 @@ static PyMethodDef tester_methods[] = { DEPR_STAR_POS2_LEN2_METHODDEF DEPR_STAR_POS2_LEN2_WITH_KWD_METHODDEF DEPR_STAR_NOINLINE_METHODDEF + DEPR_STAR_MULTI_METHODDEF DEPR_KWD_REQUIRED_1_METHODDEF DEPR_KWD_REQUIRED_2_METHODDEF DEPR_KWD_OPTIONAL_1_METHODDEF @@ -1786,6 +1866,8 @@ static PyMethodDef tester_methods[] = { DEPR_KWD_OPTIONAL_3_METHODDEF DEPR_KWD_REQUIRED_OPTIONAL_METHODDEF DEPR_KWD_NOINLINE_METHODDEF + DEPR_KWD_MULTI_METHODDEF + DEPR_MULTI_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_testclinic_depr.c.h b/Modules/clinic/_testclinic_depr.c.h index 661fdaf2a0718..b8365bd559b4e 100644 --- a/Modules/clinic/_testclinic_depr.c.h +++ b/Modules/clinic/_testclinic_depr.c.h @@ -419,8 +419,7 @@ PyDoc_STRVAR(depr_kwd_new__doc__, "The deprecation message should use the class name instead of __new__.\n" "\n" "Note: Passing keyword argument \'a\' to _testclinic.DeprKwdNew() is\n" -"deprecated. Corresponding parameter will become positional-only in\n" -"Python 3.14.\n" +"deprecated. Parameter \'a\' will become positional-only in Python 3.14.\n" ""); static PyObject * @@ -479,8 +478,8 @@ depr_kwd_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (kwargs && PyDict_GET_SIZE(kwargs) && nargs < 1 && fastargs[0]) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword argument 'a' to _testclinic.DeprKwdNew() is " - "deprecated. Corresponding parameter will become positional-only " - "in Python 3.14.", 1)) + "deprecated. Parameter 'a' will become positional-only in Python " + "3.14.", 1)) { goto exit; } @@ -503,8 +502,7 @@ PyDoc_STRVAR(depr_kwd_init__doc__, "The deprecation message should use the class name instead of __init__.\n" "\n" "Note: Passing keyword argument \'a\' to _testclinic.DeprKwdInit() is\n" -"deprecated. Corresponding parameter will become positional-only in\n" -"Python 3.14.\n" +"deprecated. Parameter \'a\' will become positional-only in Python 3.14.\n" ""); static int @@ -563,8 +561,8 @@ depr_kwd_init(PyObject *self, PyObject *args, PyObject *kwargs) if (kwargs && PyDict_GET_SIZE(kwargs) && nargs < 1 && fastargs[0]) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword argument 'a' to _testclinic.DeprKwdInit() is " - "deprecated. Corresponding parameter will become positional-only " - "in Python 3.14.", 1)) + "deprecated. Parameter 'a' will become positional-only in Python " + "3.14.", 1)) { goto exit; } @@ -641,8 +639,8 @@ depr_kwd_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs) } if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to " - "_testclinic.DeprKwdInitNoInline() is deprecated. Corresponding " - "parameters will become positional-only in Python 3.14.", 1)) + "_testclinic.DeprKwdInitNoInline() is deprecated. Parameters 'b' " + "and 'c' will become positional-only in Python 3.14.", 1)) { goto exit; } @@ -1482,13 +1480,110 @@ depr_star_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py return return_value; } +PyDoc_STRVAR(depr_star_multi__doc__, +"depr_star_multi($module, /, a, b, c, d, e, f, g, *, h)\n" +"--\n" +"\n" +"Note: Passing more than 1 positional argument to depr_star_multi() is\n" +"deprecated. Parameter \'b\' will become a keyword-only parameter in\n" +"Python 3.16. Parameters \'c\' and \'d\' will become keyword-only\n" +"parameters in Python 3.15. Parameters \'e\', \'f\' and \'g\' will become\n" +"keyword-only parameters in Python 3.14.\n" +""); + +#define DEPR_STAR_MULTI_METHODDEF \ + {"depr_star_multi", _PyCFunction_CAST(depr_star_multi), METH_FASTCALL|METH_KEYWORDS, depr_star_multi__doc__}, + +static PyObject * +depr_star_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyObject *d, PyObject *e, PyObject *f, PyObject *g, + PyObject *h); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_star_multi'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_star_multi'.") +# else +# warning "Update the clinic input of 'depr_star_multi'." +# endif +#endif + +static PyObject * +depr_star_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 8 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), &_Py_ID(h), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", "c", "d", "e", "f", "g", "h", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_star_multi", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[8]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + PyObject *e; + PyObject *f; + PyObject *g; + PyObject *h; + + if (nargs > 1 && nargs <= 7) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 1 positional argument to depr_star_multi() is " + "deprecated. Parameter 'b' will become a keyword-only parameter " + "in Python 3.16. Parameters 'c' and 'd' will become keyword-only " + "parameters in Python 3.15. Parameters 'e', 'f' and 'g' will " + "become keyword-only parameters in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 7, 7, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + e = args[4]; + f = args[5]; + g = args[6]; + h = args[7]; + return_value = depr_star_multi_impl(module, a, b, c, d, e, f, g, h); + +exit: + return return_value; +} + PyDoc_STRVAR(depr_kwd_required_1__doc__, "depr_kwd_required_1($module, a, /, b)\n" "--\n" "\n" "Note: Passing keyword argument \'b\' to depr_kwd_required_1() is\n" -"deprecated. Corresponding parameter will become positional-only in\n" -"Python 3.14.\n" +"deprecated. Parameter \'b\' will become positional-only in Python 3.14.\n" ""); #define DEPR_KWD_REQUIRED_1_METHODDEF \ @@ -1548,8 +1643,8 @@ depr_kwd_required_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (nargs < 2) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword argument 'b' to depr_kwd_required_1() is " - "deprecated. Corresponding parameter will become positional-only " - "in Python 3.14.", 1)) + "deprecated. Parameter 'b' will become positional-only in Python " + "3.14.", 1)) { goto exit; } @@ -1567,7 +1662,7 @@ PyDoc_STRVAR(depr_kwd_required_2__doc__, "--\n" "\n" "Note: Passing keyword arguments \'b\' and \'c\' to depr_kwd_required_2()\n" -"is deprecated. Corresponding parameters will become positional-only in\n" +"is deprecated. Parameters \'b\' and \'c\' will become positional-only in\n" "Python 3.14.\n" ""); @@ -1630,7 +1725,7 @@ depr_kwd_required_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (nargs < 3) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to depr_kwd_required_2() " - "is deprecated. Corresponding parameters will become " + "is deprecated. Parameters 'b' and 'c' will become " "positional-only in Python 3.14.", 1)) { goto exit; @@ -1650,8 +1745,7 @@ PyDoc_STRVAR(depr_kwd_optional_1__doc__, "--\n" "\n" "Note: Passing keyword argument \'b\' to depr_kwd_optional_1() is\n" -"deprecated. Corresponding parameter will become positional-only in\n" -"Python 3.14.\n" +"deprecated. Parameter \'b\' will become positional-only in Python 3.14.\n" ""); #define DEPR_KWD_OPTIONAL_1_METHODDEF \ @@ -1712,8 +1806,8 @@ depr_kwd_optional_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (kwnames && PyTuple_GET_SIZE(kwnames) && nargs < 2 && args[1]) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword argument 'b' to depr_kwd_optional_1() is " - "deprecated. Corresponding parameter will become positional-only " - "in Python 3.14.", 1)) + "deprecated. Parameter 'b' will become positional-only in Python " + "3.14.", 1)) { goto exit; } @@ -1735,7 +1829,7 @@ PyDoc_STRVAR(depr_kwd_optional_2__doc__, "--\n" "\n" "Note: Passing keyword arguments \'b\' and \'c\' to depr_kwd_optional_2()\n" -"is deprecated. Corresponding parameters will become positional-only in\n" +"is deprecated. Parameters \'b\' and \'c\' will become positional-only in\n" "Python 3.14.\n" ""); @@ -1799,7 +1893,7 @@ depr_kwd_optional_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2 && args[1]) || (nargs < 3 && args[2]))) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to depr_kwd_optional_2() " - "is deprecated. Corresponding parameters will become " + "is deprecated. Parameters 'b' and 'c' will become " "positional-only in Python 3.14.", 1)) { goto exit; @@ -1828,7 +1922,7 @@ PyDoc_STRVAR(depr_kwd_optional_3__doc__, "--\n" "\n" "Note: Passing keyword arguments \'a\', \'b\' and \'c\' to\n" -"depr_kwd_optional_3() is deprecated. Corresponding parameters will\n" +"depr_kwd_optional_3() is deprecated. Parameters \'a\', \'b\' and \'c\' will\n" "become positional-only in Python 3.14.\n" ""); @@ -1892,8 +1986,8 @@ depr_kwd_optional_3(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 1 && args[0]) || (nargs < 2 && args[1]) || (nargs < 3 && args[2]))) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'a', 'b' and 'c' to " - "depr_kwd_optional_3() is deprecated. Corresponding parameters " - "will become positional-only in Python 3.14.", 1)) + "depr_kwd_optional_3() is deprecated. Parameters 'a', 'b' and 'c'" + " will become positional-only in Python 3.14.", 1)) { goto exit; } @@ -1926,7 +2020,7 @@ PyDoc_STRVAR(depr_kwd_required_optional__doc__, "--\n" "\n" "Note: Passing keyword arguments \'b\' and \'c\' to\n" -"depr_kwd_required_optional() is deprecated. Corresponding parameters\n" +"depr_kwd_required_optional() is deprecated. Parameters \'b\' and \'c\'\n" "will become positional-only in Python 3.14.\n" ""); @@ -1990,8 +2084,8 @@ depr_kwd_required_optional(PyObject *module, PyObject *const *args, Py_ssize_t n if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2) || (nargs < 3 && args[2]))) { if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to " - "depr_kwd_required_optional() is deprecated. Corresponding " - "parameters will become positional-only in Python 3.14.", 1)) + "depr_kwd_required_optional() is deprecated. Parameters 'b' and " + "'c' will become positional-only in Python 3.14.", 1)) { goto exit; } @@ -2014,7 +2108,7 @@ PyDoc_STRVAR(depr_kwd_noinline__doc__, "--\n" "\n" "Note: Passing keyword arguments \'b\' and \'c\' to depr_kwd_noinline() is\n" -"deprecated. Corresponding parameters will become positional-only in\n" +"deprecated. Parameters \'b\' and \'c\' will become positional-only in\n" "Python 3.14.\n" ""); @@ -2081,8 +2175,8 @@ depr_kwd_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO } if (PyErr_WarnEx(PyExc_DeprecationWarning, "Passing keyword arguments 'b' and 'c' to depr_kwd_noinline() is " - "deprecated. Corresponding parameters will become positional-only" - " in Python 3.14.", 1)) + "deprecated. Parameters 'b' and 'c' will become positional-only " + "in Python 3.14.", 1)) { goto exit; } @@ -2092,4 +2186,209 @@ depr_kwd_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO exit: return return_value; } -/*[clinic end generated code: output=fc558c1efdcab076 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(depr_kwd_multi__doc__, +"depr_kwd_multi($module, a, /, b, c, d, e, f, g, h)\n" +"--\n" +"\n" +"Note: Passing keyword arguments \'b\', \'c\', \'d\', \'e\', \'f\' and \'g\' to\n" +"depr_kwd_multi() is deprecated. Parameter \'b\' will become positional-\n" +"only in Python 3.14. Parameters \'c\' and \'d\' will become positional-\n" +"only in Python 3.15. Parameters \'e\', \'f\' and \'g\' will become\n" +"positional-only in Python 3.16.\n" +""); + +#define DEPR_KWD_MULTI_METHODDEF \ + {"depr_kwd_multi", _PyCFunction_CAST(depr_kwd_multi), METH_FASTCALL|METH_KEYWORDS, depr_kwd_multi__doc__}, + +static PyObject * +depr_kwd_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyObject *d, PyObject *e, PyObject *f, PyObject *g, + PyObject *h); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_kwd_multi'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_kwd_multi'.") +# else +# warning "Update the clinic input of 'depr_kwd_multi'." +# endif +#endif + +static PyObject * +depr_kwd_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 7 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), &_Py_ID(h), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", "c", "d", "e", "f", "g", "h", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_kwd_multi", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[8]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + PyObject *e; + PyObject *f; + PyObject *g; + PyObject *h; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 8, 8, 0, argsbuf); + if (!args) { + goto exit; + } + if (nargs < 7) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'b', 'c', 'd', 'e', 'f' and 'g' to " + "depr_kwd_multi() is deprecated. Parameter 'b' will become " + "positional-only in Python 3.14. Parameters 'c' and 'd' will " + "become positional-only in Python 3.15. Parameters 'e', 'f' and " + "'g' will become positional-only in Python 3.16.", 1)) + { + goto exit; + } + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + e = args[4]; + f = args[5]; + g = args[6]; + h = args[7]; + return_value = depr_kwd_multi_impl(module, a, b, c, d, e, f, g, h); + +exit: + return return_value; +} + +PyDoc_STRVAR(depr_multi__doc__, +"depr_multi($module, a, /, b, c, d, e, f, *, g)\n" +"--\n" +"\n" +"Note: Passing keyword arguments \'b\' and \'c\' to depr_multi() is\n" +"deprecated. Parameter \'b\' will become positional-only in Python 3.14.\n" +"Parameter \'c\' will become positional-only in Python 3.15.\n" +"\n" +"\n" +"Note: Passing more than 4 positional arguments to depr_multi() is\n" +"deprecated. Parameter \'e\' will become a keyword-only parameter in\n" +"Python 3.15. Parameter \'f\' will become a keyword-only parameter in\n" +"Python 3.14.\n" +""); + +#define DEPR_MULTI_METHODDEF \ + {"depr_multi", _PyCFunction_CAST(depr_multi), METH_FASTCALL|METH_KEYWORDS, depr_multi__doc__}, + +static PyObject * +depr_multi_impl(PyObject *module, PyObject *a, PyObject *b, PyObject *c, + PyObject *d, PyObject *e, PyObject *f, PyObject *g); + +// Emit compiler warnings when we get to Python 3.14. +#if PY_VERSION_HEX >= 0x030e00C0 +# error "Update the clinic input of 'depr_multi'." +#elif PY_VERSION_HEX >= 0x030e00A0 +# ifdef _MSC_VER +# pragma message ("Update the clinic input of 'depr_multi'.") +# else +# warning "Update the clinic input of 'depr_multi'." +# endif +#endif + +static PyObject * +depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 6 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", "c", "d", "e", "f", "g", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "depr_multi", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[7]; + PyObject *a; + PyObject *b; + PyObject *c; + PyObject *d; + PyObject *e; + PyObject *f; + PyObject *g; + + if (nargs > 4 && nargs <= 6) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing more than 4 positional arguments to depr_multi() is " + "deprecated. Parameter 'e' will become a keyword-only parameter " + "in Python 3.15. Parameter 'f' will become a keyword-only " + "parameter in Python 3.14.", 1)) + { + goto exit; + } + } + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 6, 6, 1, argsbuf); + if (!args) { + goto exit; + } + if (nargs < 3) { + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Passing keyword arguments 'b' and 'c' to depr_multi() is " + "deprecated. Parameter 'b' will become positional-only in Python " + "3.14. Parameter 'c' will become positional-only in Python 3.15.", 1)) + { + goto exit; + } + } + a = args[0]; + b = args[1]; + c = args[2]; + d = args[3]; + e = args[4]; + f = args[5]; + g = args[6]; + return_value = depr_multi_impl(module, a, b, c, d, e, f, g); + +exit: + return return_value; +} +/*[clinic end generated code: output=ee8b1933e4bf4dc4 input=a9049054013a1b77]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index fe84e810760b4..2c6fc4022e380 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -35,6 +35,7 @@ Iterator, Sequence, ) +from operator import attrgetter from types import FunctionType, NoneType from typing import ( TYPE_CHECKING, @@ -922,41 +923,38 @@ def deprecate_positional_use( params: dict[int, Parameter], ) -> str: assert len(params) > 0 - names = [repr(p.name) for p in params.values()] - first_pos, first_param = next(iter(params.items())) - last_pos, last_param = next(reversed(params.items())) - - # Pretty-print list of names. - pstr = pprint_words(names) - - # For now, assume there's only one deprecation level. - assert first_param.deprecated_positional == last_param.deprecated_positional - thenceforth = first_param.deprecated_positional - assert thenceforth is not None - major, minor = thenceforth + first_pos = next(iter(params)) + last_pos = next(reversed(params)) # Format the deprecation message. - if first_pos == 0: - preamble = "Passing positional arguments to " if len(params) == 1: condition = f"nargs == {first_pos+1}" - if first_pos: - preamble = f"Passing {first_pos+1} positional arguments to " - message = preamble + ( - f"{func.fulldisplayname}() is deprecated. Parameter {pstr} will " - f"become a keyword-only parameter in Python {major}.{minor}." - ) + amount = f"{first_pos+1} " if first_pos else "" + pl = "s" else: condition = f"nargs > {first_pos} && nargs <= {last_pos+1}" - if first_pos: - preamble = ( - f"Passing more than {first_pos} positional " - f"argument{'s' if first_pos != 1 else ''} to " + amount = f"more than {first_pos} " if first_pos else "" + pl = "s" if first_pos != 1 else "" + message = ( + f"Passing {amount}positional argument{pl} to " + f"{func.fulldisplayname}() is deprecated." + ) + + for (major, minor), group in itertools.groupby( + params.values(), key=attrgetter("deprecated_positional") + ): + names = [repr(p.name) for p in group] + pstr = pprint_words(names) + if len(names) == 1: + message += ( + f" Parameter {pstr} will become a keyword-only parameter " + f"in Python {major}.{minor}." + ) + else: + message += ( + f" Parameters {pstr} will become keyword-only parameters " + f"in Python {major}.{minor}." ) - message = preamble + ( - f"{func.fulldisplayname}() is deprecated. Parameters {pstr} will " - f"become keyword-only parameters in Python {major}.{minor}." - ) # Append deprecation warning to docstring. docstring = textwrap.fill(f"Note: {message}") @@ -977,19 +975,8 @@ def deprecate_keyword_use( argname_fmt: str | None, ) -> str: assert len(params) > 0 - names = [repr(p.name) for p in params.values()] - first_param = next(iter(params.values())) last_param = next(reversed(params.values())) - # Pretty-print list of names. - pstr = pprint_words(names) - - # For now, assume there's only one deprecation level. - assert first_param.deprecated_keyword == last_param.deprecated_keyword - thenceforth = first_param.deprecated_keyword - assert thenceforth is not None - major, minor = thenceforth - # Format the deprecation message. containscheck = "" conditions = [] @@ -1013,16 +1000,25 @@ def deprecate_keyword_use( condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}" else: condition = f"kwnames && PyTuple_GET_SIZE(kwnames) && {condition}" - if len(params) == 1: - what1 = "argument" - what2 = "parameter" - else: - what1 = "arguments" - what2 = "parameters" + names = [repr(p.name) for p in params.values()] + pstr = pprint_words(names) + pl = 's' if len(params) != 1 else '' message = ( - f"Passing keyword {what1} {pstr} to {func.fulldisplayname}() is deprecated. " - f"Corresponding {what2} will become positional-only in Python {major}.{minor}." + f"Passing keyword argument{pl} {pstr} to " + f"{func.fulldisplayname}() is deprecated." ) + + for (major, minor), group in itertools.groupby( + params.values(), key=attrgetter("deprecated_keyword") + ): + names = [repr(p.name) for p in group] + pstr = pprint_words(names) + pl = 's' if len(names) != 1 else '' + message += ( + f" Parameter{pl} {pstr} will become positional-only " + f"in Python {major}.{minor}." + ) + if containscheck: errcheck = f""" if (PyErr_Occurred()) {{{{ // {containscheck}() above can fail @@ -5528,9 +5524,15 @@ def parse_star(self, function: Function, version: VersionTuple | None) -> None: self.keyword_only = True else: if self.keyword_only: - fail(f"Function {function.name!r}: '* [from ...]' must come before '*'") + fail(f"Function {function.name!r}: '* [from ...]' must precede '*'") if self.deprecated_positional: - fail(f"Function {function.name!r} uses '* [from ...]' more than once.") + if self.deprecated_positional == version: + fail(f"Function {function.name!r} uses '* [from " + f"{version[0]}.{version[1]}]' more than once.") + if self.deprecated_positional < version: + fail(f"Function {function.name!r}: '* [from " + f"{version[0]}.{version[1]}]' must precede '* [from " + f"{self.deprecated_positional[0]}.{self.deprecated_positional[1]}]'") self.deprecated_positional = version def parse_opening_square_bracket(self, function: Function) -> None: @@ -5582,7 +5584,13 @@ def parse_slash(self, function: Function, version: VersionTuple | None) -> None: fail(f"Function {function.name!r} uses '/' more than once.") else: if self.deprecated_keyword: - fail(f"Function {function.name!r} uses '/ [from ...]' more than once.") + if self.deprecated_keyword == version: + fail(f"Function {function.name!r} uses '/ [from " + f"{version[0]}.{version[1]}]' more than once.") + if self.deprecated_keyword > version: + fail(f"Function {function.name!r}: '/ [from " + f"{version[0]}.{version[1]}]' must precede '/ [from " + f"{self.deprecated_keyword[0]}.{self.deprecated_keyword[1]}]'") if self.deprecated_positional: fail(f"Function {function.name!r}: '/ [from ...]' must precede '* [from ...]'") if self.keyword_only: @@ -5613,7 +5621,7 @@ def parse_slash(self, function: Function, version: VersionTuple | None) -> None: if p.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD: if version is None: p.kind = inspect.Parameter.POSITIONAL_ONLY - else: + elif p.deprecated_keyword is None: p.deprecated_keyword = version def state_parameter_docstring_start(self, line: str) -> None: From webhook-mailer at python.org Mon Aug 21 11:17:03 2023 From: webhook-mailer at python.org (encukou) Date: Mon, 21 Aug 2023 15:17:03 -0000 Subject: [Python-checkins] [3.11] gh-107396: tarfiles: set self.exception before _init_read_gz() (GH-107485) (GH-108208) Message-ID: https://github.com/python/cpython/commit/75617ac3f002a8e9d0dedab45f2293d921cd1a25 commit: 75617ac3f002a8e9d0dedab45f2293d921cd1a25 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: encukou date: 2023-08-21T17:16:59+02:00 summary: [3.11] gh-107396: tarfiles: set self.exception before _init_read_gz() (GH-107485) (GH-108208) gh-107396: tarfiles: set self.exception before _init_read_gz() (GH-107485) In the stack call of: _init_read_gz() ``` _read, tarfile.py:548 read, tarfile.py:526 _init_read_gz, tarfile.py:491 ``` a try;except exists that uses `self.exception`, so it needs to be set before calling _init_read_gz(). (cherry picked from commit 37135d25e269ede92bc7da363bebfa574782e59a) Co-authored-by: balmeida-nokia <83089745+balmeida-nokia at users.noreply.github.com> files: A Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst M Lib/tarfile.py M Lib/test/test_tarfile.py diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 130b5e0f45dcd..b7adff6e1723b 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -372,8 +372,8 @@ def __init__(self, name, mode, comptype, fileobj, bufsize): self.zlib = zlib self.crc = zlib.crc32(b"") if mode == "r": - self._init_read_gz() self.exception = zlib.error + self._init_read_gz() else: self._init_write_gz() diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index cdea033ec1244..dc7ff852363cf 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -908,6 +908,23 @@ class LzmaDetectReadTest(LzmaTest, DetectReadTest): pass +class GzipBrokenHeaderCorrectException(GzipTest, unittest.TestCase): + """ + See: https://github.com/python/cpython/issues/107396 + """ + def runTest(self): + f = io.BytesIO( + b'\x1f\x8b' # header + b'\x08' # compression method + b'\x04' # flags + b'\0\0\0\0\0\0' # timestamp, compression data, OS ID + b'\0\x01' # size + b'\0\0\0\0\0' # corrupt data (zeros) + ) + with self.assertRaises(tarfile.ReadError): + tarfile.open(fileobj=f, mode='r|gz') + + class MemberReadTest(ReadTest, unittest.TestCase): def _test_member(self, tarinfo, chksum=None, **kwargs): diff --git a/Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst b/Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst new file mode 100644 index 0000000000000..73bff4bdbe024 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-31-07-36-24.gh-issue-107396.3_Kh6D.rst @@ -0,0 +1 @@ +tarfiles; Fixed use before assignment of self.exception for gzip decompression From webhook-mailer at python.org Mon Aug 21 11:41:38 2023 From: webhook-mailer at python.org (vstinner) Date: Mon, 21 Aug 2023 15:41:38 -0000 Subject: [Python-checkins] docs: fix grammar in isolating-extensions.rst (#108037) Message-ID: https://github.com/python/cpython/commit/47022a079eb9d2a2af781abae3de4a71f80247c2 commit: 47022a079eb9d2a2af781abae3de4a71f80247c2 branch: main author: David Lechner committer: vstinner date: 2023-08-21T15:41:34Z summary: docs: fix grammar in isolating-extensions.rst (#108037) files: M Doc/howto/isolating-extensions.rst diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 2551fbe87b5c2..8f3787f2d2f14 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -64,7 +64,7 @@ Enter Per-Module State Instead of focusing on per-interpreter state, Python's C API is evolving to better support the more granular *per-module* state. -This means that C-level data is be attached to a *module object*. +This means that C-level data should be attached to a *module object*. Each interpreter creates its own module object, keeping the data separate. For testing the isolation, multiple module objects corresponding to a single extension can even be loaded in a single interpreter. From webhook-mailer at python.org Mon Aug 21 12:31:34 2023 From: webhook-mailer at python.org (iritkatriel) Date: Mon, 21 Aug 2023 16:31:34 -0000 Subject: [Python-checkins] gh-108113: Make it possible to create an optimized AST (#108154) Message-ID: https://github.com/python/cpython/commit/10a91d7e98d847b05292eab828ff9ae51308d3ee commit: 10a91d7e98d847b05292eab828ff9ae51308d3ee branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-21T16:31:30Z summary: gh-108113: Make it possible to create an optimized AST (#108154) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-18-18-21-27.gh-issue-108113.1h0poE.rst M Doc/library/ast.rst M Doc/whatsnew/3.13.rst M Include/cpython/compile.h M Lib/ast.py M Lib/test/test_ast.py M Lib/test/test_builtin.py M Parser/asdl_c.py M Python/Python-ast.c M Python/pythonrun.c diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index cd657aedf6d23..2237a07eb9d8a 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2122,10 +2122,12 @@ Async and await Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1) Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, ast.PyCF_ONLY_AST)``. + filename, mode, flags=FLAGS_VALUE, optimize=optimize)``, + where ``FLAGS_VALUE`` is ``ast.PyCF_ONLY_AST`` if ``optimize <= 0`` + and ``ast.PyCF_OPTIMIZED_AST`` otherwise. If ``type_comments=True`` is given, the parser is modified to check and return type comments as specified by :pep:`484` and :pep:`526`. @@ -2171,6 +2173,7 @@ and classes for traversing abstract syntax trees: .. versionchanged:: 3.13 The minimum supported version for feature_version is now (3,7) + The ``optimize`` argument was added. .. function:: unparse(ast_obj) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 47b868bad3192..bfab868d1c5b6 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -85,6 +85,12 @@ Other Language Changes This change will affect tools using docstrings, like :mod:`doctest`. (Contributed by Inada Naoki in :gh:`81283`.) +* The :func:`compile` built-in can now accept a new flag, + ``ast.PyCF_OPTIMIZED_AST``, which is similar to ``ast.PyCF_ONLY_AST`` + except that the returned ``AST`` is optimized according to the value + of the ``optimize`` argument. + (Contributed by Irit Katriel in :gh:`108113`). + New Modules =========== @@ -94,6 +100,14 @@ New Modules Improved Modules ================ +ast +--- + +* :func:`ast.parse` now accepts an optional argument ``optimize`` + which is passed on to the :func:`compile` built-in. This makes it + possible to obtain an optimized ``AST``. + (Contributed by Irit Katriel in :gh:`108113`). + array ----- diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h index e6cd39af2ba73..ae17cef554fa1 100644 --- a/Include/cpython/compile.h +++ b/Include/cpython/compile.h @@ -19,9 +19,10 @@ #define PyCF_TYPE_COMMENTS 0x1000 #define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 #define PyCF_ALLOW_INCOMPLETE_INPUT 0x4000 +#define PyCF_OPTIMIZED_AST (0x8000 | PyCF_ONLY_AST) #define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \ PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT | \ - PyCF_ALLOW_INCOMPLETE_INPUT) + PyCF_ALLOW_INCOMPLETE_INPUT | PyCF_OPTIMIZED_AST) typedef struct { int cf_flags; /* bitmask of CO_xxx flags relevant to future */ diff --git a/Lib/ast.py b/Lib/ast.py index a307f3ecd0617..45b95963f8188 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -32,13 +32,15 @@ def parse(source, filename='', mode='exec', *, - type_comments=False, feature_version=None): + type_comments=False, feature_version=None, optimize=-1): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). Pass type_comments=True to get back type comments where the syntax allows. """ flags = PyCF_ONLY_AST + if optimize > 0: + flags |= PyCF_OPTIMIZED_AST if type_comments: flags |= PyCF_TYPE_COMMENTS if feature_version is None: @@ -50,7 +52,7 @@ def parse(source, filename='', mode='exec', *, feature_version = minor # Else it should be an int giving the minor version for 3.x. return compile(source, filename, mode, flags, - _feature_version=feature_version) + _feature_version=feature_version, optimize=optimize) def literal_eval(node_or_string): diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 5346b39043f0f..f3c7229f0b6c7 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -357,6 +357,34 @@ def test_ast_validation(self): tree = ast.parse(snippet) compile(tree, '', 'exec') + def test_optimization_levels__debug__(self): + cases = [(-1, '__debug__'), (0, '__debug__'), (1, False), (2, False)] + for (optval, expected) in cases: + with self.subTest(optval=optval, expected=expected): + res = ast.parse("__debug__", optimize=optval) + self.assertIsInstance(res.body[0], ast.Expr) + if isinstance(expected, bool): + self.assertIsInstance(res.body[0].value, ast.Constant) + self.assertEqual(res.body[0].value.value, expected) + else: + self.assertIsInstance(res.body[0].value, ast.Name) + self.assertEqual(res.body[0].value.id, expected) + + def test_optimization_levels_const_folding(self): + folded = ('Expr', (1, 0, 1, 5), ('Constant', (1, 0, 1, 5), 3, None)) + not_folded = ('Expr', (1, 0, 1, 5), + ('BinOp', (1, 0, 1, 5), + ('Constant', (1, 0, 1, 1), 1, None), + ('Add',), + ('Constant', (1, 4, 1, 5), 2, None))) + + cases = [(-1, not_folded), (0, not_folded), (1, folded), (2, folded)] + for (optval, expected) in cases: + with self.subTest(optval=optval): + tree = ast.parse("1 + 2", optimize=optval) + res = to_tuple(tree.body[0]) + self.assertEqual(res, expected) + def test_invalid_position_information(self): invalid_linenos = [ (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index f5a5c037f1bf1..ee3ba6ab07bbd 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -369,16 +369,17 @@ def f(): """doc""" (1, False, 'doc', False, False), (2, False, None, False, False)] for optval, *expected in values: + with self.subTest(optval=optval): # test both direct compilation and compilation via AST - codeobjs = [] - codeobjs.append(compile(codestr, "", "exec", optimize=optval)) - tree = ast.parse(codestr) - codeobjs.append(compile(tree, "", "exec", optimize=optval)) - for code in codeobjs: - ns = {} - exec(code, ns) - rv = ns['f']() - self.assertEqual(rv, tuple(expected)) + codeobjs = [] + codeobjs.append(compile(codestr, "", "exec", optimize=optval)) + tree = ast.parse(codestr) + codeobjs.append(compile(tree, "", "exec", optimize=optval)) + for code in codeobjs: + ns = {} + exec(code, ns) + rv = ns['f']() + self.assertEqual(rv, tuple(expected)) def test_compile_top_level_await_no_coro(self): """Make sure top level non-await codes get the correct coroutine flags""" @@ -517,6 +518,28 @@ def test_compile_async_generator(self): exec(co, glob) self.assertEqual(type(glob['ticker']()), AsyncGeneratorType) + def test_compile_ast(self): + args = ("a*(1+2)", "f.py", "exec") + raw = compile(*args, flags = ast.PyCF_ONLY_AST).body[0] + opt = compile(*args, flags = ast.PyCF_OPTIMIZED_AST).body[0] + + for tree in (raw, opt): + self.assertIsInstance(tree.value, ast.BinOp) + self.assertIsInstance(tree.value.op, ast.Mult) + self.assertIsInstance(tree.value.left, ast.Name) + self.assertEqual(tree.value.left.id, 'a') + + raw_right = raw.value.right # expect BinOp(1, '+', 2) + self.assertIsInstance(raw_right, ast.BinOp) + self.assertIsInstance(raw_right.left, ast.Constant) + self.assertEqual(raw_right.left.value, 1) + self.assertIsInstance(raw_right.right, ast.Constant) + self.assertEqual(raw_right.right.value, 2) + + opt_right = opt.value.right # expect Constant(3) + self.assertIsInstance(opt_right, ast.Constant) + self.assertEqual(opt_right.value, 3) + def test_delattr(self): sys.spam = 1 delattr(sys, 'spam') diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-18-18-21-27.gh-issue-108113.1h0poE.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-18-18-21-27.gh-issue-108113.1h0poE.rst new file mode 100644 index 0000000000000..66680578c9b43 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-18-18-21-27.gh-issue-108113.1h0poE.rst @@ -0,0 +1,8 @@ +The :func:`compile` built-in can now accept a new flag, +``ast.PyCF_OPTIMIZED_AST``, which is similar to ``ast.PyCF_ONLY_AST`` +except that the returned ``AST`` is optimized according to the value +of the ``optimize`` argument. + +:func:`ast.parse` now accepts an optional argument ``optimize`` +which is passed on to the :func:`compile` built-in. This makes it +possible to obtain an optimized ``AST``. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 2a36610527f89..1733cd4b15aa4 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1208,6 +1208,9 @@ def visitModule(self, mod): self.emit('if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0) {', 1) self.emit("return -1;", 2) self.emit('}', 1) + self.emit('if (PyModule_AddIntMacro(m, PyCF_OPTIMIZED_AST) < 0) {', 1) + self.emit("return -1;", 2) + self.emit('}', 1) for dfn in mod.dfns: self.visit(dfn) self.emit("return 0;", 1) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 8047b1259c5d8..60dd121d60b8d 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -12659,6 +12659,9 @@ astmodule_exec(PyObject *m) if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0) { return -1; } + if (PyModule_AddIntMacro(m, PyCF_OPTIMIZED_AST) < 0) { + return -1; + } if (PyModule_AddObjectRef(m, "mod", state->mod_type) < 0) { return -1; } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 721c527745c44..b2e04cfa317c0 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -21,6 +21,7 @@ #include "pycore_pyerrors.h" // _PyErr_GetRaisedException, _Py_Offer_Suggestions #include "pycore_pylifecycle.h" // _Py_UnhandledKeyboardInterrupt #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_symtable.h" // _PyFuture_FromAST() #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_traceback.h" // _PyTraceBack_Print_Indented() @@ -1790,6 +1791,24 @@ run_pyc_file(FILE *fp, PyObject *globals, PyObject *locals, return NULL; } +static int +ast_optimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, + int optimize, PyArena *arena) +{ + PyFutureFeatures future; + if (!_PyFuture_FromAST(mod, filename, &future)) { + return -1; + } + int flags = future.ff_features | cf->cf_flags; + if (optimize == -1) { + optimize = _Py_GetConfig()->optimization_level; + } + if (!_PyAST_Optimize(mod, arena, optimize, flags)) { + return -1; + } + return 0; +} + PyObject * Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) @@ -1806,6 +1825,12 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, return NULL; } if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { + if ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_OPTIMIZED_AST) { + if (ast_optimize(mod, filename, flags, optimize, arena) < 0) { + _PyArena_Free(arena); + return NULL; + } + } PyObject *result = PyAST_mod2obj(mod); _PyArena_Free(arena); return result; From webhook-mailer at python.org Mon Aug 21 13:05:19 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 21 Aug 2023 17:05:19 -0000 Subject: [Python-checkins] gh-107298: Fix references to deprecated and removed PyUnicode C API (GH-108077) Message-ID: https://github.com/python/cpython/commit/db55383829ccd5ce80c551d60f26851346741fdf commit: db55383829ccd5ce80c551d60f26851346741fdf branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-21T20:05:15+03:00 summary: gh-107298: Fix references to deprecated and removed PyUnicode C API (GH-108077) files: M Doc/whatsnew/3.11.rst M Doc/whatsnew/3.3.rst M Doc/whatsnew/3.6.rst M Doc/whatsnew/3.9.rst M Misc/NEWS.d/3.11.0b1.rst diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index ec5263ec35c52..fce00d37555c2 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2591,22 +2591,22 @@ Pending Removal in Python 3.12 The following C APIs have been deprecated in earlier Python releases, and will be removed in Python 3.12. -* :c:func:`PyUnicode_AS_DATA` -* :c:func:`PyUnicode_AS_UNICODE` -* :c:func:`PyUnicode_AsUnicodeAndSize` -* :c:func:`PyUnicode_AsUnicode` -* :c:func:`PyUnicode_FromUnicode` -* :c:func:`PyUnicode_GET_DATA_SIZE` -* :c:func:`PyUnicode_GET_SIZE` -* :c:func:`PyUnicode_GetSize` +* :c:func:`!PyUnicode_AS_DATA` +* :c:func:`!PyUnicode_AS_UNICODE` +* :c:func:`!PyUnicode_AsUnicodeAndSize` +* :c:func:`!PyUnicode_AsUnicode` +* :c:func:`!PyUnicode_FromUnicode` +* :c:func:`!PyUnicode_GET_DATA_SIZE` +* :c:func:`!PyUnicode_GET_SIZE` +* :c:func:`!PyUnicode_GetSize` * :c:func:`PyUnicode_IS_COMPACT` * :c:func:`PyUnicode_IS_READY` * :c:func:`PyUnicode_READY` -* :c:func:`Py_UNICODE_WSTR_LENGTH` -* :c:func:`_PyUnicode_AsUnicode` -* :c:macro:`PyUnicode_WCHAR_KIND` +* :c:func:`!PyUnicode_WSTR_LENGTH` +* :c:func:`!_PyUnicode_AsUnicode` +* :c:macro:`!PyUnicode_WCHAR_KIND` * :c:type:`PyUnicodeObject` -* :c:func:`PyUnicode_InternImmortal()` +* :c:func:`!PyUnicode_InternImmortal` .. _whatsnew311-c-api-removed: @@ -2614,7 +2614,7 @@ and will be removed in Python 3.12. Removed ------- -* :c:func:`PyFrame_BlockSetup` and :c:func:`PyFrame_BlockPop` have been +* :c:func:`!PyFrame_BlockSetup` and :c:func:`!PyFrame_BlockPop` have been removed. (Contributed by Mark Shannon in :issue:`40222`.) diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index bcdc0222be6de..7e12c27906567 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -249,7 +249,7 @@ Changes introduced by :pep:`393` are the following: non-BMP code points. * The value of :data:`sys.maxunicode` is now always ``1114111`` (``0x10FFFF`` - in hexadecimal). The :c:func:`PyUnicode_GetMax` function still returns + in hexadecimal). The :c:func:`!PyUnicode_GetMax` function still returns either ``0xFFFF`` or ``0x10FFFF`` for backward compatibility, and it should not be used with the new Unicode API (see :issue:`13054`). @@ -2196,7 +2196,7 @@ Changes to Python's build process and to the C API include: * :c:macro:`PyUnicode_DATA`, :c:macro:`PyUnicode_1BYTE_DATA`, :c:macro:`PyUnicode_2BYTE_DATA`, :c:macro:`PyUnicode_4BYTE_DATA` * :c:macro:`PyUnicode_KIND` with :c:enum:`PyUnicode_Kind` enum: - :c:data:`PyUnicode_WCHAR_KIND`, :c:data:`PyUnicode_1BYTE_KIND`, + :c:data:`!PyUnicode_WCHAR_KIND`, :c:data:`PyUnicode_1BYTE_KIND`, :c:data:`PyUnicode_2BYTE_KIND`, :c:data:`PyUnicode_4BYTE_KIND` * :c:macro:`PyUnicode_READ`, :c:macro:`PyUnicode_READ_CHAR`, :c:macro:`PyUnicode_WRITE` * :c:macro:`PyUnicode_MAX_CHAR_VALUE` @@ -2270,58 +2270,58 @@ removed in Python 4. All functions using this type are deprecated: Unicode functions and methods using :c:type:`Py_UNICODE` and :c:expr:`Py_UNICODE*` types: -* :c:macro:`PyUnicode_FromUnicode`: use :c:func:`PyUnicode_FromWideChar` or +* :c:macro:`!PyUnicode_FromUnicode`: use :c:func:`PyUnicode_FromWideChar` or :c:func:`PyUnicode_FromKindAndData` -* :c:macro:`PyUnicode_AS_UNICODE`, :c:func:`PyUnicode_AsUnicode`, - :c:func:`PyUnicode_AsUnicodeAndSize`: use :c:func:`PyUnicode_AsWideCharString` -* :c:macro:`PyUnicode_AS_DATA`: use :c:macro:`PyUnicode_DATA` with +* :c:macro:`!PyUnicode_AS_UNICODE`, :c:func:`!PyUnicode_AsUnicode`, + :c:func:`!PyUnicode_AsUnicodeAndSize`: use :c:func:`PyUnicode_AsWideCharString` +* :c:macro:`!PyUnicode_AS_DATA`: use :c:macro:`PyUnicode_DATA` with :c:macro:`PyUnicode_READ` and :c:macro:`PyUnicode_WRITE` -* :c:macro:`PyUnicode_GET_SIZE`, :c:func:`PyUnicode_GetSize`: use +* :c:macro:`!PyUnicode_GET_SIZE`, :c:func:`!PyUnicode_GetSize`: use :c:macro:`PyUnicode_GET_LENGTH` or :c:func:`PyUnicode_GetLength` -* :c:macro:`PyUnicode_GET_DATA_SIZE`: use +* :c:macro:`!PyUnicode_GET_DATA_SIZE`: use ``PyUnicode_GET_LENGTH(str) * PyUnicode_KIND(str)`` (only work on ready strings) -* :c:func:`PyUnicode_AsUnicodeCopy`: use :c:func:`PyUnicode_AsUCS4Copy` or +* :c:func:`!PyUnicode_AsUnicodeCopy`: use :c:func:`PyUnicode_AsUCS4Copy` or :c:func:`PyUnicode_AsWideCharString` -* :c:func:`PyUnicode_GetMax` +* :c:func:`!PyUnicode_GetMax` Functions and macros manipulating Py_UNICODE* strings: -* :c:macro:`Py_UNICODE_strlen`: use :c:func:`PyUnicode_GetLength` or +* :c:macro:`!Py_UNICODE_strlen()`: use :c:func:`PyUnicode_GetLength` or :c:macro:`PyUnicode_GET_LENGTH` -* :c:macro:`Py_UNICODE_strcat`: use :c:func:`PyUnicode_CopyCharacters` or +* :c:macro:`!Py_UNICODE_strcat()`: use :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_FromFormat` -* :c:macro:`Py_UNICODE_strcpy`, :c:macro:`Py_UNICODE_strncpy`, - :c:macro:`Py_UNICODE_COPY`: use :c:func:`PyUnicode_CopyCharacters` or +* :c:macro:`!Py_UNICODE_strcpy()`, :c:macro:`!Py_UNICODE_strncpy()`, + :c:macro:`!Py_UNICODE_COPY()`: use :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_Substring` -* :c:macro:`Py_UNICODE_strcmp`: use :c:func:`PyUnicode_Compare` -* :c:macro:`Py_UNICODE_strncmp`: use :c:func:`PyUnicode_Tailmatch` -* :c:macro:`Py_UNICODE_strchr`, :c:macro:`Py_UNICODE_strrchr`: use +* :c:macro:`!Py_UNICODE_strcmp()`: use :c:func:`PyUnicode_Compare` +* :c:macro:`!Py_UNICODE_strncmp()`: use :c:func:`PyUnicode_Tailmatch` +* :c:macro:`!Py_UNICODE_strchr()`, :c:macro:`!Py_UNICODE_strrchr()`: use :c:func:`PyUnicode_FindChar` -* :c:macro:`Py_UNICODE_FILL`: use :c:func:`PyUnicode_Fill` -* :c:macro:`Py_UNICODE_MATCH` +* :c:macro:`!Py_UNICODE_FILL()`: use :c:func:`PyUnicode_Fill` +* :c:macro:`!Py_UNICODE_MATCH` Encoders: -* :c:func:`PyUnicode_Encode`: use :c:func:`PyUnicode_AsEncodedObject` -* :c:func:`PyUnicode_EncodeUTF7` -* :c:func:`PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or +* :c:func:`!PyUnicode_Encode`: use :c:func:`PyUnicode_AsEncodedObject` +* :c:func:`!PyUnicode_EncodeUTF7` +* :c:func:`!PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or :c:func:`PyUnicode_AsUTF8String` -* :c:func:`PyUnicode_EncodeUTF32` -* :c:func:`PyUnicode_EncodeUTF16` -* :c:func:`PyUnicode_EncodeUnicodeEscape` use +* :c:func:`!PyUnicode_EncodeUTF32` +* :c:func:`!PyUnicode_EncodeUTF16` +* :c:func:`!PyUnicode_EncodeUnicodeEscape` use :c:func:`PyUnicode_AsUnicodeEscapeString` -* :c:func:`PyUnicode_EncodeRawUnicodeEscape` use +* :c:func:`!PyUnicode_EncodeRawUnicodeEscape` use :c:func:`PyUnicode_AsRawUnicodeEscapeString` -* :c:func:`PyUnicode_EncodeLatin1`: use :c:func:`PyUnicode_AsLatin1String` -* :c:func:`PyUnicode_EncodeASCII`: use :c:func:`PyUnicode_AsASCIIString` -* :c:func:`PyUnicode_EncodeCharmap` -* :c:func:`PyUnicode_TranslateCharmap` -* :c:func:`PyUnicode_EncodeMBCS`: use :c:func:`PyUnicode_AsMBCSString` or +* :c:func:`!PyUnicode_EncodeLatin1`: use :c:func:`PyUnicode_AsLatin1String` +* :c:func:`!PyUnicode_EncodeASCII`: use :c:func:`PyUnicode_AsASCIIString` +* :c:func:`!PyUnicode_EncodeCharmap` +* :c:func:`!PyUnicode_TranslateCharmap` +* :c:func:`!PyUnicode_EncodeMBCS`: use :c:func:`PyUnicode_AsMBCSString` or :c:func:`PyUnicode_EncodeCodePage` (with ``CP_ACP`` code_page) -* :c:func:`PyUnicode_EncodeDecimal`, - :c:func:`PyUnicode_TransformDecimalToASCII` +* :c:func:`!PyUnicode_EncodeDecimal`, + :c:func:`!PyUnicode_TransformDecimalToASCII` Deprecated features diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 4359a9012dd53..c15d8be651fd1 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2066,9 +2066,9 @@ environment. (Contributed by Brett Cannon in :issue:`25154`.) Deprecated functions and types of the C API ------------------------------------------- -Undocumented functions :c:func:`PyUnicode_AsEncodedObject`, -:c:func:`PyUnicode_AsDecodedObject`, :c:func:`PyUnicode_AsEncodedUnicode` -and :c:func:`PyUnicode_AsDecodedUnicode` are deprecated now. +Undocumented functions :c:func:`!PyUnicode_AsEncodedObject`, +:c:func:`!PyUnicode_AsDecodedObject`, :c:func:`!PyUnicode_AsEncodedUnicode` +and :c:func:`!PyUnicode_AsDecodedUnicode` are deprecated now. Use the :ref:`generic codec based API ` instead. diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 3e3e0ff5c41f4..5891d9a55534f 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -1370,8 +1370,8 @@ Porting to Python 3.9 (Contributed by Victor Stinner in :issue:`40241`.) * The ``Py_UNICODE_COPY``, ``Py_UNICODE_FILL``, ``PyUnicode_WSTR_LENGTH``, - :c:func:`PyUnicode_FromUnicode`, :c:func:`PyUnicode_AsUnicode`, - ``_PyUnicode_AsUnicode``, and :c:func:`PyUnicode_AsUnicodeAndSize` are + :c:func:`!PyUnicode_FromUnicode`, :c:func:`!PyUnicode_AsUnicode`, + ``_PyUnicode_AsUnicode``, and :c:func:`!PyUnicode_AsUnicodeAndSize` are marked as deprecated in C. They have been deprecated by :pep:`393` since Python 3.3. (Contributed by Inada Naoki in :issue:`36346`.) diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index 71efc21cbc4a6..766ada4f8e117 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -2068,9 +2068,9 @@ casts when the Python C API is used in C++. Patch by Victor Stinner. .. nonce: Cx-95G .. section: C API -Mark functions as deprecated by :pep:`623`: :c:func:`PyUnicode_AS_DATA`, -:c:func:`PyUnicode_AS_UNICODE`, :c:func:`PyUnicode_GET_DATA_SIZE`, -:c:func:`PyUnicode_GET_SIZE`. Patch by Victor Stinner. +Mark functions as deprecated by :pep:`623`: :c:func:`!PyUnicode_AS_DATA`, +:c:func:`!PyUnicode_AS_UNICODE`, :c:func:`!PyUnicode_GET_DATA_SIZE`, +:c:func:`!PyUnicode_GET_SIZE`. Patch by Victor Stinner. .. From webhook-mailer at python.org Mon Aug 21 13:15:56 2023 From: webhook-mailer at python.org (vstinner) Date: Mon, 21 Aug 2023 17:15:56 -0000 Subject: [Python-checkins] gh-108220: Internal header files require Py_BUILD_CORE to be defined (#108221) Message-ID: https://github.com/python/cpython/commit/21c0844742cf15db8e56e8848ecbb2e25f314aed commit: 21c0844742cf15db8e56e8848ecbb2e25f314aed branch: main author: Victor Stinner committer: vstinner date: 2023-08-21T19:15:52+02:00 summary: gh-108220: Internal header files require Py_BUILD_CORE to be defined (#108221) * pycore_intrinsics.h does nothing if included twice (add #ifndef and #define). * Update Tools/cases_generator/generate_cases.py to generate the Py_BUILD_CORE test. * _bz2, _lzma, _opcode and zlib extensions now define the Py_BUILD_CORE_MODULE macro to use internal headers (pycore_code.h, pycore_intrinsics.h and pycore_blocks_output_buffer.h). files: M Include/internal/pycore_blocks_output_buffer.h M Include/internal/pycore_code.h M Include/internal/pycore_codecs.h M Include/internal/pycore_emscripten_signal.h M Include/internal/pycore_fileutils.h M Include/internal/pycore_fileutils_windows.h M Include/internal/pycore_frame.h M Include/internal/pycore_import.h M Include/internal/pycore_instruments.h M Include/internal/pycore_intrinsics.h M Include/internal/pycore_opcode_metadata.h M Modules/_bz2module.c M Modules/_lzmamodule.c M Modules/_opcode.c M Modules/zlibmodule.c M Tools/cases_generator/generate_cases.py diff --git a/Include/internal/pycore_blocks_output_buffer.h b/Include/internal/pycore_blocks_output_buffer.h index 28cf6fba4eeba..573e10359b7bd 100644 --- a/Include/internal/pycore_blocks_output_buffer.h +++ b/Include/internal/pycore_blocks_output_buffer.h @@ -40,6 +40,10 @@ extern "C" { #include "Python.h" +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + typedef struct { // List of bytes objects PyObject *list; @@ -314,4 +318,4 @@ _BlocksOutputBuffer_OnError(_BlocksOutputBuffer *buffer) #ifdef __cplusplus } #endif -#endif /* Py_INTERNAL_BLOCKS_OUTPUT_BUFFER_H */ \ No newline at end of file +#endif /* Py_INTERNAL_BLOCKS_OUTPUT_BUFFER_H */ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 00099376635e9..f5127a8114435 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -4,6 +4,10 @@ extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + #define CODE_MAX_WATCHERS 8 /* PEP 659 diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h index a2465192eacd5..a2a7151d50ade 100644 --- a/Include/internal/pycore_codecs.h +++ b/Include/internal/pycore_codecs.h @@ -4,6 +4,10 @@ extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + extern PyObject* _PyCodec_Lookup(const char *encoding); /* Text codec specific encoding and decoding API. diff --git a/Include/internal/pycore_emscripten_signal.h b/Include/internal/pycore_emscripten_signal.h index 8b3287d85da4b..d1bcb9a92c772 100644 --- a/Include/internal/pycore_emscripten_signal.h +++ b/Include/internal/pycore_emscripten_signal.h @@ -3,6 +3,10 @@ #if defined(__EMSCRIPTEN__) +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + void _Py_CheckEmscriptenSignals(void); diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 0ed139f79b142..25b383b9780ad 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -5,7 +5,7 @@ extern "C" { #endif #ifndef Py_BUILD_CORE -# error "Py_BUILD_CORE must be defined to include this header" +# error "this header requires Py_BUILD_CORE define" #endif #include /* struct lconv */ diff --git a/Include/internal/pycore_fileutils_windows.h b/Include/internal/pycore_fileutils_windows.h index e804d385e7670..b79aa9fb46537 100644 --- a/Include/internal/pycore_fileutils_windows.h +++ b/Include/internal/pycore_fileutils_windows.h @@ -5,7 +5,7 @@ extern "C" { #endif #ifndef Py_BUILD_CORE -# error "Py_BUILD_CORE must be defined to include this header" +# error "this header requires Py_BUILD_CORE define" #endif #ifdef MS_WINDOWS diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 0dc2a1814cb1d..ae77367f6a3c9 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -4,6 +4,10 @@ extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + #include #include // offsetof() #include "pycore_code.h" // STATS diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 077508e6c58f6..34f572bd59296 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -5,6 +5,10 @@ extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + #include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_time.h" // _PyTime_t diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 56de9f8717148..e15447acec2c0 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -1,7 +1,9 @@ - #ifndef Py_INTERNAL_INSTRUMENT_H #define Py_INTERNAL_INSTRUMENT_H +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif #include "pycore_bitutils.h" // _Py_popcount32 #include "pycore_frame.h" diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 37d4efc12bb77..3a8dd95cff8e5 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -1,3 +1,9 @@ +#ifndef Py_INTERNAL_INTRINSIC_H +#define Py_INTERNAL_INTRINSIC_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif /* Unary Functions: */ #define INTRINSIC_1_INVALID 0 @@ -40,3 +46,5 @@ typedef struct { extern const intrinsic_func1_info _PyIntrinsics_UnaryFunctions[]; extern const intrinsic_func2_info _PyIntrinsics_BinaryFunctions[]; + +#endif // !Py_INTERNAL_INTRINSIC_H diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 396d194ed2734..fab91e611b617 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -3,6 +3,10 @@ // Python/bytecodes.c // Do not edit! +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + #include diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 0a84f25ca4cbe..3d0d4ee5e79c2 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -1,5 +1,9 @@ /* _bz2 - Low-level Python interface to libbzip2. */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #include diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index c548f8fa3839e..eb90c308d16d1 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -5,6 +5,10 @@ */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 4f85e7ee26de2..3e13dbb6edc26 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -1,3 +1,7 @@ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #include "compile.h" #include "opcode.h" diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index a98a37adadcff..9b76afa0e56f7 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -3,6 +3,10 @@ /* Windows users: read Python's PCbuild\readme.txt */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #include "zlib.h" diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 7b1880b98a8fe..c3b729df44a6a 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -9,6 +9,7 @@ import os import posixpath import sys +import textwrap import typing from collections.abc import Iterator @@ -403,6 +404,12 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No self.write_provenance_header() + self.out.emit("\n" + textwrap.dedent(""" + #ifndef Py_BUILD_CORE + # error "this header requires Py_BUILD_CORE define" + #endif + """).strip()) + self.out.emit("\n#include ") self.write_pseudo_instrs() From webhook-mailer at python.org Mon Aug 21 13:35:40 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 21 Aug 2023 17:35:40 -0000 Subject: [Python-checkins] [3.11] gh-107298: Fix references to deprecated and removed PyUnicode C API (GH-108077) (GH-108226) Message-ID: https://github.com/python/cpython/commit/145d9252b70dff1afd798a1cde7bb5bdf2b103b0 commit: 145d9252b70dff1afd798a1cde7bb5bdf2b103b0 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: serhiy-storchaka date: 2023-08-21T20:35:36+03:00 summary: [3.11] gh-107298: Fix references to deprecated and removed PyUnicode C API (GH-108077) (GH-108226) (cherry picked from commit db55383829ccd5ce80c551d60f26851346741fdf) Co-authored-by: Serhiy Storchaka files: M Doc/whatsnew/3.11.rst M Doc/whatsnew/3.3.rst M Doc/whatsnew/3.6.rst M Doc/whatsnew/3.9.rst M Misc/NEWS.d/3.11.0b1.rst diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index f102aa599069c..9054452fe12d5 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2591,22 +2591,22 @@ Pending Removal in Python 3.12 The following C APIs have been deprecated in earlier Python releases, and will be removed in Python 3.12. -* :c:func:`PyUnicode_AS_DATA` -* :c:func:`PyUnicode_AS_UNICODE` -* :c:func:`PyUnicode_AsUnicodeAndSize` -* :c:func:`PyUnicode_AsUnicode` -* :c:func:`PyUnicode_FromUnicode` -* :c:func:`PyUnicode_GET_DATA_SIZE` -* :c:func:`PyUnicode_GET_SIZE` -* :c:func:`PyUnicode_GetSize` +* :c:func:`!PyUnicode_AS_DATA` +* :c:func:`!PyUnicode_AS_UNICODE` +* :c:func:`!PyUnicode_AsUnicodeAndSize` +* :c:func:`!PyUnicode_AsUnicode` +* :c:func:`!PyUnicode_FromUnicode` +* :c:func:`!PyUnicode_GET_DATA_SIZE` +* :c:func:`!PyUnicode_GET_SIZE` +* :c:func:`!PyUnicode_GetSize` * :c:func:`PyUnicode_IS_COMPACT` * :c:func:`PyUnicode_IS_READY` * :c:func:`PyUnicode_READY` -* :c:func:`Py_UNICODE_WSTR_LENGTH` -* :c:func:`_PyUnicode_AsUnicode` -* :c:macro:`PyUnicode_WCHAR_KIND` +* :c:func:`!PyUnicode_WSTR_LENGTH` +* :c:func:`!_PyUnicode_AsUnicode` +* :c:macro:`!PyUnicode_WCHAR_KIND` * :c:type:`PyUnicodeObject` -* :c:func:`PyUnicode_InternImmortal()` +* :c:func:`!PyUnicode_InternImmortal` .. _whatsnew311-c-api-removed: @@ -2614,7 +2614,7 @@ and will be removed in Python 3.12. Removed ------- -* :c:func:`PyFrame_BlockSetup` and :c:func:`PyFrame_BlockPop` have been +* :c:func:`!PyFrame_BlockSetup` and :c:func:`!PyFrame_BlockPop` have been removed. (Contributed by Mark Shannon in :issue:`40222`.) diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 9fb06c1d1f807..91ffc99325a01 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -249,7 +249,7 @@ Changes introduced by :pep:`393` are the following: non-BMP code points. * The value of :data:`sys.maxunicode` is now always ``1114111`` (``0x10FFFF`` - in hexadecimal). The :c:func:`PyUnicode_GetMax` function still returns + in hexadecimal). The :c:func:`!PyUnicode_GetMax` function still returns either ``0xFFFF`` or ``0x10FFFF`` for backward compatibility, and it should not be used with the new Unicode API (see :issue:`13054`). @@ -2195,7 +2195,7 @@ Changes to Python's build process and to the C API include: * :c:macro:`PyUnicode_DATA`, :c:macro:`PyUnicode_1BYTE_DATA`, :c:macro:`PyUnicode_2BYTE_DATA`, :c:macro:`PyUnicode_4BYTE_DATA` * :c:macro:`PyUnicode_KIND` with :c:enum:`PyUnicode_Kind` enum: - :c:data:`PyUnicode_WCHAR_KIND`, :c:data:`PyUnicode_1BYTE_KIND`, + :c:data:`!PyUnicode_WCHAR_KIND`, :c:data:`PyUnicode_1BYTE_KIND`, :c:data:`PyUnicode_2BYTE_KIND`, :c:data:`PyUnicode_4BYTE_KIND` * :c:macro:`PyUnicode_READ`, :c:macro:`PyUnicode_READ_CHAR`, :c:macro:`PyUnicode_WRITE` * :c:macro:`PyUnicode_MAX_CHAR_VALUE` @@ -2269,58 +2269,58 @@ removed in Python 4. All functions using this type are deprecated: Unicode functions and methods using :c:type:`Py_UNICODE` and :c:expr:`Py_UNICODE*` types: -* :c:macro:`PyUnicode_FromUnicode`: use :c:func:`PyUnicode_FromWideChar` or +* :c:macro:`!PyUnicode_FromUnicode`: use :c:func:`PyUnicode_FromWideChar` or :c:func:`PyUnicode_FromKindAndData` -* :c:macro:`PyUnicode_AS_UNICODE`, :c:func:`PyUnicode_AsUnicode`, - :c:func:`PyUnicode_AsUnicodeAndSize`: use :c:func:`PyUnicode_AsWideCharString` -* :c:macro:`PyUnicode_AS_DATA`: use :c:macro:`PyUnicode_DATA` with +* :c:macro:`!PyUnicode_AS_UNICODE`, :c:func:`!PyUnicode_AsUnicode`, + :c:func:`!PyUnicode_AsUnicodeAndSize`: use :c:func:`PyUnicode_AsWideCharString` +* :c:macro:`!PyUnicode_AS_DATA`: use :c:macro:`PyUnicode_DATA` with :c:macro:`PyUnicode_READ` and :c:macro:`PyUnicode_WRITE` -* :c:macro:`PyUnicode_GET_SIZE`, :c:func:`PyUnicode_GetSize`: use +* :c:macro:`!PyUnicode_GET_SIZE`, :c:func:`!PyUnicode_GetSize`: use :c:macro:`PyUnicode_GET_LENGTH` or :c:func:`PyUnicode_GetLength` -* :c:macro:`PyUnicode_GET_DATA_SIZE`: use +* :c:macro:`!PyUnicode_GET_DATA_SIZE`: use ``PyUnicode_GET_LENGTH(str) * PyUnicode_KIND(str)`` (only work on ready strings) -* :c:func:`PyUnicode_AsUnicodeCopy`: use :c:func:`PyUnicode_AsUCS4Copy` or +* :c:func:`!PyUnicode_AsUnicodeCopy`: use :c:func:`PyUnicode_AsUCS4Copy` or :c:func:`PyUnicode_AsWideCharString` -* :c:func:`PyUnicode_GetMax` +* :c:func:`!PyUnicode_GetMax` Functions and macros manipulating Py_UNICODE* strings: -* :c:macro:`Py_UNICODE_strlen`: use :c:func:`PyUnicode_GetLength` or +* :c:macro:`!Py_UNICODE_strlen()`: use :c:func:`PyUnicode_GetLength` or :c:macro:`PyUnicode_GET_LENGTH` -* :c:macro:`Py_UNICODE_strcat`: use :c:func:`PyUnicode_CopyCharacters` or +* :c:macro:`!Py_UNICODE_strcat()`: use :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_FromFormat` -* :c:macro:`Py_UNICODE_strcpy`, :c:macro:`Py_UNICODE_strncpy`, - :c:macro:`Py_UNICODE_COPY`: use :c:func:`PyUnicode_CopyCharacters` or +* :c:macro:`!Py_UNICODE_strcpy()`, :c:macro:`!Py_UNICODE_strncpy()`, + :c:macro:`!Py_UNICODE_COPY()`: use :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_Substring` -* :c:macro:`Py_UNICODE_strcmp`: use :c:func:`PyUnicode_Compare` -* :c:macro:`Py_UNICODE_strncmp`: use :c:func:`PyUnicode_Tailmatch` -* :c:macro:`Py_UNICODE_strchr`, :c:macro:`Py_UNICODE_strrchr`: use +* :c:macro:`!Py_UNICODE_strcmp()`: use :c:func:`PyUnicode_Compare` +* :c:macro:`!Py_UNICODE_strncmp()`: use :c:func:`PyUnicode_Tailmatch` +* :c:macro:`!Py_UNICODE_strchr()`, :c:macro:`!Py_UNICODE_strrchr()`: use :c:func:`PyUnicode_FindChar` -* :c:macro:`Py_UNICODE_FILL`: use :c:func:`PyUnicode_Fill` -* :c:macro:`Py_UNICODE_MATCH` +* :c:macro:`!Py_UNICODE_FILL()`: use :c:func:`PyUnicode_Fill` +* :c:macro:`!Py_UNICODE_MATCH` Encoders: -* :c:func:`PyUnicode_Encode`: use :c:func:`PyUnicode_AsEncodedObject` -* :c:func:`PyUnicode_EncodeUTF7` -* :c:func:`PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or +* :c:func:`!PyUnicode_Encode`: use :c:func:`PyUnicode_AsEncodedObject` +* :c:func:`!PyUnicode_EncodeUTF7` +* :c:func:`!PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or :c:func:`PyUnicode_AsUTF8String` -* :c:func:`PyUnicode_EncodeUTF32` -* :c:func:`PyUnicode_EncodeUTF16` -* :c:func:`PyUnicode_EncodeUnicodeEscape` use +* :c:func:`!PyUnicode_EncodeUTF32` +* :c:func:`!PyUnicode_EncodeUTF16` +* :c:func:`!PyUnicode_EncodeUnicodeEscape` use :c:func:`PyUnicode_AsUnicodeEscapeString` -* :c:func:`PyUnicode_EncodeRawUnicodeEscape` use +* :c:func:`!PyUnicode_EncodeRawUnicodeEscape` use :c:func:`PyUnicode_AsRawUnicodeEscapeString` -* :c:func:`PyUnicode_EncodeLatin1`: use :c:func:`PyUnicode_AsLatin1String` -* :c:func:`PyUnicode_EncodeASCII`: use :c:func:`PyUnicode_AsASCIIString` -* :c:func:`PyUnicode_EncodeCharmap` -* :c:func:`PyUnicode_TranslateCharmap` -* :c:func:`PyUnicode_EncodeMBCS`: use :c:func:`PyUnicode_AsMBCSString` or +* :c:func:`!PyUnicode_EncodeLatin1`: use :c:func:`PyUnicode_AsLatin1String` +* :c:func:`!PyUnicode_EncodeASCII`: use :c:func:`PyUnicode_AsASCIIString` +* :c:func:`!PyUnicode_EncodeCharmap` +* :c:func:`!PyUnicode_TranslateCharmap` +* :c:func:`!PyUnicode_EncodeMBCS`: use :c:func:`PyUnicode_AsMBCSString` or :c:func:`PyUnicode_EncodeCodePage` (with ``CP_ACP`` code_page) -* :c:func:`PyUnicode_EncodeDecimal`, - :c:func:`PyUnicode_TransformDecimalToASCII` +* :c:func:`!PyUnicode_EncodeDecimal`, + :c:func:`!PyUnicode_TransformDecimalToASCII` Deprecated features diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 1b7cef14d912e..e6e813e49ceb8 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2066,9 +2066,9 @@ environment. (Contributed by Brett Cannon in :issue:`25154`.) Deprecated functions and types of the C API ------------------------------------------- -Undocumented functions :c:func:`PyUnicode_AsEncodedObject`, -:c:func:`PyUnicode_AsDecodedObject`, :c:func:`PyUnicode_AsEncodedUnicode` -and :c:func:`PyUnicode_AsDecodedUnicode` are deprecated now. +Undocumented functions :c:func:`!PyUnicode_AsEncodedObject`, +:c:func:`!PyUnicode_AsDecodedObject`, :c:func:`!PyUnicode_AsEncodedUnicode` +and :c:func:`!PyUnicode_AsDecodedUnicode` are deprecated now. Use the :ref:`generic codec based API ` instead. diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 666d53ed6da7b..664c8d8a545db 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -1370,8 +1370,8 @@ Porting to Python 3.9 (Contributed by Victor Stinner in :issue:`40241`.) * The ``Py_UNICODE_COPY``, ``Py_UNICODE_FILL``, ``PyUnicode_WSTR_LENGTH``, - :c:func:`PyUnicode_FromUnicode`, :c:func:`PyUnicode_AsUnicode`, - ``_PyUnicode_AsUnicode``, and :c:func:`PyUnicode_AsUnicodeAndSize` are + :c:func:`!PyUnicode_FromUnicode`, :c:func:`!PyUnicode_AsUnicode`, + ``_PyUnicode_AsUnicode``, and :c:func:`!PyUnicode_AsUnicodeAndSize` are marked as deprecated in C. They have been deprecated by :pep:`393` since Python 3.3. (Contributed by Inada Naoki in :issue:`36346`.) diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index 6b601489a7728..1da722b21680e 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -2068,9 +2068,9 @@ casts when the Python C API is used in C++. Patch by Victor Stinner. .. nonce: Cx-95G .. section: C API -Mark functions as deprecated by :pep:`623`: :c:func:`PyUnicode_AS_DATA`, -:c:func:`PyUnicode_AS_UNICODE`, :c:func:`PyUnicode_GET_DATA_SIZE`, -:c:func:`PyUnicode_GET_SIZE`. Patch by Victor Stinner. +Mark functions as deprecated by :pep:`623`: :c:func:`!PyUnicode_AS_DATA`, +:c:func:`!PyUnicode_AS_UNICODE`, :c:func:`!PyUnicode_GET_DATA_SIZE`, +:c:func:`!PyUnicode_GET_SIZE`. Patch by Victor Stinner. .. From webhook-mailer at python.org Mon Aug 21 13:50:38 2023 From: webhook-mailer at python.org (corona10) Date: Mon, 21 Aug 2023 17:50:38 -0000 Subject: [Python-checkins] gh-108223: Add --disable-gil to configure (gh-108227) Message-ID: https://github.com/python/cpython/commit/b16ecb88e70d696a93ce993661973330baeafee1 commit: b16ecb88e70d696a93ce993661973330baeafee1 branch: main author: Sam Gross committer: corona10 date: 2023-08-21T17:50:35Z summary: gh-108223: Add --disable-gil to configure (gh-108227) The `--disable-gil` flags does not do anything yet other than define the Py_NOGIL macro. This is intended to support setting up additional buildbots. files: M configure M configure.ac M pyconfig.h.in diff --git a/configure b/configure index aaacf8d2669c1..f78b45a2c72dd 100755 --- a/configure +++ b/configure @@ -1103,6 +1103,7 @@ with_openssl_rpath with_ssl_default_suites with_builtin_hashlib_hashes enable_test_modules +enable_gil ' ac_precious_vars='build_alias host_alias @@ -1803,6 +1804,8 @@ Optional Features: use big digits (30 or 15 bits) for Python longs (default is 30)] --disable-test-modules don't build nor install test modules + --disable-gil enable experimental support for running without the + GIL (default is no) Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] @@ -27904,6 +27907,34 @@ fi printf "%s\n" "$TEST_MODULES" >&6; } +# Check for --disable-gil +# --disable-gil +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --disable-gil" >&5 +printf %s "checking for --disable-gil... " >&6; } +# Check whether --enable-gil was given. +if test ${enable_gil+y} +then : + enableval=$enable_gil; if test "x$enable_gil" = xyes +then : + disable_gil=no +else $as_nop + disable_gil=yes +fi +else $as_nop + disable_gil=no + +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $disable_gil" >&5 +printf "%s\n" "$disable_gil" >&6; } + +if test "$disable_gil" = "yes" +then + +printf "%s\n" "#define Py_NOGIL 1" >>confdefs.h + +fi + # stdlib not available diff --git a/configure.ac b/configure.ac index ddf6da0b9da12..fd18a452f2b61 100644 --- a/configure.ac +++ b/configure.ac @@ -7126,6 +7126,21 @@ AC_ARG_ENABLE([test-modules], AC_MSG_RESULT([$TEST_MODULES]) AC_SUBST([TEST_MODULES]) +# Check for --disable-gil +# --disable-gil +AC_MSG_CHECKING([for --disable-gil]) +AC_ARG_ENABLE([gil], + [AS_HELP_STRING([--disable-gil], [enable experimental support for running without the GIL (default is no)])], + [AS_VAR_IF([enable_gil], [yes], [disable_gil=no], [disable_gil=yes])], [disable_gil=no] +) +AC_MSG_RESULT([$disable_gil]) + +if test "$disable_gil" = "yes" +then + AC_DEFINE([Py_NOGIL], [1], + [Define if you want to disable the GIL]) +fi + AC_DEFUN([PY_STDLIB_MOD_SET_NA], [ m4_foreach([mod], [$@], [ AS_VAR_SET([py_cv_module_]mod, [n/a])]) diff --git a/pyconfig.h.in b/pyconfig.h.in index 181dc3d7d1137..418ccade8e8bb 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1609,6 +1609,9 @@ SipHash13: 3, externally defined: 0 */ #undef Py_HASH_ALGORITHM +/* Define if you want to disable the GIL */ +#undef Py_NOGIL + /* Define if you want to enable internal statistics gathering. */ #undef Py_STATS From webhook-mailer at python.org Mon Aug 21 14:06:03 2023 From: webhook-mailer at python.org (vstinner) Date: Mon, 21 Aug 2023 18:06:03 -0000 Subject: [Python-checkins] gh-108216: Cleanup #include in internal header files (#108228) Message-ID: https://github.com/python/cpython/commit/0dd3fc2a640b273979f94299b545e1e40ac0633c commit: 0dd3fc2a640b273979f94299b545e1e40ac0633c branch: main author: Victor Stinner committer: vstinner date: 2023-08-21T18:05:59Z summary: gh-108216: Cleanup #include in internal header files (#108228) * Add missing includes. * Remove unused includes. * Update old include/symbol names to newer names. * Mention at least one included symbol. * Sort includes. * Update Tools/cases_generator/generate_cases.py used to generated pycore_opcode_metadata.h. * Update Parser/asdl_c.py used to generate pycore_ast.h. * Cleanup also includes in _testcapimodule.c and _testinternalcapi.c. files: M Include/internal/pycore_ast.h M Include/internal/pycore_atomic.h M Include/internal/pycore_bitutils.h M Include/internal/pycore_ceval.h M Include/internal/pycore_ceval_state.h M Include/internal/pycore_condvar.h M Include/internal/pycore_context.h M Include/internal/pycore_dict.h M Include/internal/pycore_faulthandler.h M Include/internal/pycore_fileutils.h M Include/internal/pycore_gil.h M Include/internal/pycore_global_objects.h M Include/internal/pycore_import.h M Include/internal/pycore_instruments.h M Include/internal/pycore_interp.h M Include/internal/pycore_object.h M Include/internal/pycore_opcode_metadata.h M Include/internal/pycore_opcode_utils.h M Include/internal/pycore_optimizer.h M Include/internal/pycore_pymem_init.h M Include/internal/pycore_pystate.h M Include/internal/pycore_pythread.h M Include/internal/pycore_runtime.h M Include/internal/pycore_runtime_init.h M Include/internal/pycore_runtime_init_generated.h M Include/internal/pycore_signal.h M Include/internal/pycore_typeobject.h M Modules/_json.c M Modules/_testcapimodule.c M Modules/_testinternalcapi.c M Parser/asdl_c.py M Tools/cases_generator/generate_cases.py diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index b568902bb1e38..f222d485e0b54 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -10,7 +10,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_asdl.h" +#include "pycore_asdl.h" // _ASDL_SEQ_HEAD typedef struct _mod *mod_ty; diff --git a/Include/internal/pycore_atomic.h b/Include/internal/pycore_atomic.h index 425d69f868b52..48d246ea08f3d 100644 --- a/Include/internal/pycore_atomic.h +++ b/Include/internal/pycore_atomic.h @@ -8,19 +8,19 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "dynamic_annotations.h" /* _Py_ANNOTATE_MEMORY_ORDER */ -#include "pyconfig.h" +#include "pyconfig.h" // HAVE_STD_ATOMIC +#include "dynamic_annotations.h" // _Py_ANNOTATE_MEMORY_ORDER #ifdef HAVE_STD_ATOMIC -# include +# include // atomic_store_explicit() #endif #if defined(_MSC_VER) -#include -#if defined(_M_IX86) || defined(_M_X64) -# include -#endif +# include // _InterlockedExchange64() +# if defined(_M_IX86) || defined(_M_X64) +# include // _InterlockedExchange_HLEAcquire() +# endif #endif /* This is modeled after the atomics interface from C1x, according to diff --git a/Include/internal/pycore_bitutils.h b/Include/internal/pycore_bitutils.h index e6bf61ef425bd..50f6937752381 100644 --- a/Include/internal/pycore_bitutils.h +++ b/Include/internal/pycore_bitutils.h @@ -26,10 +26,10 @@ extern "C" { #endif #ifdef _MSC_VER - /* Get _byteswap_ushort(), _byteswap_ulong(), _byteswap_uint64() */ -# include +# include // _byteswap_uint64() #endif + static inline uint16_t _Py_bswap16(uint16_t word) { diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 0e3a99be8c36a..f32ed3b7e30b7 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -8,6 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_interp.h" // PyInterpreterState.eval_frame +#include "pycore_pystate.h" // _PyThreadState_GET() + /* Forward declarations */ struct pyruntimestate; struct _ceval_runtime_state; @@ -16,10 +19,6 @@ struct _ceval_runtime_state; # define Py_DEFAULT_RECURSION_LIMIT 1000 #endif -#include "pycore_interp.h" // PyInterpreterState.eval_frame -#include "pycore_pystate.h" // _PyThreadState_GET() - - extern void _Py_FinishPendingCalls(PyThreadState *tstate); extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock); extern void _PyEval_FiniState(struct _ceval_state *ceval); diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 1ebfcc9bebd0a..6e3d669dc646a 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -8,8 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif - -#include "pycore_atomic.h" /* _Py_atomic_address */ #include "pycore_gil.h" // struct _gil_runtime_state diff --git a/Include/internal/pycore_condvar.h b/Include/internal/pycore_condvar.h index 981c962bf7dfd..db8682a4c077a 100644 --- a/Include/internal/pycore_condvar.h +++ b/Include/internal/pycore_condvar.h @@ -10,7 +10,7 @@ not present in unistd.h. But they still can be implemented as an external library (e.g. gnu pth in pthread emulation) */ # ifdef HAVE_PTHREAD_H -# include /* _POSIX_THREADS */ +# include // _POSIX_THREADS # endif #endif @@ -21,7 +21,7 @@ #define Py_HAVE_CONDVAR #ifdef HAVE_PTHREAD_H -# include +# include // pthread_mutex_t #endif #define PyMUTEX_T pthread_mutex_t @@ -38,7 +38,7 @@ /* include windows if it hasn't been done before */ #define WIN32_LEAN_AND_MEAN -#include +#include // CRITICAL_SECTION /* options */ /* non-emulated condition variables are provided for those that want diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index 52dfe3ef23387..f1898cf873312 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -5,7 +5,7 @@ # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_hamt.h" /* PyHamtObject */ +#include "pycore_hamt.h" // PyHamtObject extern PyTypeObject _PyContextTokenMissing_Type; diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 2ad6ef0f7c04d..df7bc7e58f6c9 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -9,9 +9,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_dict_state.h" -#include "pycore_object.h" -#include "pycore_runtime.h" // _PyRuntime +#include "pycore_object.h" // PyDictOrValues // Unsafe flavor of PyDict_GetItemWithError(): no error checking extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); diff --git a/Include/internal/pycore_faulthandler.h b/Include/internal/pycore_faulthandler.h index e6aec7745a647..6dd7d8d7ca979 100644 --- a/Include/internal/pycore_faulthandler.h +++ b/Include/internal/pycore_faulthandler.h @@ -9,7 +9,7 @@ extern "C" { #endif #ifdef HAVE_SIGACTION -# include +# include // sigaction #endif diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 25b383b9780ad..96e3d1c7abcdd 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -8,7 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include /* struct lconv */ +#include // struct lconv + /* A routine to check if a file descriptor can be select()-ed. */ #ifdef _MSC_VER @@ -268,7 +269,7 @@ extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t // so provide our own implementations. Remove them in case they get added // to the Games API family #if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) -#include +#include // HRESULT extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootEnd); #endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ diff --git a/Include/internal/pycore_gil.h b/Include/internal/pycore_gil.h index 8ebad37b686cd..a1a15070eef46 100644 --- a/Include/internal/pycore_gil.h +++ b/Include/internal/pycore_gil.h @@ -8,8 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_atomic.h" /* _Py_atomic_address */ -#include "pycore_condvar.h" /* PyCOND_T */ +#include "pycore_atomic.h" // _Py_atomic_address +#include "pycore_condvar.h" // PyCOND_T #ifndef Py_HAVE_CONDVAR # error You need either a POSIX-compatible or a Windows system! diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h index 442f8516278b0..327fcc24cb29f 100644 --- a/Include/internal/pycore_global_objects.h +++ b/Include/internal/pycore_global_objects.h @@ -8,11 +8,11 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_hashtable.h" // _Py_hashtable_t -#include "pycore_gc.h" // PyGC_Head +#include "pycore_context.h" // _PyContextTokenMissing +#include "pycore_gc.h" // _PyGC_Head_UNUSED #include "pycore_global_strings.h" // struct _Py_global_strings #include "pycore_hamt.h" // PyHamtNode_Bitmap -#include "pycore_context.h" // _PyContextTokenMissing +#include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_typeobject.h" // pytype_slotdef diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 34f572bd59296..e23f9931cea73 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -103,7 +103,7 @@ struct _import_state { }; #ifdef HAVE_DLOPEN -# include +# include // RTLD_NOW, RTLD_LAZY # if HAVE_DECL_RTLD_NOW # define _Py_DLOPEN_FLAGS RTLD_NOW # else diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index e15447acec2c0..43214aef7f7a1 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -6,9 +6,7 @@ #endif #include "pycore_bitutils.h" // _Py_popcount32 -#include "pycore_frame.h" - -#include "cpython/code.h" +#include "pycore_frame.h" // _PyInterpreterFrame #ifdef __cplusplus extern "C" { diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 91c473e58eaba..c21c352608d16 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include +#include // bool #include "pycore_ast_state.h" // struct ast_state #include "pycore_atexit.h" // struct atexit_state @@ -21,16 +21,16 @@ extern "C" { #include "pycore_exceptions.h" // struct _Py_exc_state #include "pycore_floatobject.h" // struct _Py_float_state #include "pycore_function.h" // FUNC_MAX_WATCHERS -#include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_gc.h" // struct _gc_runtime_state -#include "pycore_global_objects.h" // struct _Py_interp_static_objects +#include "pycore_genobject.h" // struct _Py_async_gen_state +#include "pycore_global_objects.h"// struct _Py_interp_cached_objects #include "pycore_import.h" // struct _import_state #include "pycore_instruments.h" // _PY_MONITORING_EVENTS #include "pycore_list.h" // struct _Py_list_state -#include "pycore_object_state.h" // struct _py_object_state -#include "pycore_obmalloc.h" // struct obmalloc_state +#include "pycore_object_state.h" // struct _py_object_state +#include "pycore_obmalloc.h" // struct _obmalloc_state #include "pycore_tuple.h" // struct _Py_tuple_state -#include "pycore_typeobject.h" // struct type_cache +#include "pycore_typeobject.h" // struct types_state #include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_warnings.h" // struct _warnings_runtime_state diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 857d6efec3b3b..5e2d13a522055 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -12,7 +12,6 @@ extern "C" { #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_runtime.h" // _PyRuntime /* Check if an object is consistent. For example, ensure that the reference counter is greater than or equal to 1, and ensure that ob_type is not NULL. diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index fab91e611b617..e35db0c4c5a59 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -7,7 +7,7 @@ # error "this header requires Py_BUILD_CORE define" #endif -#include +#include // bool #define IS_PSEUDO_INSTR(OP) ( \ diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 50ff2af38d2cb..f17612908cebd 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_opcode.h" // _PyOpcode_Jump +#include "pycore_opcode.h" // JUMP_FORWARD #define MAX_REAL_OPCODE 254 diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 2ae657c4e117f..f9f16c48cbc21 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_uops.h" +#include "pycore_uops.h" // _PyUOpInstruction int _Py_uop_analyze_and_optimize(PyCodeObject *code, _PyUOpInstruction *trace, int trace_len, int curr_stackentries); diff --git a/Include/internal/pycore_pymem_init.h b/Include/internal/pycore_pymem_init.h index 78232738cb09d..119fa16fb911a 100644 --- a/Include/internal/pycore_pymem_init.h +++ b/Include/internal/pycore_pymem_init.h @@ -8,8 +8,6 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_pymem.h" - /********************************/ /* the allocators' initializers */ diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index acc6cf953343f..f6ca9102b139e 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_runtime.h" /* PyRuntimeState */ +#include "pycore_runtime.h" // _PyRuntime /* Check if the current thread is the main thread. diff --git a/Include/internal/pycore_pythread.h b/Include/internal/pycore_pythread.h index f53921494c158..44846c0fc4b4c 100644 --- a/Include/internal/pycore_pythread.h +++ b/Include/internal/pycore_pythread.h @@ -13,9 +13,9 @@ extern "C" { /* This means pthreads are not implemented in libc headers, hence the macro not present in unistd.h. But they still can be implemented as an external library (e.g. gnu pth in pthread emulation) */ -# ifdef HAVE_PTHREAD_H -# include /* _POSIX_THREADS */ -# endif +# ifdef HAVE_PTHREAD_H +# include // _POSIX_THREADS +# endif # ifndef _POSIX_THREADS /* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then enough of the Posix threads package is implemented to support python @@ -34,12 +34,12 @@ extern "C" { #endif /* _POSIX_THREADS */ #if defined(_POSIX_THREADS) || defined(HAVE_PTHREAD_STUBS) -# define _USE_PTHREADS +# define _USE_PTHREADS #endif #if defined(_USE_PTHREADS) && defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_MONOTONIC) // monotonic is supported statically. It doesn't mean it works on runtime. -# define CONDATTR_MONOTONIC +# define CONDATTR_MONOTONIC #endif diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 63b485027486b..4a460803c69bd 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -8,24 +8,23 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_atexit.h" // struct atexit_runtime_state -#include "pycore_atomic.h" /* _Py_atomic_address */ +#include "pycore_atexit.h" // struct _atexit_runtime_state +#include "pycore_atomic.h" // _Py_atomic_address #include "pycore_ceval_state.h" // struct _ceval_runtime_state -#include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state -#include "pycore_global_objects.h" // struct _Py_global_objects +#include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_import.h" // struct _import_runtime_state #include "pycore_interp.h" // PyInterpreterState #include "pycore_object_state.h" // struct _py_object_runtime_state #include "pycore_parser.h" // struct _parser_runtime_state -#include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pyhash.h" // struct pyhash_runtime_state +#include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pythread.h" // struct _pythread_runtime_state #include "pycore_signal.h" // struct _signals_runtime_state #include "pycore_time.h" // struct _time_runtime_state #include "pycore_tracemalloc.h" // struct _tracemalloc_runtime_state -#include "pycore_typeobject.h" // struct types_runtime_state -#include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids +#include "pycore_typeobject.h" // struct _types_runtime_state +#include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_state struct _getargs_runtime_state { PyThread_type_lock mutex; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index ea29c69f59acc..c775a8a7e7eef 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -8,11 +8,17 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_long.h" -#include "pycore_object.h" -#include "pycore_parser.h" -#include "pycore_pymem_init.h" -#include "pycore_obmalloc_init.h" +#include "pycore_ceval_state.h" // _PyEval_RUNTIME_PERF_INIT +#include "pycore_faulthandler.h" // _faulthandler_runtime_state_INIT +#include "pycore_floatobject.h" // _py_float_format_unknown +#include "pycore_object.h" // _PyObject_HEAD_INIT +#include "pycore_obmalloc_init.h" // _obmalloc_global_state_INIT +#include "pycore_parser.h" // _parser_runtime_state_INIT +#include "pycore_pyhash.h" // pyhash_state_INIT +#include "pycore_pymem_init.h" // _pymem_allocators_standard_INIT +#include "pycore_runtime_init_generated.h" // _Py_bytes_characters_INIT +#include "pycore_signal.h" // _signals_RUNTIME_INIT +#include "pycore_tracemalloc.h" // _tracemalloc_runtime_state_INIT extern PyTypeObject _PyExc_MemoryError; diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 8c0fcdb1bdae1..694a409fbf4f4 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -8,6 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_long.h" // _PyLong_DIGIT_INIT() + + /* The following is auto-generated by Tools/build/generate_global_objects.py. */ #define _Py_small_ints_INIT { \ _PyLong_DIGIT_INIT(-5), \ diff --git a/Include/internal/pycore_signal.h b/Include/internal/pycore_signal.h index 46b57d556e5ef..8f876759c0d07 100644 --- a/Include/internal/pycore_signal.h +++ b/Include/internal/pycore_signal.h @@ -10,8 +10,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_atomic.h" // _Py_atomic_address -#include // NSIG +#include "pycore_atomic.h" // _Py_atomic_address +#include // NSIG // Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index aba672effe392..4d6a0189c4afe 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -4,12 +4,12 @@ extern "C" { #endif -#include "pycore_moduleobject.h" - #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_moduleobject.h" // PyModuleObject + /* state */ diff --git a/Modules/_json.c b/Modules/_json.c index c7cfe50b52faf..41495e2012f15 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -12,7 +12,7 @@ #include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_runtime.h" // _PyRuntime -#include "pycore_global_objects.h" // _Py_ID() +#include "pycore_global_strings.h" // _Py_ID() #include // bool diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 35599f8baa204..a7a98d1eea5bd 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -18,8 +18,8 @@ #undef NDEBUG #include "Python.h" -#include "frameobject.h" // PyFrame_New -#include "marshal.h" // PyMarshal_WriteLongToFile +#include "frameobject.h" // PyFrame_New() +#include "marshal.h" // PyMarshal_WriteLongToFile() #include // FLT_MAX #include @@ -37,7 +37,7 @@ #endif #ifdef bool -# error "The public headers should not include , see bpo-46748" +# error "The public headers should not include , see gh-48924" #endif // Several parts of this module are broken out into files in _testcapi/. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index d1082c7dae8ae..5571ae463373e 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -10,26 +10,24 @@ #undef NDEBUG #include "Python.h" -#include "pycore_atomic_funcs.h" // _Py_atomic_int_get() -#include "pycore_bitutils.h" // _Py_bswap32() -#include "pycore_bytesobject.h" // _PyBytes_Find() -#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc -#include "pycore_ceval.h" // _PyEval_AddPendingCall -#include "pycore_dict.h" // _PyDictOrValues_GetValues -#include "pycore_fileutils.h" // _Py_normpath -#include "pycore_frame.h" // _PyInterpreterFrame -#include "pycore_gc.h" // PyGC_Head -#include "pycore_hashtable.h" // _Py_hashtable_new() -#include "pycore_initconfig.h" // _Py_GetConfigsAsDict() -#include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() -#include "pycore_object.h" // _PyObject_IsFreed() -#include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() -#include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() -#include "pycore_pystate.h" // _PyThreadState_GET() - -#include "frameobject.h" -#include "interpreteridobject.h" // PyInterpreterID_LookUp() -#include "osdefs.h" // MAXPATHLEN +#include "pycore_atomic_funcs.h" // _Py_atomic_int_get() +#include "pycore_bitutils.h" // _Py_bswap32() +#include "pycore_bytesobject.h" // _PyBytes_Find() +#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble, _PyCompile_CleanDoc +#include "pycore_ceval.h" // _PyEval_AddPendingCall +#include "pycore_dict.h" // _PyDictOrValues_GetValues() +#include "pycore_fileutils.h" // _Py_normpath +#include "pycore_frame.h" // _PyInterpreterFrame +#include "pycore_gc.h" // PyGC_Head +#include "pycore_hashtable.h" // _Py_hashtable_new() +#include "pycore_initconfig.h" // _Py_GetConfigsAsDict() +#include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() +#include "pycore_object.h" // _PyObject_IsFreed() +#include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() +#include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() +#include "pycore_pystate.h" // _PyThreadState_GET() + +#include "interpreteridobject.h" // PyInterpreterID_LookUp() #include "clinic/_testinternalcapi.c.h" diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 1733cd4b15aa4..bf144871bb736 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1585,7 +1585,7 @@ def write_header(mod, metadata, f): # error "this header requires Py_BUILD_CORE define" #endif - #include "pycore_asdl.h" + #include "pycore_asdl.h" // _ASDL_SEQ_HEAD """).lstrip()) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index c3b729df44a6a..6ee6836305811 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -410,7 +410,7 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No #endif """).strip()) - self.out.emit("\n#include ") + self.out.emit("\n#include // bool") self.write_pseudo_instrs() From webhook-mailer at python.org Mon Aug 21 14:08:08 2023 From: webhook-mailer at python.org (gvanrossum) Date: Mon, 21 Aug 2023 18:08:08 -0000 Subject: [Python-checkins] gh-108224: Fix asyncio doc inconsistency (#108230) Message-ID: https://github.com/python/cpython/commit/1cc391d9e2ea24ca750005335507b52933fc0b52 commit: 1cc391d9e2ea24ca750005335507b52933fc0b52 branch: main author: temach committer: gvanrossum date: 2023-08-21T18:08:04Z summary: gh-108224: Fix asyncio doc inconsistency (#108230) (Spawning subprocesses does not require the event loop to run in the main thread -- only signal handling does.) files: M Doc/library/asyncio-dev.rst diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index c7d97008fb490..a9c3a0183bb72 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -99,7 +99,7 @@ To schedule a coroutine object from a different OS thread, the # Wait for the result: result = future.result() -To handle signals and to execute subprocesses, the event loop must be +To handle signals the event loop must be run in the main thread. The :meth:`loop.run_in_executor` method can be used with a From webhook-mailer at python.org Mon Aug 21 14:12:26 2023 From: webhook-mailer at python.org (vstinner) Date: Mon, 21 Aug 2023 18:12:26 -0000 Subject: [Python-checkins] gh-107211: No longer export pycore_strhex.h functions (#108229) Message-ID: https://github.com/python/cpython/commit/18fc543b377f3521b2503e0d013e1d32ccdf9190 commit: 18fc543b377f3521b2503e0d013e1d32ccdf9190 branch: main author: Victor Stinner committer: vstinner date: 2023-08-21T18:12:22Z summary: gh-107211: No longer export pycore_strhex.h functions (#108229) No longer export functions: * _Py_strhex_bytes() * _Py_strhex_with_sep() files: M Include/internal/pycore_strhex.h diff --git a/Include/internal/pycore_strhex.h b/Include/internal/pycore_strhex.h index f427b4d695bd2..225f423912f2c 100644 --- a/Include/internal/pycore_strhex.h +++ b/Include/internal/pycore_strhex.h @@ -9,21 +9,24 @@ extern "C" { #endif // Returns a str() containing the hex representation of argbuf. +// Export for '_hashlib' shared extension. PyAPI_FUNC(PyObject*) _Py_strhex(const char* argbuf, const Py_ssize_t arglen); // Returns a bytes() containing the ASCII hex representation of argbuf. -PyAPI_FUNC(PyObject*) _Py_strhex_bytes( +extern PyObject* _Py_strhex_bytes( const char* argbuf, const Py_ssize_t arglen); // These variants include support for a separator between every N bytes: -PyAPI_FUNC(PyObject*) _Py_strhex_with_sep( +extern PyObject* _Py_strhex_with_sep( const char* argbuf, const Py_ssize_t arglen, PyObject* sep, const int bytes_per_group); + +// Export for 'binascii' shared extension PyAPI_FUNC(PyObject*) _Py_strhex_bytes_with_sep( const char* argbuf, const Py_ssize_t arglen, From webhook-mailer at python.org Mon Aug 21 14:15:51 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Mon, 21 Aug 2023 18:15:51 -0000 Subject: [Python-checkins] gh-107298: Fix C API datetime documentation (GH-108034) Message-ID: https://github.com/python/cpython/commit/d63972e289e05b0d82e59f32f107312a8b3de7b5 commit: d63972e289e05b0d82e59f32f107312a8b3de7b5 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-21T21:15:46+03:00 summary: gh-107298: Fix C API datetime documentation (GH-108034) files: M Doc/c-api/datetime.rst M Doc/tools/.nitignore diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index 72fc07afbf1f4..97522da773477 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -8,11 +8,54 @@ DateTime Objects Various date and time objects are supplied by the :mod:`datetime` module. Before using any of these functions, the header file :file:`datetime.h` must be included in your source (note that this is not included by :file:`Python.h`), -and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of +and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of the module initialisation function. The macro puts a pointer to a C structure -into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following +into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following macros. +.. c:type:: PyDateTime_Date + + This subtype of :c:type:`PyObject` represents a Python date object. + +.. c:type:: PyDateTime_DateTime + + This subtype of :c:type:`PyObject` represents a Python datetime object. + +.. c:type:: PyDateTime_Time + + This subtype of :c:type:`PyObject` represents a Python time object. + +.. c:type:: PyDateTime_Delta + + This subtype of :c:type:`PyObject` represents the difference between two datetime values. + +.. c:var:: PyTypeObject PyDateTime_DateType + + This instance of :c:type:`PyTypeObject` represents the Python date type; + it is the same object as :class:`datetime.date` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_DateTimeType + + This instance of :c:type:`PyTypeObject` represents the Python datetime type; + it is the same object as :class:`datetime.datetime` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_TimeType + + This instance of :c:type:`PyTypeObject` represents the Python time type; + it is the same object as :class:`datetime.time` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_DeltaType + + This instance of :c:type:`PyTypeObject` represents Python type for + the difference between two datetime values; + it is the same object as :class:`datetime.timedelta` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_TZInfoType + + This instance of :c:type:`PyTypeObject` represents the Python time zone info type; + it is the same object as :class:`datetime.tzinfo` in the Python layer. + + Macro for access to the UTC singleton: .. c:var:: PyObject* PyDateTime_TimeZone_UTC @@ -28,7 +71,7 @@ Type-check macros: .. c:function:: int PyDate_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateType` or a subtype of - :c:data:`PyDateTime_DateType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DateType`. *ob* must not be ``NULL``. This function always succeeds. @@ -41,7 +84,7 @@ Type-check macros: .. c:function:: int PyDateTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType` or a subtype of - :c:data:`PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always succeeds. @@ -54,7 +97,7 @@ Type-check macros: .. c:function:: int PyTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TimeType` or a subtype of - :c:data:`PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always succeeds. @@ -67,7 +110,7 @@ Type-check macros: .. c:function:: int PyDelta_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DeltaType` or a subtype of - :c:data:`PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always succeeds. @@ -80,7 +123,7 @@ Type-check macros: .. c:function:: int PyTZInfo_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType` or a subtype of - :c:data:`PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always succeeds. @@ -133,7 +176,7 @@ Macros to create objects: :class:`datetime.timedelta` objects. -.. c:function:: PyObject* PyTimeZone_FromOffset(PyDateTime_DeltaType* offset) +.. c:function:: PyObject* PyTimeZone_FromOffset(PyObject *offset) Return a :class:`datetime.timezone` object with an unnamed fixed offset represented by the *offset* argument. @@ -141,7 +184,7 @@ Macros to create objects: .. versionadded:: 3.7 -.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyDateTime_DeltaType* offset, PyUnicode* name) +.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyObject *offset, PyObject *name) Return a :class:`datetime.timezone` object with a fixed offset represented by the *offset* argument and with tzname *name*. @@ -150,8 +193,8 @@ Macros to create objects: Macros to extract fields from date objects. The argument must be an instance of -:c:data:`PyDateTime_Date`, including subclasses (such as -:c:data:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is +:c:type:`PyDateTime_Date`, including subclasses (such as +:c:type:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_GET_YEAR(PyDateTime_Date *o) @@ -170,7 +213,7 @@ not checked: Macros to extract fields from datetime objects. The argument must be an -instance of :c:data:`PyDateTime_DateTime`, including subclasses. The argument +instance of :c:type:`PyDateTime_DateTime`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DATE_GET_HOUR(PyDateTime_DateTime *o) @@ -208,7 +251,7 @@ must not be ``NULL``, and the type is not checked: Macros to extract fields from time objects. The argument must be an instance of -:c:data:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``, +:c:type:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_TIME_GET_HOUR(PyDateTime_Time *o) @@ -246,7 +289,7 @@ and the type is not checked: Macros to extract fields from time delta objects. The argument must be an -instance of :c:data:`PyDateTime_Delta`, including subclasses. The argument must +instance of :c:type:`PyDateTime_Delta`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DELTA_GET_DAYS(PyDateTime_Delta *o) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 4231fb8575891..09514c89c117a 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -2,7 +2,6 @@ # as tested on the CI via check-warnings.py in reusable-docs.yml. # Keep lines sorted lexicographically to help avoid merge conflicts. -Doc/c-api/datetime.rst Doc/c-api/descriptor.rst Doc/c-api/exceptions.rst Doc/c-api/file.rst From webhook-mailer at python.org Mon Aug 21 15:18:45 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 21 Aug 2023 19:18:45 -0000 Subject: [Python-checkins] [3.12] gh-107905: Test raising `__value__` for `TypeAliasType` (GH-107997) (#108217) Message-ID: https://github.com/python/cpython/commit/f04024d5880d0ff8888ae3899074aeddf5473bfc commit: f04024d5880d0ff8888ae3899074aeddf5473bfc branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-21T21:18:42+02:00 summary: [3.12] gh-107905: Test raising `__value__` for `TypeAliasType` (GH-107997) (#108217) gh-107905: Test raising `__value__` for `TypeAliasType` (GH-107997) (cherry picked from commit 13104f3b7412dce9bf7cfd09bf2d6dad1f3cc2ed) Co-authored-by: Nikita Sobolev files: M Lib/test/test_type_aliases.py diff --git a/Lib/test/test_type_aliases.py b/Lib/test/test_type_aliases.py index 0ce97f57de686..8f0a998e1f3dc 100644 --- a/Lib/test/test_type_aliases.py +++ b/Lib/test/test_type_aliases.py @@ -168,6 +168,24 @@ def test_recursive_repr(self): self.assertEqual(repr(GenericRecursive[GenericRecursive[int]]), "GenericRecursive[GenericRecursive[int]]") + def test_raising(self): + type MissingName = list[_My_X] + with self.assertRaisesRegex( + NameError, + "cannot access free variable '_My_X' where it is not associated with a value", + ): + MissingName.__value__ + _My_X = int + self.assertEqual(MissingName.__value__, list[int]) + del _My_X + # Cache should still work: + self.assertEqual(MissingName.__value__, list[int]) + + # Explicit exception: + type ExprException = 1 / 0 + with self.assertRaises(ZeroDivisionError): + ExprException.__value__ + class TypeAliasConstructorTest(unittest.TestCase): def test_basic(self): From webhook-mailer at python.org Mon Aug 21 15:39:48 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 21 Aug 2023 19:39:48 -0000 Subject: [Python-checkins] [3.12] Fix sphinx-lint issue now that lint runs on Misc/NEWS.d (#108237) Message-ID: https://github.com/python/cpython/commit/bc97eb76bb674d76b907e7031f1b7ea11b68121b commit: bc97eb76bb674d76b907e7031f1b7ea11b68121b branch: 3.12 author: T. Wouters committer: Yhg1s date: 2023-08-21T19:39:45Z summary: [3.12] Fix sphinx-lint issue now that lint runs on Misc/NEWS.d (#108237) Fix lint warnings in Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst (effectively a backport of GH-108212). files: M Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst diff --git a/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst b/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst index f051317d141ff..1d959a3b22284 100644 --- a/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst +++ b/Misc/NEWS.d/next/Library/2023-08-17-14-45-25.gh-issue-105736.NJsH7r.rst @@ -1,3 +1,3 @@ -Harmonized the pure Python version of OrderedDict with the C version. Now, -both versions set up their internal state in `__new__`. Formerly, the pure -Python version did the set up in `__init__`. +Harmonized the pure Python version of :class:`~collections.OrderedDict` with the C version. Now, +both versions set up their internal state in ``__new__``. Formerly, the pure +Python version did the set up in ``__init__``. From webhook-mailer at python.org Mon Aug 21 15:40:12 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 21 Aug 2023 19:40:12 -0000 Subject: [Python-checkins] [3.12] docs: fix grammar in isolating-extensions.rst (GH-108037) (#108218) Message-ID: https://github.com/python/cpython/commit/a27f18a3824365173c4102179ecd28aab72690f8 commit: a27f18a3824365173c4102179ecd28aab72690f8 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-21T21:40:08+02:00 summary: [3.12] docs: fix grammar in isolating-extensions.rst (GH-108037) (#108218) docs: fix grammar in isolating-extensions.rst (GH-108037) (cherry picked from commit 47022a079eb9d2a2af781abae3de4a71f80247c2) Co-authored-by: David Lechner files: M Doc/howto/isolating-extensions.rst diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 2551fbe87b5c2..8f3787f2d2f14 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -64,7 +64,7 @@ Enter Per-Module State Instead of focusing on per-interpreter state, Python's C API is evolving to better support the more granular *per-module* state. -This means that C-level data is be attached to a *module object*. +This means that C-level data should be attached to a *module object*. Each interpreter creates its own module object, keeping the data separate. For testing the isolation, multiple module objects corresponding to a single extension can even be loaded in a single interpreter. From webhook-mailer at python.org Mon Aug 21 15:43:32 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 21 Aug 2023 19:43:32 -0000 Subject: [Python-checkins] [3.12] gh-107298: Fix references to deprecated and removed PyUnicode C API (GH-108077) (#108225) Message-ID: https://github.com/python/cpython/commit/78a2a8c026aa082a3ca71e5c86d2303a364c97be commit: 78a2a8c026aa082a3ca71e5c86d2303a364c97be branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-21T21:43:28+02:00 summary: [3.12] gh-107298: Fix references to deprecated and removed PyUnicode C API (GH-108077) (#108225) gh-107298: Fix references to deprecated and removed PyUnicode C API (GH-108077) (cherry picked from commit db55383829ccd5ce80c551d60f26851346741fdf) Co-authored-by: Serhiy Storchaka files: M Doc/whatsnew/3.11.rst M Doc/whatsnew/3.3.rst M Doc/whatsnew/3.6.rst M Doc/whatsnew/3.9.rst M Misc/NEWS.d/3.11.0b1.rst diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 712af6217b6a4..0ac8d02ab15c8 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2591,22 +2591,22 @@ Pending Removal in Python 3.12 The following C APIs have been deprecated in earlier Python releases, and will be removed in Python 3.12. -* :c:func:`PyUnicode_AS_DATA` -* :c:func:`PyUnicode_AS_UNICODE` -* :c:func:`PyUnicode_AsUnicodeAndSize` -* :c:func:`PyUnicode_AsUnicode` -* :c:func:`PyUnicode_FromUnicode` -* :c:func:`PyUnicode_GET_DATA_SIZE` -* :c:func:`PyUnicode_GET_SIZE` -* :c:func:`PyUnicode_GetSize` +* :c:func:`!PyUnicode_AS_DATA` +* :c:func:`!PyUnicode_AS_UNICODE` +* :c:func:`!PyUnicode_AsUnicodeAndSize` +* :c:func:`!PyUnicode_AsUnicode` +* :c:func:`!PyUnicode_FromUnicode` +* :c:func:`!PyUnicode_GET_DATA_SIZE` +* :c:func:`!PyUnicode_GET_SIZE` +* :c:func:`!PyUnicode_GetSize` * :c:func:`PyUnicode_IS_COMPACT` * :c:func:`PyUnicode_IS_READY` * :c:func:`PyUnicode_READY` -* :c:func:`Py_UNICODE_WSTR_LENGTH` -* :c:func:`_PyUnicode_AsUnicode` -* :c:macro:`PyUnicode_WCHAR_KIND` +* :c:func:`!PyUnicode_WSTR_LENGTH` +* :c:func:`!_PyUnicode_AsUnicode` +* :c:macro:`!PyUnicode_WCHAR_KIND` * :c:type:`PyUnicodeObject` -* :c:func:`PyUnicode_InternImmortal()` +* :c:func:`!PyUnicode_InternImmortal` .. _whatsnew311-c-api-removed: @@ -2614,7 +2614,7 @@ and will be removed in Python 3.12. Removed ------- -* :c:func:`PyFrame_BlockSetup` and :c:func:`PyFrame_BlockPop` have been +* :c:func:`!PyFrame_BlockSetup` and :c:func:`!PyFrame_BlockPop` have been removed. (Contributed by Mark Shannon in :issue:`40222`.) diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 1414b2f79a67d..8cf7ad57475b2 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -249,7 +249,7 @@ Changes introduced by :pep:`393` are the following: non-BMP code points. * The value of :data:`sys.maxunicode` is now always ``1114111`` (``0x10FFFF`` - in hexadecimal). The :c:func:`PyUnicode_GetMax` function still returns + in hexadecimal). The :c:func:`!PyUnicode_GetMax` function still returns either ``0xFFFF`` or ``0x10FFFF`` for backward compatibility, and it should not be used with the new Unicode API (see :issue:`13054`). @@ -2196,7 +2196,7 @@ Changes to Python's build process and to the C API include: * :c:macro:`PyUnicode_DATA`, :c:macro:`PyUnicode_1BYTE_DATA`, :c:macro:`PyUnicode_2BYTE_DATA`, :c:macro:`PyUnicode_4BYTE_DATA` * :c:macro:`PyUnicode_KIND` with :c:enum:`PyUnicode_Kind` enum: - :c:data:`PyUnicode_WCHAR_KIND`, :c:data:`PyUnicode_1BYTE_KIND`, + :c:data:`!PyUnicode_WCHAR_KIND`, :c:data:`PyUnicode_1BYTE_KIND`, :c:data:`PyUnicode_2BYTE_KIND`, :c:data:`PyUnicode_4BYTE_KIND` * :c:macro:`PyUnicode_READ`, :c:macro:`PyUnicode_READ_CHAR`, :c:macro:`PyUnicode_WRITE` * :c:macro:`PyUnicode_MAX_CHAR_VALUE` @@ -2270,58 +2270,58 @@ removed in Python 4. All functions using this type are deprecated: Unicode functions and methods using :c:type:`Py_UNICODE` and :c:expr:`Py_UNICODE*` types: -* :c:macro:`PyUnicode_FromUnicode`: use :c:func:`PyUnicode_FromWideChar` or +* :c:macro:`!PyUnicode_FromUnicode`: use :c:func:`PyUnicode_FromWideChar` or :c:func:`PyUnicode_FromKindAndData` -* :c:macro:`PyUnicode_AS_UNICODE`, :c:func:`PyUnicode_AsUnicode`, - :c:func:`PyUnicode_AsUnicodeAndSize`: use :c:func:`PyUnicode_AsWideCharString` -* :c:macro:`PyUnicode_AS_DATA`: use :c:macro:`PyUnicode_DATA` with +* :c:macro:`!PyUnicode_AS_UNICODE`, :c:func:`!PyUnicode_AsUnicode`, + :c:func:`!PyUnicode_AsUnicodeAndSize`: use :c:func:`PyUnicode_AsWideCharString` +* :c:macro:`!PyUnicode_AS_DATA`: use :c:macro:`PyUnicode_DATA` with :c:macro:`PyUnicode_READ` and :c:macro:`PyUnicode_WRITE` -* :c:macro:`PyUnicode_GET_SIZE`, :c:func:`PyUnicode_GetSize`: use +* :c:macro:`!PyUnicode_GET_SIZE`, :c:func:`!PyUnicode_GetSize`: use :c:macro:`PyUnicode_GET_LENGTH` or :c:func:`PyUnicode_GetLength` -* :c:macro:`PyUnicode_GET_DATA_SIZE`: use +* :c:macro:`!PyUnicode_GET_DATA_SIZE`: use ``PyUnicode_GET_LENGTH(str) * PyUnicode_KIND(str)`` (only work on ready strings) -* :c:func:`PyUnicode_AsUnicodeCopy`: use :c:func:`PyUnicode_AsUCS4Copy` or +* :c:func:`!PyUnicode_AsUnicodeCopy`: use :c:func:`PyUnicode_AsUCS4Copy` or :c:func:`PyUnicode_AsWideCharString` -* :c:func:`PyUnicode_GetMax` +* :c:func:`!PyUnicode_GetMax` Functions and macros manipulating Py_UNICODE* strings: -* :c:macro:`Py_UNICODE_strlen`: use :c:func:`PyUnicode_GetLength` or +* :c:macro:`!Py_UNICODE_strlen()`: use :c:func:`PyUnicode_GetLength` or :c:macro:`PyUnicode_GET_LENGTH` -* :c:macro:`Py_UNICODE_strcat`: use :c:func:`PyUnicode_CopyCharacters` or +* :c:macro:`!Py_UNICODE_strcat()`: use :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_FromFormat` -* :c:macro:`Py_UNICODE_strcpy`, :c:macro:`Py_UNICODE_strncpy`, - :c:macro:`Py_UNICODE_COPY`: use :c:func:`PyUnicode_CopyCharacters` or +* :c:macro:`!Py_UNICODE_strcpy()`, :c:macro:`!Py_UNICODE_strncpy()`, + :c:macro:`!Py_UNICODE_COPY()`: use :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_Substring` -* :c:macro:`Py_UNICODE_strcmp`: use :c:func:`PyUnicode_Compare` -* :c:macro:`Py_UNICODE_strncmp`: use :c:func:`PyUnicode_Tailmatch` -* :c:macro:`Py_UNICODE_strchr`, :c:macro:`Py_UNICODE_strrchr`: use +* :c:macro:`!Py_UNICODE_strcmp()`: use :c:func:`PyUnicode_Compare` +* :c:macro:`!Py_UNICODE_strncmp()`: use :c:func:`PyUnicode_Tailmatch` +* :c:macro:`!Py_UNICODE_strchr()`, :c:macro:`!Py_UNICODE_strrchr()`: use :c:func:`PyUnicode_FindChar` -* :c:macro:`Py_UNICODE_FILL`: use :c:func:`PyUnicode_Fill` -* :c:macro:`Py_UNICODE_MATCH` +* :c:macro:`!Py_UNICODE_FILL()`: use :c:func:`PyUnicode_Fill` +* :c:macro:`!Py_UNICODE_MATCH` Encoders: -* :c:func:`PyUnicode_Encode`: use :c:func:`PyUnicode_AsEncodedObject` -* :c:func:`PyUnicode_EncodeUTF7` -* :c:func:`PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or +* :c:func:`!PyUnicode_Encode`: use :c:func:`PyUnicode_AsEncodedObject` +* :c:func:`!PyUnicode_EncodeUTF7` +* :c:func:`!PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or :c:func:`PyUnicode_AsUTF8String` -* :c:func:`PyUnicode_EncodeUTF32` -* :c:func:`PyUnicode_EncodeUTF16` -* :c:func:`PyUnicode_EncodeUnicodeEscape` use +* :c:func:`!PyUnicode_EncodeUTF32` +* :c:func:`!PyUnicode_EncodeUTF16` +* :c:func:`!PyUnicode_EncodeUnicodeEscape` use :c:func:`PyUnicode_AsUnicodeEscapeString` -* :c:func:`PyUnicode_EncodeRawUnicodeEscape` use +* :c:func:`!PyUnicode_EncodeRawUnicodeEscape` use :c:func:`PyUnicode_AsRawUnicodeEscapeString` -* :c:func:`PyUnicode_EncodeLatin1`: use :c:func:`PyUnicode_AsLatin1String` -* :c:func:`PyUnicode_EncodeASCII`: use :c:func:`PyUnicode_AsASCIIString` -* :c:func:`PyUnicode_EncodeCharmap` -* :c:func:`PyUnicode_TranslateCharmap` -* :c:func:`PyUnicode_EncodeMBCS`: use :c:func:`PyUnicode_AsMBCSString` or +* :c:func:`!PyUnicode_EncodeLatin1`: use :c:func:`PyUnicode_AsLatin1String` +* :c:func:`!PyUnicode_EncodeASCII`: use :c:func:`PyUnicode_AsASCIIString` +* :c:func:`!PyUnicode_EncodeCharmap` +* :c:func:`!PyUnicode_TranslateCharmap` +* :c:func:`!PyUnicode_EncodeMBCS`: use :c:func:`PyUnicode_AsMBCSString` or :c:func:`PyUnicode_EncodeCodePage` (with ``CP_ACP`` code_page) -* :c:func:`PyUnicode_EncodeDecimal`, - :c:func:`PyUnicode_TransformDecimalToASCII` +* :c:func:`!PyUnicode_EncodeDecimal`, + :c:func:`!PyUnicode_TransformDecimalToASCII` Deprecated features diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 1093d24531505..be20ee1afb4e8 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2066,9 +2066,9 @@ environment. (Contributed by Brett Cannon in :issue:`25154`.) Deprecated functions and types of the C API ------------------------------------------- -Undocumented functions :c:func:`PyUnicode_AsEncodedObject`, -:c:func:`PyUnicode_AsDecodedObject`, :c:func:`PyUnicode_AsEncodedUnicode` -and :c:func:`PyUnicode_AsDecodedUnicode` are deprecated now. +Undocumented functions :c:func:`!PyUnicode_AsEncodedObject`, +:c:func:`!PyUnicode_AsDecodedObject`, :c:func:`!PyUnicode_AsEncodedUnicode` +and :c:func:`!PyUnicode_AsDecodedUnicode` are deprecated now. Use the :ref:`generic codec based API ` instead. diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 666d53ed6da7b..664c8d8a545db 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -1370,8 +1370,8 @@ Porting to Python 3.9 (Contributed by Victor Stinner in :issue:`40241`.) * The ``Py_UNICODE_COPY``, ``Py_UNICODE_FILL``, ``PyUnicode_WSTR_LENGTH``, - :c:func:`PyUnicode_FromUnicode`, :c:func:`PyUnicode_AsUnicode`, - ``_PyUnicode_AsUnicode``, and :c:func:`PyUnicode_AsUnicodeAndSize` are + :c:func:`!PyUnicode_FromUnicode`, :c:func:`!PyUnicode_AsUnicode`, + ``_PyUnicode_AsUnicode``, and :c:func:`!PyUnicode_AsUnicodeAndSize` are marked as deprecated in C. They have been deprecated by :pep:`393` since Python 3.3. (Contributed by Inada Naoki in :issue:`36346`.) diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index 6b601489a7728..1da722b21680e 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -2068,9 +2068,9 @@ casts when the Python C API is used in C++. Patch by Victor Stinner. .. nonce: Cx-95G .. section: C API -Mark functions as deprecated by :pep:`623`: :c:func:`PyUnicode_AS_DATA`, -:c:func:`PyUnicode_AS_UNICODE`, :c:func:`PyUnicode_GET_DATA_SIZE`, -:c:func:`PyUnicode_GET_SIZE`. Patch by Victor Stinner. +Mark functions as deprecated by :pep:`623`: :c:func:`!PyUnicode_AS_DATA`, +:c:func:`!PyUnicode_AS_UNICODE`, :c:func:`!PyUnicode_GET_DATA_SIZE`, +:c:func:`!PyUnicode_GET_SIZE`. Patch by Victor Stinner. .. From webhook-mailer at python.org Mon Aug 21 15:44:12 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 21 Aug 2023 19:44:12 -0000 Subject: [Python-checkins] [3.12] gh-108224: Fix asyncio doc inconsistency (GH-108230) (#108231) Message-ID: https://github.com/python/cpython/commit/f51cdfaaac71f2279156af5ffd042122443699d9 commit: f51cdfaaac71f2279156af5ffd042122443699d9 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-21T21:44:08+02:00 summary: [3.12] gh-108224: Fix asyncio doc inconsistency (GH-108230) (#108231) gh-108224: Fix asyncio doc inconsistency (GH-108230) (Spawning subprocesses does not require the event loop to run in the main thread -- only signal handling does.) (cherry picked from commit 1cc391d9e2ea24ca750005335507b52933fc0b52) Co-authored-by: temach files: M Doc/library/asyncio-dev.rst diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index c7d97008fb490..a9c3a0183bb72 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -99,7 +99,7 @@ To schedule a coroutine object from a different OS thread, the # Wait for the result: result = future.result() -To handle signals and to execute subprocesses, the event loop must be +To handle signals the event loop must be run in the main thread. The :meth:`loop.run_in_executor` method can be used with a From webhook-mailer at python.org Mon Aug 21 15:44:49 2023 From: webhook-mailer at python.org (Yhg1s) Date: Mon, 21 Aug 2023 19:44:49 -0000 Subject: [Python-checkins] [3.12] gh-107298: Fix C API datetime documentation (GH-108034). (#108234) Message-ID: https://github.com/python/cpython/commit/47f60c3f67b2914af0c85227ddffc583cea72791 commit: 47f60c3f67b2914af0c85227ddffc583cea72791 branch: 3.12 author: Serhiy Storchaka committer: Yhg1s date: 2023-08-21T21:44:45+02:00 summary: [3.12] gh-107298: Fix C API datetime documentation (GH-108034). (#108234) (cherry picked from commit d63972e289e05b0d82e59f32f107312a8b3de7b5) files: M Doc/c-api/datetime.rst M Doc/tools/.nitignore diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index 72fc07afbf1f4..97522da773477 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -8,11 +8,54 @@ DateTime Objects Various date and time objects are supplied by the :mod:`datetime` module. Before using any of these functions, the header file :file:`datetime.h` must be included in your source (note that this is not included by :file:`Python.h`), -and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of +and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of the module initialisation function. The macro puts a pointer to a C structure -into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following +into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following macros. +.. c:type:: PyDateTime_Date + + This subtype of :c:type:`PyObject` represents a Python date object. + +.. c:type:: PyDateTime_DateTime + + This subtype of :c:type:`PyObject` represents a Python datetime object. + +.. c:type:: PyDateTime_Time + + This subtype of :c:type:`PyObject` represents a Python time object. + +.. c:type:: PyDateTime_Delta + + This subtype of :c:type:`PyObject` represents the difference between two datetime values. + +.. c:var:: PyTypeObject PyDateTime_DateType + + This instance of :c:type:`PyTypeObject` represents the Python date type; + it is the same object as :class:`datetime.date` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_DateTimeType + + This instance of :c:type:`PyTypeObject` represents the Python datetime type; + it is the same object as :class:`datetime.datetime` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_TimeType + + This instance of :c:type:`PyTypeObject` represents the Python time type; + it is the same object as :class:`datetime.time` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_DeltaType + + This instance of :c:type:`PyTypeObject` represents Python type for + the difference between two datetime values; + it is the same object as :class:`datetime.timedelta` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_TZInfoType + + This instance of :c:type:`PyTypeObject` represents the Python time zone info type; + it is the same object as :class:`datetime.tzinfo` in the Python layer. + + Macro for access to the UTC singleton: .. c:var:: PyObject* PyDateTime_TimeZone_UTC @@ -28,7 +71,7 @@ Type-check macros: .. c:function:: int PyDate_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateType` or a subtype of - :c:data:`PyDateTime_DateType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DateType`. *ob* must not be ``NULL``. This function always succeeds. @@ -41,7 +84,7 @@ Type-check macros: .. c:function:: int PyDateTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType` or a subtype of - :c:data:`PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always succeeds. @@ -54,7 +97,7 @@ Type-check macros: .. c:function:: int PyTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TimeType` or a subtype of - :c:data:`PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always succeeds. @@ -67,7 +110,7 @@ Type-check macros: .. c:function:: int PyDelta_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DeltaType` or a subtype of - :c:data:`PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always succeeds. @@ -80,7 +123,7 @@ Type-check macros: .. c:function:: int PyTZInfo_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType` or a subtype of - :c:data:`PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always succeeds. @@ -133,7 +176,7 @@ Macros to create objects: :class:`datetime.timedelta` objects. -.. c:function:: PyObject* PyTimeZone_FromOffset(PyDateTime_DeltaType* offset) +.. c:function:: PyObject* PyTimeZone_FromOffset(PyObject *offset) Return a :class:`datetime.timezone` object with an unnamed fixed offset represented by the *offset* argument. @@ -141,7 +184,7 @@ Macros to create objects: .. versionadded:: 3.7 -.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyDateTime_DeltaType* offset, PyUnicode* name) +.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyObject *offset, PyObject *name) Return a :class:`datetime.timezone` object with a fixed offset represented by the *offset* argument and with tzname *name*. @@ -150,8 +193,8 @@ Macros to create objects: Macros to extract fields from date objects. The argument must be an instance of -:c:data:`PyDateTime_Date`, including subclasses (such as -:c:data:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is +:c:type:`PyDateTime_Date`, including subclasses (such as +:c:type:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_GET_YEAR(PyDateTime_Date *o) @@ -170,7 +213,7 @@ not checked: Macros to extract fields from datetime objects. The argument must be an -instance of :c:data:`PyDateTime_DateTime`, including subclasses. The argument +instance of :c:type:`PyDateTime_DateTime`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DATE_GET_HOUR(PyDateTime_DateTime *o) @@ -208,7 +251,7 @@ must not be ``NULL``, and the type is not checked: Macros to extract fields from time objects. The argument must be an instance of -:c:data:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``, +:c:type:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_TIME_GET_HOUR(PyDateTime_Time *o) @@ -246,7 +289,7 @@ and the type is not checked: Macros to extract fields from time delta objects. The argument must be an -instance of :c:data:`PyDateTime_Delta`, including subclasses. The argument must +instance of :c:type:`PyDateTime_Delta`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DELTA_GET_DAYS(PyDateTime_Delta *o) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index ada46064e358e..4eb466a909a49 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -3,7 +3,6 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. Doc/c-api/arg.rst -Doc/c-api/datetime.rst Doc/c-api/descriptor.rst Doc/c-api/exceptions.rst Doc/c-api/file.rst From webhook-mailer at python.org Mon Aug 21 16:16:27 2023 From: webhook-mailer at python.org (vstinner) Date: Mon, 21 Aug 2023 20:16:27 -0000 Subject: [Python-checkins] gh-108223: test.pythoninfo and libregrtest log Py_NOGIL (#108238) Message-ID: https://github.com/python/cpython/commit/5afe0c17ca14df430736e549542a4b85e7e7c7ac commit: 5afe0c17ca14df430736e549542a4b85e7e7c7ac branch: main author: Victor Stinner committer: vstinner date: 2023-08-21T22:16:23+02:00 summary: gh-108223: test.pythoninfo and libregrtest log Py_NOGIL (#108238) Enable with --disable-gil --without-pydebug: $ make pythoninfo|grep NOGIL sysconfig[Py_NOGIL]: 1 $ ./python -m test ... == Python build: nogil debug ... files: M Lib/test/libregrtest/utils.py M Lib/test/pythoninfo.py diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index fd46819fd903f..89a149ec5d6b3 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -228,6 +228,11 @@ def get_build_info(): ldflags_nodist = sysconfig.get_config_var('PY_LDFLAGS_NODIST') or '' build = [] + + # --disable-gil + if sysconfig.get_config_var('Py_NOGIL'): + build.append("nogil") + if hasattr(sys, 'gettotalrefcount'): # --with-pydebug build.append('debug') diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index e4e098dd84cfb..ad7d5291af42f 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -492,6 +492,7 @@ def collect_sysconfig(info_add): 'PY_STDMODULE_CFLAGS', 'Py_DEBUG', 'Py_ENABLE_SHARED', + 'Py_NOGIL', 'SHELL', 'SOABI', 'prefix', From webhook-mailer at python.org Mon Aug 21 16:37:11 2023 From: webhook-mailer at python.org (vstinner) Date: Mon, 21 Aug 2023 20:37:11 -0000 Subject: [Python-checkins] gh-108223: Document --disable-gil flag in configure (#108236) Message-ID: https://github.com/python/cpython/commit/4b32d4f49cf1e9b49b072391256dbf5eddba4d93 commit: 4b32d4f49cf1e9b49b072391256dbf5eddba4d93 branch: main author: Sam Gross committer: vstinner date: 2023-08-21T20:37:07Z summary: gh-108223: Document --disable-gil flag in configure (#108236) files: M Doc/using/configure.rst diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index f4adea82a87c3..50f60ee54fed1 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -185,6 +185,15 @@ General Options .. versionadded:: 3.11 +.. cmdoption:: --disable-gil + + Enables **experimental** support for running Python without the + :term:`global interpreter lock` (GIL). + + See :pep:`703` "Making the Global Interpreter Lock Optional in CPython". + + .. versionadded:: 3.13 + WebAssembly Options ------------------- From webhook-mailer at python.org Mon Aug 21 17:21:30 2023 From: webhook-mailer at python.org (gvanrossum) Date: Mon, 21 Aug 2023 21:21:30 -0000 Subject: [Python-checkins] [3.11] gh-108224: Fix asyncio doc inconsistency (GH-108230) (#108232) Message-ID: https://github.com/python/cpython/commit/4d4871e715737bf699cf9773e275c7993ae13779 commit: 4d4871e715737bf699cf9773e275c7993ae13779 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: gvanrossum date: 2023-08-21T14:21:26-07:00 summary: [3.11] gh-108224: Fix asyncio doc inconsistency (GH-108230) (#108232) (Spawning subprocesses does not require the event loop to run in the main thread -- only signal handling does.) (cherry picked from commit 1cc391d9e2ea24ca750005335507b52933fc0b52) Co-authored-by: temach files: M Doc/library/asyncio-dev.rst diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index c7d97008fb490..a9c3a0183bb72 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -99,7 +99,7 @@ To schedule a coroutine object from a different OS thread, the # Wait for the result: result = future.result() -To handle signals and to execute subprocesses, the event loop must be +To handle signals the event loop must be run in the main thread. The :meth:`loop.run_in_executor` method can be used with a From webhook-mailer at python.org Mon Aug 21 17:22:22 2023 From: webhook-mailer at python.org (gvanrossum) Date: Mon, 21 Aug 2023 21:22:22 -0000 Subject: [Python-checkins] gh-107265: Fix code_hash for ENTER_EXECUTOR case (#108188) Message-ID: https://github.com/python/cpython/commit/e6db23f66d8741db0ffc526d8fd75373a5543e3e commit: e6db23f66d8741db0ffc526d8fd75373a5543e3e branch: main author: Dong-hee Na committer: gvanrossum date: 2023-08-21T14:22:18-07:00 summary: gh-107265: Fix code_hash for ENTER_EXECUTOR case (#108188) files: M Lib/test/test_capi/test_misc.py M Objects/codeobject.c diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index ea0504333bab0..1cd4c56b49bba 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2341,7 +2341,7 @@ def long_loop(): long_loop() self.assertEqual(opt.get_count(), 10) - def test_code_richcompare(self): + def test_code_restore_for_ENTER_EXECUTOR(self): def testfunc(x): i = 0 while i < x: @@ -2350,7 +2350,9 @@ def testfunc(x): opt = _testinternalcapi.get_counter_optimizer() with temporary_optimizer(opt): testfunc(1000) - self.assertEqual(testfunc.__code__, testfunc.__code__.replace()) + code, replace_code = testfunc.__code__, testfunc.__code__.replace() + self.assertEqual(code, replace_code) + self.assertEqual(hash(code), hash(replace_code)) def get_first_executor(func): diff --git a/Objects/codeobject.c b/Objects/codeobject.c index c34905c319606..dca5804a91d2c 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1781,30 +1781,33 @@ code_richcompare(PyObject *self, PyObject *other, int op) for (int i = 0; i < Py_SIZE(co); i++) { _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; _Py_CODEUNIT cp_instr = _PyCode_CODE(cp)[i]; + uint8_t co_code = co_instr.op.code; + uint8_t co_arg = co_instr.op.arg; + uint8_t cp_code = cp_instr.op.code; + uint8_t cp_arg = cp_instr.op.arg; - if (co_instr.op.code == ENTER_EXECUTOR) { - const int exec_index = co_instr.op.arg; + if (co_code == ENTER_EXECUTOR) { + const int exec_index = co_arg; _PyExecutorObject *exec = co->co_executors->executors[exec_index]; - co_instr.op.code = exec->vm_data.opcode; - co_instr.op.arg = exec->vm_data.oparg; + co_code = exec->vm_data.opcode; + co_arg = exec->vm_data.oparg; } - assert(co_instr.op.code != ENTER_EXECUTOR); - co_instr.op.code = _PyOpcode_Deopt[co_instr.op.code]; + assert(co_code != ENTER_EXECUTOR); + co_code = _PyOpcode_Deopt[co_code]; - if (cp_instr.op.code == ENTER_EXECUTOR) { - const int exec_index = cp_instr.op.arg; + if (cp_code == ENTER_EXECUTOR) { + const int exec_index = cp_arg; _PyExecutorObject *exec = cp->co_executors->executors[exec_index]; - cp_instr.op.code = exec->vm_data.opcode; - cp_instr.op.arg = exec->vm_data.oparg; + cp_code = exec->vm_data.opcode; + cp_arg = exec->vm_data.oparg; } - assert(cp_instr.op.code != ENTER_EXECUTOR); - cp_instr.op.code = _PyOpcode_Deopt[cp_instr.op.code]; + assert(cp_code != ENTER_EXECUTOR); + cp_code = _PyOpcode_Deopt[cp_code]; - eq = co_instr.cache == cp_instr.cache; - if (!eq) { + if (co_code != cp_code || co_arg != cp_arg) { goto unequal; } - i += _PyOpcode_Caches[co_instr.op.code]; + i += _PyOpcode_Caches[co_code]; } /* compare constants */ @@ -1883,10 +1886,22 @@ code_hash(PyCodeObject *co) SCRAMBLE_IN(co->co_firstlineno); SCRAMBLE_IN(Py_SIZE(co)); for (int i = 0; i < Py_SIZE(co); i++) { - int deop = _Py_GetBaseOpcode(co, i); - SCRAMBLE_IN(deop); - SCRAMBLE_IN(_PyCode_CODE(co)[i].op.arg); - i += _PyOpcode_Caches[deop]; + _Py_CODEUNIT co_instr = _PyCode_CODE(co)[i]; + uint8_t co_code = co_instr.op.code; + uint8_t co_arg = co_instr.op.arg; + if (co_code == ENTER_EXECUTOR) { + _PyExecutorObject *exec = co->co_executors->executors[co_arg]; + assert(exec != NULL); + assert(exec->vm_data.opcode != ENTER_EXECUTOR); + co_code = _PyOpcode_Deopt[exec->vm_data.opcode]; + co_arg = exec->vm_data.oparg; + } + else { + co_code = _Py_GetBaseOpcode(co, i); + } + SCRAMBLE_IN(co_code); + SCRAMBLE_IN(co_arg); + i += _PyOpcode_Caches[co_code]; } if ((Py_hash_t)uhash == -1) { return -2; From webhook-mailer at python.org Mon Aug 21 18:44:34 2023 From: webhook-mailer at python.org (iritkatriel) Date: Mon, 21 Aug 2023 22:44:34 -0000 Subject: [Python-checkins] gh-107901: Fix missing line number on BACKWARD_JUMP at the end of a for loop (#108242) Message-ID: https://github.com/python/cpython/commit/a1cc74c4eebc55795877eb3be019a1bec34402f8 commit: a1cc74c4eebc55795877eb3be019a1bec34402f8 branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-21T23:44:31+01:00 summary: gh-107901: Fix missing line number on BACKWARD_JUMP at the end of a for loop (#108242) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-21-21-13-30.gh-issue-107901.hszvdk.rst M Lib/test/test_compile.py M Python/flowgraph.c diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 2ed8ae0496137..a470d6187aa67 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1031,6 +1031,20 @@ async def test(aseq): code_lines = self.get_code_lines(test.__code__) self.assertEqual(expected_lines, code_lines) + def test_lineno_of_backward_jump(self): + # Issue gh-107901 + def f(): + for i in x: + if y: + pass + + linenos = list(inst.positions.lineno + for inst in dis.get_instructions(f.__code__) + if inst.opname == 'JUMP_BACKWARD') + + self.assertTrue(len(linenos) > 0) + self.assertTrue(all(l is not None for l in linenos)) + def test_big_dict_literal(self): # The compiler has a flushing point in "compiler_dict" that calls compiles # a portion of the dictionary literal when the loop that iterates over the items diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-21-21-13-30.gh-issue-107901.hszvdk.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-21-21-13-30.gh-issue-107901.hszvdk.rst new file mode 100644 index 0000000000000..112e093736dd5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-21-21-13-30.gh-issue-107901.hszvdk.rst @@ -0,0 +1 @@ +Fix missing line number on :opcode:`JUMP_BACKWARD` at the end of a for loop. diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 9d7865661a803..719ed92105074 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -531,7 +531,7 @@ normalize_jumps_in_block(cfg_builder *g, basicblock *b) { if (backwards_jump == NULL) { return ERROR; } - basicblock_addop(backwards_jump, JUMP, target->b_label.id, NO_LOCATION); + basicblock_addop(backwards_jump, JUMP, target->b_label.id, last->i_loc); backwards_jump->b_instr[0].i_target = target; last->i_opcode = reversed_opcode; last->i_target = b->b_next; From webhook-mailer at python.org Mon Aug 21 19:41:17 2023 From: webhook-mailer at python.org (vstinner) Date: Mon, 21 Aug 2023 23:41:17 -0000 Subject: [Python-checkins] Fix test_generators: save/restore warnings filters (#108246) Message-ID: https://github.com/python/cpython/commit/531930f47f6b2a548d31e62cb4ad3e215a24bf53 commit: 531930f47f6b2a548d31e62cb4ad3e215a24bf53 branch: main author: Victor Stinner committer: vstinner date: 2023-08-21T23:41:13Z summary: Fix test_generators: save/restore warnings filters (#108246) Previously, depending on existing filters, the test could modify the warnings and so fail as "env changed". files: M Lib/test/test_generators.py diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index a8a344ab8de48..d48f0d47ba196 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -2258,6 +2258,7 @@ def printsolution(self, x): caught ValueError (xyz) >>> import warnings +>>> old_filters = warnings.filters.copy() >>> warnings.filterwarnings("ignore", category=DeprecationWarning) # Filter DeprecationWarning: regarding the (type, val, tb) signature of throw(). @@ -2331,8 +2332,7 @@ def printsolution(self, x): ... ValueError: 7 ->>> warnings.filters.pop(0) -('ignore', None, , None, 0) +>>> warnings.filters[:] = old_filters # Re-enable DeprecationWarning: the (type, val, tb) exception representation is deprecated, # and may be removed in a future version of Python. From webhook-mailer at python.org Mon Aug 21 20:16:05 2023 From: webhook-mailer at python.org (zooba) Date: Tue, 22 Aug 2023 00:16:05 -0000 Subject: [Python-checkins] gh-106242: Make ntpath.realpath errors consistent with abspath when there are embedded nulls (GH-108248) Message-ID: https://github.com/python/cpython/commit/de33b5c662ea8d35d81ed857c6a39e34ab94c510 commit: de33b5c662ea8d35d81ed857c6a39e34ab94c510 branch: main author: Steve Dower committer: zooba date: 2023-08-22T00:16:02Z summary: gh-106242: Make ntpath.realpath errors consistent with abspath when there are embedded nulls (GH-108248) * gh-106242: Make ntpath.realpath errors consistent with abspath when there are embedded nulls * Update 2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst mention Windows and the former incorrect ValueError. --------- Co-authored-by: Gregory P. Smith files: A Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst M Lib/ntpath.py M Lib/test/test_ntpath.py diff --git a/Lib/ntpath.py b/Lib/ntpath.py index dadcdc0c495da..df3402d46c9cc 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -721,6 +721,14 @@ def realpath(path, *, strict=False): try: path = _getfinalpathname(path) initial_winerror = 0 + except ValueError as ex: + # gh-106242: Raised for embedded null characters + # In strict mode, we convert into an OSError. + # Non-strict mode returns the path as-is, since we've already + # made it absolute. + if strict: + raise OSError(str(ex)) from None + path = normpath(path) except OSError as ex: if strict: raise @@ -740,6 +748,10 @@ def realpath(path, *, strict=False): try: if _getfinalpathname(spath) == path: path = spath + except ValueError as ex: + # Unexpected, as an invalid path should not have gained a prefix + # at any point, but we ignore this error just in case. + pass except OSError as ex: # If the path does not exist and originally did not exist, then # strip the prefix anyway. diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 78e1cb582512b..d91dcdfb0c5fa 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -394,6 +394,10 @@ def test_realpath_basic(self): d = drives.pop().encode() self.assertEqual(ntpath.realpath(d), d) + # gh-106242: Embedded nulls and non-strict fallback to abspath + self.assertEqual(ABSTFN + "\0spam", + ntpath.realpath(os_helper.TESTFN + "\0spam", strict=False)) + @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_strict(self): @@ -404,6 +408,8 @@ def test_realpath_strict(self): self.addCleanup(os_helper.unlink, ABSTFN) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN, strict=True) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN + "2", strict=True) + # gh-106242: Embedded nulls should raise OSError (not ValueError) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "\0spam", strict=True) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') diff --git a/Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst b/Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst new file mode 100644 index 0000000000000..ffe42ec5dc3fa --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst @@ -0,0 +1,4 @@ +Fixes :func:`~os.path.realpath` to behave consistently when passed a path +containing an embedded null character on Windows. In strict mode, it now +raises :exc:`OSError` instead of the unexpected :exc:`ValueError`, and in +non-strict mode will make the path absolute. From webhook-mailer at python.org Mon Aug 21 20:37:36 2023 From: webhook-mailer at python.org (vstinner) Date: Tue, 22 Aug 2023 00:37:36 -0000 Subject: [Python-checkins] Define _Py_NULL as nullptr on C23 and newer (#108244) Message-ID: https://github.com/python/cpython/commit/c965cf6dd1704a0138a4ef0a9c670e297cf66797 commit: c965cf6dd1704a0138a4ef0a9c670e297cf66797 branch: main author: Victor Stinner committer: vstinner date: 2023-08-22T02:37:32+02:00 summary: Define _Py_NULL as nullptr on C23 and newer (#108244) C23 standard added nullptr constant: https://en.wikipedia.org/wiki/C23_(C_standard_revision) files: M Include/pyport.h diff --git a/Include/pyport.h b/Include/pyport.h index d7c6ae64f2bf2..2dc241389243d 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -24,9 +24,10 @@ #define _Py_CAST(type, expr) ((type)(expr)) // Static inline functions should use _Py_NULL rather than using directly NULL -// to prevent C++ compiler warnings. On C++11 and newer, _Py_NULL is defined as -// nullptr. -#if defined(__cplusplus) && __cplusplus >= 201103 +// to prevent C++ compiler warnings. On C23 and newer and on C++11 and newer, +// _Py_NULL is defined as nullptr. +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \ + || (defined(__cplusplus) && __cplusplus >= 201103) # define _Py_NULL nullptr #else # define _Py_NULL NULL From webhook-mailer at python.org Mon Aug 21 21:16:15 2023 From: webhook-mailer at python.org (vstinner) Date: Tue, 22 Aug 2023 01:16:15 -0000 Subject: [Python-checkins] Fix test_faulthandler for sanitizers (#108245) Message-ID: https://github.com/python/cpython/commit/58f9c635002673e35a56eeaa122cade7a2b6bae1 commit: 58f9c635002673e35a56eeaa122cade7a2b6bae1 branch: main author: Victor Stinner committer: vstinner date: 2023-08-22T01:16:12Z summary: Fix test_faulthandler for sanitizers (#108245) Set environment options to ask sanitizers to not handle SIGSEGV. This change allows running test_enable_fd() and test_enable_file() with sanitizers. Previously, they were skipped. files: M Lib/test/test_faulthandler.py diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 2e97de592712c..907c2cda86cba 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -9,7 +9,6 @@ from test import support from test.support import os_helper from test.support import script_helper, is_android -from test.support import skip_if_sanitizer import tempfile import unittest from textwrap import dedent @@ -64,8 +63,20 @@ def get_output(self, code, filename=None, fd=None): pass_fds = [] if fd is not None: pass_fds.append(fd) + env = dict(os.environ) + + # Sanitizers must not handle SIGSEGV (ex: for test_enable_fd()) + option = 'handle_segv=0' + for name in ('ASAN_OPTIONS', 'MSAN_OPTIONS', 'UBSAN_OPTIONS'): + if name in env: + env[name] += f':{option}' + else: + env[name] = option + with support.SuppressCrashReport(): - process = script_helper.spawn_python('-c', code, pass_fds=pass_fds) + process = script_helper.spawn_python('-c', code, + pass_fds=pass_fds, + env=env) with process: output, stderr = process.communicate() exitcode = process.wait() @@ -304,8 +315,6 @@ def test_gil_released(self): 3, 'Segmentation fault') - @skip_if_sanitizer(memory=True, ub=True, reason="sanitizer " - "builds change crashing process output.") @skip_segfault_on_android def test_enable_file(self): with temporary_filename() as filename: @@ -321,8 +330,6 @@ def test_enable_file(self): @unittest.skipIf(sys.platform == "win32", "subprocess doesn't support pass_fds on Windows") - @skip_if_sanitizer(memory=True, ub=True, reason="sanitizer " - "builds change crashing process output.") @skip_segfault_on_android def test_enable_fd(self): with tempfile.TemporaryFile('wb+') as fp: From webhook-mailer at python.org Tue Aug 22 03:37:57 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 22 Aug 2023 07:37:57 -0000 Subject: [Python-checkins] gh-107801: Improve the accuracy of io.TextIOWrapper.seek docs (#107933) Message-ID: https://github.com/python/cpython/commit/7f87ebbc3f52680c939791f397b9a478edf0c8d4 commit: 7f87ebbc3f52680c939791f397b9a478edf0c8d4 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-22T09:37:53+02:00 summary: gh-107801: Improve the accuracy of io.TextIOWrapper.seek docs (#107933) Clearly document the supported seek() operations: - Rewind to the start of the stream - Restore a previous stream position (given by tell()) - Fast-forward to the end of the stream files: M Doc/library/io.rst M Modules/_io/clinic/textio.c.h M Modules/_io/textio.c diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 66273d9ed1ff0..792bf43d9811b 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -1044,6 +1044,22 @@ Text I/O .. versionchanged:: 3.11 The method supports ``encoding="locale"`` option. + .. method:: seek(cookie, whence=os.SEEK_SET, /) + + Set the stream position. + Return the new stream position as an :class:`int`. + + Four operations are supported, + given by the following argument combinations: + + * ``seek(0, SEEK_SET)``: Rewind to the start of the stream. + * ``seek(cookie, SEEK_SET)``: Restore a previous position; + *cookie* **must be** a number returned by :meth:`!tell`. + * ``seek(0, SEEK_END)``: Fast-forward to the end of the stream. + * ``seek(0, SEEK_CUR)``: Leave the current stream position unchanged. + + Any other argument combinations are invalid, + and may raise exceptions. .. class:: StringIO(initial_value='', newline='\n') diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 5c2162529ff2d..9bccd71e1d3ea 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -756,9 +756,27 @@ _io_TextIOWrapper_readline(textio *self, PyObject *const *args, Py_ssize_t nargs } PyDoc_STRVAR(_io_TextIOWrapper_seek__doc__, -"seek($self, cookie, whence=0, /)\n" +"seek($self, cookie, whence=os.SEEK_SET, /)\n" "--\n" -"\n"); +"\n" +"Set the stream position, and return the new stream position.\n" +"\n" +" cookie\n" +" Zero or an opaque number returned by tell().\n" +" whence\n" +" The relative position to seek from.\n" +"\n" +"Four operations are supported, given by the following argument\n" +"combinations:\n" +"\n" +"- seek(0, SEEK_SET): Rewind to the start of the stream.\n" +"- seek(cookie, SEEK_SET): Restore a previous position;\n" +" \'cookie\' must be a number returned by tell().\n" +"- seek(0, SEEK_END): Fast-forward to the end of the stream.\n" +"- seek(0, SEEK_CUR): Leave the current stream position unchanged.\n" +"\n" +"Any other argument combinations are invalid,\n" +"and may raise exceptions."); #define _IO_TEXTIOWRAPPER_SEEK_METHODDEF \ {"seek", _PyCFunction_CAST(_io_TextIOWrapper_seek), METH_FASTCALL, _io_TextIOWrapper_seek__doc__}, @@ -957,4 +975,4 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) { return _io_TextIOWrapper_close_impl(self); } -/*[clinic end generated code: output=e1060638b65e8a63 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6bd981a58fcbc778 input=a9049054013a1b77]*/ diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 6ce90b2ed774c..d87bd0054d443 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2428,13 +2428,29 @@ _textiowrapper_encoder_setstate(textio *self, cookie_type *cookie) /*[clinic input] _io.TextIOWrapper.seek cookie as cookieObj: object - whence: int = 0 + Zero or an opaque number returned by tell(). + whence: int(c_default='0') = os.SEEK_SET + The relative position to seek from. / + +Set the stream position, and return the new stream position. + +Four operations are supported, given by the following argument +combinations: + +- seek(0, SEEK_SET): Rewind to the start of the stream. +- seek(cookie, SEEK_SET): Restore a previous position; + 'cookie' must be a number returned by tell(). +- seek(0, SEEK_END): Fast-forward to the end of the stream. +- seek(0, SEEK_CUR): Leave the current stream position unchanged. + +Any other argument combinations are invalid, +and may raise exceptions. [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence) -/*[clinic end generated code: output=0a15679764e2d04d input=0458abeb3d7842be]*/ +/*[clinic end generated code: output=0a15679764e2d04d input=0f68adcb02cf2823]*/ { PyObject *posobj; cookie_type cookie; From webhook-mailer at python.org Tue Aug 22 03:41:54 2023 From: webhook-mailer at python.org (pablogsal) Date: Tue, 22 Aug 2023 07:41:54 -0000 Subject: [Python-checkins] gh-108179: Add error message for parser stack overflows (#108256) Message-ID: https://github.com/python/cpython/commit/86617518c4ac824e2b6dc20691ba5a08df04f285 commit: 86617518c4ac824e2b6dc20691ba5a08df04f285 branch: main author: Dennis Sweeney <36520290+sweeneyde at users.noreply.github.com> committer: pablogsal date: 2023-08-22T08:41:50+01:00 summary: gh-108179: Add error message for parser stack overflows (#108256) files: M Lib/test/test_syntax.py M Parser/parser.c M Parser/pegen.h M Parser/pegen_errors.c M Tools/peg_generator/pegen/c_generator.py diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index f3d6cd7bad0ee..4c988382f8b41 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -2335,7 +2335,7 @@ def test_error_on_parser_stack_overflow(self): source = "-" * 100000 + "4" for mode in ["exec", "eval", "single"]: with self.subTest(mode=mode): - with self.assertRaises(MemoryError): + with self.assertRaisesRegex(MemoryError, r"too complex"): compile(source, "", mode) @support.cpython_only diff --git a/Parser/parser.c b/Parser/parser.c index 44312cff125ae..95cbef4a64a10 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -1128,8 +1128,7 @@ static mod_ty file_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1175,8 +1174,7 @@ static mod_ty interactive_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1219,8 +1217,7 @@ static mod_ty eval_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1269,8 +1266,7 @@ static mod_ty func_type_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1331,8 +1327,7 @@ static expr_ty fstring_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1381,8 +1376,7 @@ static asdl_stmt_seq* statements_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1425,8 +1419,7 @@ static asdl_stmt_seq* statement_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1493,8 +1486,7 @@ static asdl_stmt_seq* statement_newline_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1625,8 +1617,7 @@ static asdl_stmt_seq* simple_stmts_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1719,8 +1710,7 @@ static stmt_ty simple_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2101,8 +2091,7 @@ static stmt_ty compound_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2292,8 +2281,7 @@ static stmt_ty assignment_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2511,8 +2499,7 @@ static expr_ty annotated_rhs_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2582,8 +2569,7 @@ static AugOperator* augassign_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2914,8 +2900,7 @@ static stmt_ty return_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2979,8 +2964,7 @@ static stmt_ty raise_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3080,8 +3064,7 @@ static stmt_ty global_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3145,8 +3128,7 @@ static stmt_ty nonlocal_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3210,8 +3192,7 @@ static stmt_ty del_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3296,8 +3277,7 @@ static stmt_ty yield_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3358,8 +3338,7 @@ static stmt_ty assert_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3426,8 +3405,7 @@ static stmt_ty import_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3503,8 +3481,7 @@ static stmt_ty import_name_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3570,8 +3547,7 @@ static stmt_ty import_from_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3690,8 +3666,7 @@ static asdl_alias_seq* import_from_targets_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3826,8 +3801,7 @@ static asdl_alias_seq* import_from_as_names_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3870,8 +3844,7 @@ static alias_ty import_from_as_name_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3935,8 +3908,7 @@ static asdl_alias_seq* dotted_as_names_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3979,8 +3951,7 @@ static alias_ty dotted_as_name_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4046,8 +4017,7 @@ static expr_ty dotted_name_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, dotted_name_type, &_res)) { @@ -4081,8 +4051,7 @@ static expr_ty dotted_name_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4150,8 +4119,7 @@ static asdl_stmt_seq* block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4246,8 +4214,7 @@ static asdl_expr_seq* decorators_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4290,8 +4257,7 @@ static stmt_ty class_def_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4358,8 +4324,7 @@ static stmt_ty class_def_raw_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4454,8 +4419,7 @@ static stmt_ty function_def_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4523,8 +4487,7 @@ static stmt_ty function_def_raw_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4694,8 +4657,7 @@ static arguments_ty params_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4757,8 +4719,7 @@ static arguments_ty parameters_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4921,8 +4882,7 @@ static asdl_arg_seq* slash_no_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5002,8 +4962,7 @@ static SlashWithDefault* slash_with_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5092,8 +5051,7 @@ static StarEtc* star_etc_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5254,8 +5212,7 @@ static arg_ty kwds_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5320,8 +5277,7 @@ static arg_ty param_no_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5401,8 +5357,7 @@ static arg_ty param_no_default_star_annotation_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5480,8 +5435,7 @@ static NameDefaultPair* param_with_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5567,8 +5521,7 @@ static NameDefaultPair* param_maybe_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5652,8 +5605,7 @@ static arg_ty param_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5717,8 +5669,7 @@ static arg_ty param_star_annotation_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5782,8 +5733,7 @@ static expr_ty annotation_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5829,8 +5779,7 @@ static expr_ty star_annotation_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5876,8 +5825,7 @@ static expr_ty default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5945,8 +5893,7 @@ static stmt_ty if_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6086,8 +6033,7 @@ static stmt_ty elif_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6224,8 +6170,7 @@ static asdl_stmt_seq* else_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6293,8 +6238,7 @@ static stmt_ty while_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6390,8 +6334,7 @@ static stmt_ty for_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6588,8 +6531,7 @@ static stmt_ty with_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6858,8 +6800,7 @@ static withitem_ty with_item_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6957,8 +6898,7 @@ static stmt_ty try_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7147,8 +7087,7 @@ static excepthandler_ty except_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7301,8 +7240,7 @@ static excepthandler_ty except_star_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7416,8 +7354,7 @@ static asdl_stmt_seq* finally_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7487,8 +7424,7 @@ static stmt_ty match_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7586,8 +7522,7 @@ static expr_ty subject_expr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7673,8 +7608,7 @@ static match_case_ty case_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7748,8 +7682,7 @@ static expr_ty guard_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7795,8 +7728,7 @@ static pattern_ty patterns_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7876,8 +7808,7 @@ static pattern_ty pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7934,8 +7865,7 @@ static pattern_ty as_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8021,8 +7951,7 @@ static pattern_ty or_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8091,8 +8020,7 @@ static pattern_ty closed_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8274,8 +8202,7 @@ static pattern_ty literal_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8509,8 +8436,7 @@ static expr_ty literal_expr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8698,8 +8624,7 @@ static expr_ty complex_number_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8805,8 +8730,7 @@ static expr_ty signed_number_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8889,8 +8813,7 @@ static expr_ty signed_real_number_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8973,8 +8896,7 @@ static expr_ty real_number_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9017,8 +8939,7 @@ static expr_ty imaginary_number_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9061,8 +8982,7 @@ static pattern_ty capture_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9123,8 +9043,7 @@ static expr_ty pattern_capture_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9171,8 +9090,7 @@ static pattern_ty wildcard_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9233,8 +9151,7 @@ static pattern_ty value_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9299,8 +9216,7 @@ static expr_ty attr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, attr_type, &_res)) { @@ -9334,8 +9250,7 @@ static expr_ty attr_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9403,8 +9318,7 @@ static expr_ty name_or_attr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9461,8 +9375,7 @@ static pattern_ty group_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9511,8 +9424,7 @@ static pattern_ty sequence_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9618,8 +9530,7 @@ static asdl_seq* open_sequence_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9668,8 +9579,7 @@ static asdl_seq* maybe_sequence_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9716,8 +9626,7 @@ static pattern_ty maybe_star_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9774,8 +9683,7 @@ static pattern_ty star_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9884,8 +9792,7 @@ static pattern_ty mapping_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10084,8 +9991,7 @@ static asdl_seq* items_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10123,8 +10029,7 @@ static KeyPatternPair* key_value_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10173,8 +10078,7 @@ static expr_ty double_star_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10225,8 +10129,7 @@ static pattern_ty class_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10456,8 +10359,7 @@ static asdl_pattern_seq* positional_patterns_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10500,8 +10402,7 @@ static asdl_seq* keyword_patterns_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10539,8 +10440,7 @@ static KeyPatternPair* keyword_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10589,8 +10489,7 @@ static stmt_ty type_alias_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10663,8 +10562,7 @@ static asdl_type_param_seq* type_params_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10713,8 +10611,7 @@ static asdl_type_param_seq* type_param_seq_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10766,8 +10663,7 @@ static type_param_ty type_param_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10974,8 +10870,7 @@ static expr_ty type_param_bound_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11021,8 +10916,7 @@ static expr_ty expressions_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11150,8 +11044,7 @@ static expr_ty expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11305,8 +11198,7 @@ static expr_ty yield_expr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11412,8 +11304,7 @@ static expr_ty star_expressions_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11536,8 +11427,7 @@ static expr_ty star_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11625,8 +11515,7 @@ static asdl_expr_seq* star_named_expressions_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11673,8 +11562,7 @@ static expr_ty star_named_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11757,8 +11645,7 @@ static expr_ty assignment_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11832,8 +11719,7 @@ static expr_ty named_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11911,8 +11797,7 @@ static expr_ty disjunction_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12000,8 +11885,7 @@ static expr_ty conjunction_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12089,8 +11973,7 @@ static expr_ty inversion_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12178,8 +12061,7 @@ static expr_ty comparison_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12272,8 +12154,7 @@ static CmpopExprPair* compare_op_bitwise_or_pair_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12482,8 +12363,7 @@ static CmpopExprPair* eq_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12529,8 +12409,7 @@ static CmpopExprPair* noteq_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12576,8 +12455,7 @@ static CmpopExprPair* lte_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12623,8 +12501,7 @@ static CmpopExprPair* lt_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12670,8 +12547,7 @@ static CmpopExprPair* gte_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12717,8 +12593,7 @@ static CmpopExprPair* gt_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12764,8 +12639,7 @@ static CmpopExprPair* notin_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12814,8 +12688,7 @@ static CmpopExprPair* in_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12861,8 +12734,7 @@ static CmpopExprPair* isnot_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12911,8 +12783,7 @@ static CmpopExprPair* is_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12960,8 +12831,7 @@ static expr_ty bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, bitwise_or_type, &_res)) { @@ -12995,8 +12865,7 @@ static expr_ty bitwise_or_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13084,8 +12953,7 @@ static expr_ty bitwise_xor_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, bitwise_xor_type, &_res)) { @@ -13119,8 +12987,7 @@ static expr_ty bitwise_xor_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13208,8 +13075,7 @@ static expr_ty bitwise_and_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, bitwise_and_type, &_res)) { @@ -13243,8 +13109,7 @@ static expr_ty bitwise_and_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13332,8 +13197,7 @@ static expr_ty shift_expr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, shift_expr_type, &_res)) { @@ -13367,8 +13231,7 @@ static expr_ty shift_expr_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13495,8 +13358,7 @@ static expr_ty sum_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, sum_type, &_res)) { @@ -13530,8 +13392,7 @@ static expr_ty sum_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13664,8 +13525,7 @@ static expr_ty term_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, term_type, &_res)) { @@ -13699,8 +13559,7 @@ static expr_ty term_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13942,8 +13801,7 @@ static expr_ty factor_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14103,8 +13961,7 @@ static expr_ty power_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14190,8 +14047,7 @@ static expr_ty await_primary_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14286,8 +14142,7 @@ static expr_ty primary_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, primary_type, &_res)) { @@ -14321,8 +14176,7 @@ static expr_ty primary_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14528,8 +14382,7 @@ static expr_ty slices_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14620,8 +14473,7 @@ static expr_ty slice_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14725,8 +14577,7 @@ static expr_ty atom_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15008,8 +14859,7 @@ static expr_ty group_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15077,8 +14927,7 @@ static expr_ty lambdef_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15148,8 +14997,7 @@ static arguments_ty lambda_params_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15211,8 +15059,7 @@ static arguments_ty lambda_parameters_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15377,8 +15224,7 @@ static asdl_arg_seq* lambda_slash_no_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15458,8 +15304,7 @@ static SlashWithDefault* lambda_slash_with_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15547,8 +15392,7 @@ static StarEtc* lambda_star_etc_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15676,8 +15520,7 @@ static arg_ty lambda_kwds_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15742,8 +15585,7 @@ static arg_ty lambda_param_no_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15815,8 +15657,7 @@ static NameDefaultPair* lambda_param_with_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15894,8 +15735,7 @@ static NameDefaultPair* lambda_param_maybe_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15973,8 +15813,7 @@ static arg_ty lambda_param_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16035,8 +15874,7 @@ static expr_ty fstring_middle_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16100,8 +15938,7 @@ static expr_ty fstring_replacement_field_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16196,8 +16033,7 @@ static ResultTokenWithMetadata* fstring_conversion_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16243,8 +16079,7 @@ static ResultTokenWithMetadata* fstring_full_format_spec_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16308,8 +16143,7 @@ static expr_ty fstring_format_spec_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16371,8 +16205,7 @@ static expr_ty string_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16415,8 +16248,7 @@ static expr_ty strings_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16482,8 +16314,7 @@ static expr_ty list_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16550,8 +16381,7 @@ static expr_ty tuple_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16618,8 +16448,7 @@ static expr_ty set_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16686,8 +16515,7 @@ static expr_ty dict_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16779,8 +16607,7 @@ static asdl_seq* double_starred_kvpairs_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16827,8 +16654,7 @@ static KeyValuePair* double_starred_kvpair_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16893,8 +16719,7 @@ static KeyValuePair* kvpair_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16943,8 +16768,7 @@ static asdl_comprehension_seq* for_if_clauses_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16990,8 +16814,7 @@ static comprehension_ty for_if_clause_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17118,8 +16941,7 @@ static expr_ty listcomp_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17208,8 +17030,7 @@ static expr_ty setcomp_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17300,8 +17121,7 @@ static expr_ty genexp_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17390,8 +17210,7 @@ static expr_ty dictcomp_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17480,8 +17299,7 @@ static expr_ty arguments_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17556,8 +17374,7 @@ static expr_ty args_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17657,8 +17474,7 @@ static asdl_seq* kwargs_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17745,8 +17561,7 @@ static expr_ty starred_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17829,8 +17644,7 @@ static KeywordOrStarred* kwarg_or_starred_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17940,8 +17754,7 @@ static KeywordOrStarred* kwarg_or_double_starred_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18063,8 +17876,7 @@ static expr_ty star_targets_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18158,8 +17970,7 @@ static asdl_expr_seq* star_targets_list_seq_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18206,8 +18017,7 @@ static asdl_expr_seq* star_targets_tuple_seq_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18284,8 +18094,7 @@ static expr_ty star_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18376,8 +18185,7 @@ static expr_ty target_with_star_atom_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18518,8 +18326,7 @@ static expr_ty star_atom_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18679,8 +18486,7 @@ static expr_ty single_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18774,8 +18580,7 @@ static expr_ty single_subscript_attribute_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18895,8 +18700,7 @@ static expr_ty t_primary_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, t_primary_type, &_res)) { @@ -18930,8 +18734,7 @@ static expr_ty t_primary_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19152,8 +18955,7 @@ static void * t_lookahead_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19229,8 +19031,7 @@ static asdl_expr_seq* del_targets_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19280,8 +19081,7 @@ static expr_ty del_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19418,8 +19218,7 @@ static expr_ty del_t_atom_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19586,8 +19385,7 @@ static asdl_expr_seq* type_expressions_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19831,8 +19629,7 @@ static Token* func_type_comment_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19925,8 +19722,7 @@ static void * invalid_arguments_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20169,8 +19965,7 @@ static void * invalid_kwarg_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20316,8 +20111,7 @@ expression_without_invalid_rule(Parser *p) int _prev_call_invalid = p->call_invalid_rules; p->call_invalid_rules = 0; if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->call_invalid_rules = _prev_call_invalid; @@ -20436,8 +20230,7 @@ static void * invalid_legacy_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20488,8 +20281,7 @@ static void * invalid_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20605,8 +20397,7 @@ static void * invalid_named_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20732,8 +20523,7 @@ static void * invalid_assignment_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20938,8 +20728,7 @@ static expr_ty invalid_ann_assign_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21026,8 +20815,7 @@ static void * invalid_del_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21073,8 +20861,7 @@ static void * invalid_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21122,8 +20909,7 @@ static void * invalid_comprehension_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21241,8 +21027,7 @@ static void * invalid_dict_comprehension_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21303,8 +21088,7 @@ static void * invalid_parameters_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21521,8 +21305,7 @@ static void * invalid_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21571,8 +21354,7 @@ static void * invalid_star_etc_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21714,8 +21496,7 @@ static void * invalid_kwds_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21830,8 +21611,7 @@ static void * invalid_parameters_helper_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21899,8 +21679,7 @@ static void * invalid_lambda_parameters_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22119,8 +21898,7 @@ static void * invalid_lambda_parameters_helper_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22185,8 +21963,7 @@ static void * invalid_lambda_star_etc_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22301,8 +22078,7 @@ static void * invalid_lambda_kwds_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22417,8 +22193,7 @@ static void * invalid_double_type_comments_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22473,8 +22248,7 @@ static void * invalid_with_item_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22525,8 +22299,7 @@ static void * invalid_for_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22576,8 +22349,7 @@ static void * invalid_group_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22659,8 +22431,7 @@ static void * invalid_import_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22712,8 +22483,7 @@ static void * invalid_import_from_targets_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22764,8 +22534,7 @@ static void * invalid_with_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22864,8 +22633,7 @@ static void * invalid_with_stmt_indent_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22976,8 +22744,7 @@ static void * invalid_try_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23156,8 +22923,7 @@ static void * invalid_except_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23315,8 +23081,7 @@ static void * invalid_finally_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23369,8 +23134,7 @@ static void * invalid_except_stmt_indent_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23461,8 +23225,7 @@ static void * invalid_except_star_stmt_indent_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23525,8 +23288,7 @@ static void * invalid_match_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23612,8 +23374,7 @@ static void * invalid_case_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23705,8 +23466,7 @@ static void * invalid_as_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23787,8 +23547,7 @@ static void * invalid_class_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23838,8 +23597,7 @@ static asdl_pattern_seq* invalid_class_argument_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23894,8 +23652,7 @@ static void * invalid_if_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23981,8 +23738,7 @@ static void * invalid_elif_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24066,8 +23822,7 @@ static void * invalid_else_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24120,8 +23875,7 @@ static void * invalid_while_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24207,8 +23961,7 @@ static void * invalid_for_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24313,8 +24066,7 @@ static void * invalid_def_raw_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24388,8 +24140,7 @@ static void * invalid_class_def_raw_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24484,8 +24235,7 @@ static void * invalid_double_starred_kvpairs_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24594,8 +24344,7 @@ static void * invalid_kvpair_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24702,8 +24451,7 @@ static void * invalid_starred_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24766,8 +24514,7 @@ static void * invalid_replacement_field_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25127,8 +24874,7 @@ static void * invalid_conversion_character_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25199,8 +24945,7 @@ static asdl_seq * _loop0_1_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25267,8 +25012,7 @@ static asdl_seq * _loop0_2_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25335,8 +25079,7 @@ static asdl_seq * _loop0_3_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25403,8 +25146,7 @@ static asdl_seq * _loop1_4_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25476,8 +25218,7 @@ static asdl_seq * _loop0_6_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25553,8 +25294,7 @@ static asdl_seq * _gather_5_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25595,8 +25335,7 @@ static void * _tmp_7_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25653,8 +25392,7 @@ static void * _tmp_8_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25730,8 +25468,7 @@ static void * _tmp_9_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25788,8 +25525,7 @@ static void * _tmp_10_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25846,8 +25582,7 @@ static void * _tmp_11_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25904,8 +25639,7 @@ static void * _tmp_12_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25951,8 +25685,7 @@ static void * _tmp_13_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26020,8 +25753,7 @@ static void * _tmp_14_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26067,8 +25799,7 @@ static asdl_seq * _loop1_15_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26140,8 +25871,7 @@ static void * _tmp_16_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26198,8 +25928,7 @@ static void * _tmp_17_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26256,8 +25985,7 @@ static void * _tmp_18_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26303,8 +26031,7 @@ static asdl_seq * _loop0_20_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26380,8 +26107,7 @@ static asdl_seq * _gather_19_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26422,8 +26148,7 @@ static asdl_seq * _loop0_22_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26499,8 +26224,7 @@ static asdl_seq * _gather_21_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26541,8 +26265,7 @@ static void * _tmp_23_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26599,8 +26322,7 @@ static void * _tmp_24_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26646,8 +26368,7 @@ static asdl_seq * _loop0_25_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26714,8 +26435,7 @@ static asdl_seq * _loop1_26_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26787,8 +26507,7 @@ static asdl_seq * _loop0_28_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26864,8 +26583,7 @@ static asdl_seq * _gather_27_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26906,8 +26624,7 @@ static void * _tmp_29_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26953,8 +26670,7 @@ static asdl_seq * _loop0_31_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27030,8 +26746,7 @@ static asdl_seq * _gather_30_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27072,8 +26787,7 @@ static void * _tmp_32_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27119,8 +26833,7 @@ static asdl_seq * _loop1_33_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27192,8 +26905,7 @@ static void * _tmp_34_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27242,8 +26954,7 @@ static void * _tmp_35_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27289,8 +27000,7 @@ static void * _tmp_36_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27336,8 +27046,7 @@ static asdl_seq * _loop0_37_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27404,8 +27113,7 @@ static asdl_seq * _loop0_38_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27472,8 +27180,7 @@ static asdl_seq * _loop0_39_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27540,8 +27247,7 @@ static asdl_seq * _loop1_40_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27613,8 +27319,7 @@ static asdl_seq * _loop0_41_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27681,8 +27386,7 @@ static asdl_seq * _loop1_42_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27754,8 +27458,7 @@ static asdl_seq * _loop1_43_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27827,8 +27530,7 @@ static asdl_seq * _loop1_44_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27900,8 +27602,7 @@ static asdl_seq * _loop0_45_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27968,8 +27669,7 @@ static asdl_seq * _loop1_46_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28041,8 +27741,7 @@ static asdl_seq * _loop0_47_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28109,8 +27808,7 @@ static asdl_seq * _loop1_48_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28182,8 +27880,7 @@ static asdl_seq * _loop0_49_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28250,8 +27947,7 @@ static asdl_seq * _loop0_50_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28318,8 +28014,7 @@ static asdl_seq * _loop1_51_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28391,8 +28086,7 @@ static asdl_seq * _loop0_53_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28468,8 +28162,7 @@ static asdl_seq * _gather_52_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28510,8 +28203,7 @@ static asdl_seq * _loop0_55_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28587,8 +28279,7 @@ static asdl_seq * _gather_54_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28629,8 +28320,7 @@ static asdl_seq * _loop0_57_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28706,8 +28396,7 @@ static asdl_seq * _gather_56_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28748,8 +28437,7 @@ static asdl_seq * _loop0_59_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28825,8 +28513,7 @@ static asdl_seq * _gather_58_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28867,8 +28554,7 @@ static void * _tmp_60_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28944,8 +28630,7 @@ static asdl_seq * _loop1_61_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29017,8 +28702,7 @@ static asdl_seq * _loop1_62_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29090,8 +28774,7 @@ static void * _tmp_63_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29137,8 +28820,7 @@ static void * _tmp_64_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29184,8 +28866,7 @@ static asdl_seq * _loop1_65_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29257,8 +28938,7 @@ static asdl_seq * _loop0_67_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29334,8 +29014,7 @@ static asdl_seq * _gather_66_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29376,8 +29055,7 @@ static void * _tmp_68_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29434,8 +29112,7 @@ static void * _tmp_69_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29492,8 +29169,7 @@ static void * _tmp_70_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29569,8 +29245,7 @@ static void * _tmp_71_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29646,8 +29321,7 @@ static asdl_seq * _loop0_73_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29723,8 +29397,7 @@ static asdl_seq * _gather_72_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29765,8 +29438,7 @@ static asdl_seq * _loop0_75_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29842,8 +29514,7 @@ static asdl_seq * _gather_74_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29884,8 +29555,7 @@ static void * _tmp_76_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29942,8 +29612,7 @@ static asdl_seq * _loop0_78_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30019,8 +29688,7 @@ static asdl_seq * _gather_77_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30061,8 +29729,7 @@ static asdl_seq * _loop0_80_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30138,8 +29805,7 @@ static asdl_seq * _gather_79_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30180,8 +29846,7 @@ static asdl_seq * _loop0_82_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30257,8 +29922,7 @@ static asdl_seq * _gather_81_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30299,8 +29963,7 @@ static asdl_seq * _loop1_83_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30372,8 +30035,7 @@ static asdl_seq * _loop1_84_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30445,8 +30107,7 @@ static asdl_seq * _loop0_86_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30522,8 +30183,7 @@ static asdl_seq * _gather_85_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30564,8 +30224,7 @@ static asdl_seq * _loop1_87_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30637,8 +30296,7 @@ static asdl_seq * _loop1_88_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30710,8 +30368,7 @@ static asdl_seq * _loop1_89_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30783,8 +30440,7 @@ static void * _tmp_90_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30827,8 +30483,7 @@ static asdl_seq * _loop0_92_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30904,8 +30559,7 @@ static asdl_seq * _gather_91_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30946,8 +30600,7 @@ static void * _tmp_93_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30993,8 +30646,7 @@ static void * _tmp_94_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31051,8 +30703,7 @@ static void * _tmp_95_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31128,8 +30779,7 @@ static void * _tmp_96_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31186,8 +30836,7 @@ static void * _tmp_97_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31282,8 +30931,7 @@ static void * _tmp_98_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31340,8 +30988,7 @@ static asdl_seq * _loop0_99_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31408,8 +31055,7 @@ static asdl_seq * _loop0_100_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31476,8 +31122,7 @@ static asdl_seq * _loop0_101_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31544,8 +31189,7 @@ static asdl_seq * _loop1_102_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31617,8 +31261,7 @@ static asdl_seq * _loop0_103_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31685,8 +31328,7 @@ static asdl_seq * _loop1_104_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31758,8 +31400,7 @@ static asdl_seq * _loop1_105_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31831,8 +31472,7 @@ static asdl_seq * _loop1_106_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31904,8 +31544,7 @@ static asdl_seq * _loop0_107_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31972,8 +31611,7 @@ static asdl_seq * _loop1_108_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32045,8 +31683,7 @@ static asdl_seq * _loop0_109_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32113,8 +31750,7 @@ static asdl_seq * _loop1_110_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32186,8 +31822,7 @@ static asdl_seq * _loop0_111_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32254,8 +31889,7 @@ static asdl_seq * _loop1_112_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32327,8 +31961,7 @@ static void * _tmp_113_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32385,8 +32018,7 @@ static asdl_seq * _loop0_114_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32453,8 +32085,7 @@ static asdl_seq * _loop1_115_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32526,8 +32157,7 @@ static void * _tmp_116_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32576,8 +32206,7 @@ static asdl_seq * _loop0_118_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32653,8 +32282,7 @@ static asdl_seq * _gather_117_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32695,8 +32323,7 @@ static asdl_seq * _loop1_119_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32768,8 +32395,7 @@ static asdl_seq * _loop0_120_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32836,8 +32462,7 @@ static asdl_seq * _loop0_121_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32904,8 +32529,7 @@ static void * _tmp_122_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32964,8 +32588,7 @@ static asdl_seq * _loop0_124_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33042,8 +32665,7 @@ static asdl_seq * _gather_123_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33084,8 +32706,7 @@ static void * _tmp_125_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33131,8 +32752,7 @@ static asdl_seq * _loop0_127_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33208,8 +32828,7 @@ static asdl_seq * _gather_126_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33250,8 +32869,7 @@ static asdl_seq * _loop0_129_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33327,8 +32945,7 @@ static asdl_seq * _gather_128_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33369,8 +32986,7 @@ static asdl_seq * _loop0_131_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33446,8 +33062,7 @@ static asdl_seq * _gather_130_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33488,8 +33103,7 @@ static asdl_seq * _loop0_133_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33565,8 +33179,7 @@ static asdl_seq * _gather_132_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33607,8 +33220,7 @@ static asdl_seq * _loop0_134_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33675,8 +33287,7 @@ static asdl_seq * _loop0_136_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33752,8 +33363,7 @@ static asdl_seq * _gather_135_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33794,8 +33404,7 @@ static asdl_seq * _loop1_137_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33867,8 +33476,7 @@ static void * _tmp_138_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33908,8 +33516,7 @@ static asdl_seq * _loop0_140_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33985,8 +33592,7 @@ static asdl_seq * _gather_139_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34027,8 +33633,7 @@ static asdl_seq * _loop0_142_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34104,8 +33709,7 @@ static asdl_seq * _gather_141_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34146,8 +33750,7 @@ static asdl_seq * _loop0_144_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34223,8 +33826,7 @@ static asdl_seq * _gather_143_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34265,8 +33867,7 @@ static asdl_seq * _loop0_146_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34342,8 +33943,7 @@ static asdl_seq * _gather_145_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34384,8 +33984,7 @@ static asdl_seq * _loop0_148_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34461,8 +34060,7 @@ static asdl_seq * _gather_147_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34503,8 +34101,7 @@ static void * _tmp_149_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34545,8 +34142,7 @@ static void * _tmp_150_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34606,8 +34202,7 @@ static void * _tmp_151_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34648,8 +34243,7 @@ static void * _tmp_152_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34706,8 +34300,7 @@ static void * _tmp_153_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34783,8 +34376,7 @@ static void * _tmp_154_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34825,8 +34417,7 @@ static void * _tmp_155_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34886,8 +34477,7 @@ static void * _tmp_156_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34944,8 +34534,7 @@ static void * _tmp_157_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35002,8 +34591,7 @@ static void * _tmp_158_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35060,8 +34648,7 @@ static void * _tmp_159_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35194,8 +34781,7 @@ static void * _tmp_160_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35252,8 +34838,7 @@ static asdl_seq * _loop0_161_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35320,8 +34905,7 @@ static asdl_seq * _loop0_162_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35388,8 +34972,7 @@ static asdl_seq * _loop0_163_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35456,8 +35039,7 @@ static void * _tmp_164_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35514,8 +35096,7 @@ static void * _tmp_165_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35591,8 +35172,7 @@ static void * _tmp_166_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35649,8 +35229,7 @@ static void * _tmp_167_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35707,8 +35286,7 @@ static void * _tmp_168_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35765,8 +35343,7 @@ static asdl_seq * _loop0_169_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35833,8 +35410,7 @@ static asdl_seq * _loop0_170_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35901,8 +35477,7 @@ static asdl_seq * _loop0_171_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35969,8 +35544,7 @@ static asdl_seq * _loop1_172_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36042,8 +35616,7 @@ static void * _tmp_173_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36100,8 +35673,7 @@ static asdl_seq * _loop0_174_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36168,8 +35740,7 @@ static void * _tmp_175_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36226,8 +35797,7 @@ static asdl_seq * _loop0_176_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36294,8 +35864,7 @@ static asdl_seq * _loop1_177_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36367,8 +35936,7 @@ static void * _tmp_178_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36425,8 +35993,7 @@ static void * _tmp_179_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36486,8 +36053,7 @@ static void * _tmp_180_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36544,8 +36110,7 @@ static asdl_seq * _loop0_181_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36612,8 +36177,7 @@ static void * _tmp_182_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36670,8 +36234,7 @@ static void * _tmp_183_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36747,8 +36310,7 @@ static asdl_seq * _loop1_184_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36820,8 +36382,7 @@ static void * _tmp_185_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36878,8 +36439,7 @@ static asdl_seq * _loop0_186_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36946,8 +36506,7 @@ static asdl_seq * _loop0_187_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37014,8 +36573,7 @@ static asdl_seq * _loop0_188_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37082,8 +36640,7 @@ static asdl_seq * _loop0_190_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37159,8 +36716,7 @@ static asdl_seq * _gather_189_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37201,8 +36757,7 @@ static void * _tmp_191_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37259,8 +36814,7 @@ static asdl_seq * _loop0_192_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37327,8 +36881,7 @@ static void * _tmp_193_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37385,8 +36938,7 @@ static asdl_seq * _loop0_194_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37453,8 +37005,7 @@ static asdl_seq * _loop1_195_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37526,8 +37077,7 @@ static asdl_seq * _loop1_196_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37599,8 +37149,7 @@ static void * _tmp_197_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37660,8 +37209,7 @@ static void * _tmp_198_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37718,8 +37266,7 @@ static asdl_seq * _loop0_199_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37786,8 +37333,7 @@ static void * _tmp_200_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37844,8 +37390,7 @@ static void * _tmp_201_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37921,8 +37466,7 @@ static void * _tmp_202_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37998,8 +37542,7 @@ static asdl_seq * _loop0_204_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38075,8 +37618,7 @@ static asdl_seq * _gather_203_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38117,8 +37659,7 @@ static asdl_seq * _loop0_206_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38194,8 +37735,7 @@ static asdl_seq * _gather_205_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38236,8 +37776,7 @@ static asdl_seq * _loop0_208_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38313,8 +37852,7 @@ static asdl_seq * _gather_207_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38355,8 +37893,7 @@ static asdl_seq * _loop0_210_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38432,8 +37969,7 @@ static asdl_seq * _gather_209_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38474,8 +38010,7 @@ static asdl_seq * _loop0_212_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38551,8 +38086,7 @@ static asdl_seq * _gather_211_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38593,8 +38127,7 @@ static void * _tmp_213_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38651,8 +38184,7 @@ static asdl_seq * _loop0_214_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38719,8 +38251,7 @@ static asdl_seq * _loop1_215_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38792,8 +38323,7 @@ static void * _tmp_216_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38834,8 +38364,7 @@ static asdl_seq * _loop0_217_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38902,8 +38431,7 @@ static asdl_seq * _loop1_218_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38975,8 +38503,7 @@ static void * _tmp_219_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39018,8 +38545,7 @@ static void * _tmp_220_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39060,8 +38586,7 @@ static void * _tmp_221_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39102,8 +38627,7 @@ static void * _tmp_222_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39160,8 +38684,7 @@ static void * _tmp_223_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39202,8 +38725,7 @@ static void * _tmp_224_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39244,8 +38766,7 @@ static void * _tmp_225_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39286,8 +38807,7 @@ static void * _tmp_226_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39328,8 +38848,7 @@ static void * _tmp_227_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39374,8 +38893,7 @@ static void * _tmp_228_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39420,8 +38938,7 @@ static asdl_seq * _loop0_230_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39497,8 +39014,7 @@ static asdl_seq * _gather_229_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39539,8 +39055,7 @@ static void * _tmp_231_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39597,8 +39112,7 @@ static void * _tmp_232_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39655,8 +39169,7 @@ static void * _tmp_233_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39713,8 +39226,7 @@ static void * _tmp_234_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39771,8 +39283,7 @@ static void * _tmp_235_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39867,8 +39378,7 @@ static void * _tmp_236_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39925,8 +39435,7 @@ static void * _tmp_237_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40002,8 +39511,7 @@ static void * _tmp_238_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40060,8 +39568,7 @@ static void * _tmp_239_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40118,8 +39625,7 @@ static void * _tmp_240_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40160,8 +39666,7 @@ static void * _tmp_241_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40218,8 +39723,7 @@ static void * _tmp_242_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40276,8 +39780,7 @@ static void * _tmp_243_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40318,8 +39821,7 @@ static asdl_seq * _loop0_244_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40386,8 +39888,7 @@ static void * _tmp_245_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40444,8 +39945,7 @@ static void * _tmp_246_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40486,8 +39986,7 @@ static void * _tmp_247_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40544,8 +40043,7 @@ static void * _tmp_248_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40591,8 +40089,7 @@ static void * _tmp_249_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40649,8 +40146,7 @@ static void * _tmp_250_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40707,8 +40203,7 @@ static void * _tmp_251_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40757,8 +40252,7 @@ static void * _tmp_252_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40804,8 +40298,7 @@ static void * _tmp_253_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40851,8 +40344,7 @@ static void * _tmp_254_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40898,8 +40390,7 @@ static void * _tmp_255_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40945,8 +40436,7 @@ static void * _tmp_256_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41003,8 +40493,7 @@ static void * _tmp_257_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41061,8 +40550,7 @@ static void * _tmp_258_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41108,8 +40596,7 @@ static void * _tmp_259_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41155,8 +40642,7 @@ static void * _tmp_260_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41215,8 +40701,7 @@ static void * _tmp_261_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41262,8 +40747,7 @@ static void * _tmp_262_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41309,8 +40793,7 @@ static void * _tmp_263_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41351,8 +40834,7 @@ static void * _tmp_264_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41393,8 +40875,7 @@ static void * _tmp_265_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41451,8 +40932,7 @@ static void * _tmp_266_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41509,8 +40989,7 @@ static void * _tmp_267_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41552,8 +41031,7 @@ static void * _tmp_268_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41595,8 +41073,7 @@ static void * _tmp_269_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41638,8 +41115,7 @@ static void * _tmp_270_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41681,8 +41157,7 @@ static void * _tmp_271_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41723,8 +41198,7 @@ static void * _tmp_272_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41783,8 +41257,7 @@ static void * _tmp_273_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41825,8 +41298,7 @@ static void * _tmp_274_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41867,8 +41339,7 @@ static void * _tmp_275_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41909,8 +41380,7 @@ static void * _tmp_276_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; diff --git a/Parser/pegen.h b/Parser/pegen.h index 0852bb51d4fe7..266d5219d45a9 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -166,6 +166,8 @@ void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, Py_ssize_t end_lineno, Py_ssize_t end_col_offset, const char *errmsg, va_list va); void _Pypegen_set_syntax_error(Parser* p, Token* last_token); +void _Pypegen_stack_overflow(Parser *p); + Py_LOCAL_INLINE(void *) RAISE_ERROR_KNOWN_LOCATION(Parser *p, PyObject *errtype, Py_ssize_t lineno, Py_ssize_t col_offset, diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index e543d40ccd8ab..f400936767379 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -454,3 +454,11 @@ _Pypegen_set_syntax_error(Parser* p, Token* last_token) { // generic SyntaxError we just raised if errors are found. _PyPegen_tokenize_full_source_to_check_for_errors(p); } + +void +_Pypegen_stack_overflow(Parser *p) +{ + p->error_indicator = 1; + PyErr_SetString(PyExc_MemoryError, + "Parser stack overflowed - Python source too complex to parse"); +} diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index f57b6275f671d..301949bdae96f 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -375,8 +375,7 @@ def __init__( def add_level(self) -> None: self.print("if (p->level++ == MAXSTACK) {") with self.indent(): - self.print("p->error_indicator = 1;") - self.print("PyErr_NoMemory();") + self.print("_Pypegen_stack_overflow(p);") self.print("}") def remove_level(self) -> None: From webhook-mailer at python.org Tue Aug 22 04:20:00 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 22 Aug 2023 08:20:00 -0000 Subject: [Python-checkins] [3.11] gh-107801: Improve the accuracy of io.TextIOWrapper.seek docs (#107933) (#108264) Message-ID: https://github.com/python/cpython/commit/cc42182c978cf9e304c1b03f94486c0a0785a477 commit: cc42182c978cf9e304c1b03f94486c0a0785a477 branch: 3.11 author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-22T08:19:56Z summary: [3.11] gh-107801: Improve the accuracy of io.TextIOWrapper.seek docs (#107933) (#108264) (cherry picked from commit 7f87ebbc3f52680c939791f397b9a478edf0c8d4) Clearly document the supported seek() operations: - Rewind to the start of the stream - Restore a previous stream position (given by tell()) - Fast-forward to the end of the stream files: M Doc/library/io.rst M Modules/_io/clinic/textio.c.h M Modules/_io/textio.c diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 66273d9ed1ff0..792bf43d9811b 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -1044,6 +1044,22 @@ Text I/O .. versionchanged:: 3.11 The method supports ``encoding="locale"`` option. + .. method:: seek(cookie, whence=os.SEEK_SET, /) + + Set the stream position. + Return the new stream position as an :class:`int`. + + Four operations are supported, + given by the following argument combinations: + + * ``seek(0, SEEK_SET)``: Rewind to the start of the stream. + * ``seek(cookie, SEEK_SET)``: Restore a previous position; + *cookie* **must be** a number returned by :meth:`!tell`. + * ``seek(0, SEEK_END)``: Fast-forward to the end of the stream. + * ``seek(0, SEEK_CUR)``: Leave the current stream position unchanged. + + Any other argument combinations are invalid, + and may raise exceptions. .. class:: StringIO(initial_value='', newline='\n') diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 907785b2beaf9..91755dc3ed568 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -470,9 +470,27 @@ _io_TextIOWrapper_readline(textio *self, PyObject *const *args, Py_ssize_t nargs } PyDoc_STRVAR(_io_TextIOWrapper_seek__doc__, -"seek($self, cookie, whence=0, /)\n" +"seek($self, cookie, whence=os.SEEK_SET, /)\n" "--\n" -"\n"); +"\n" +"Set the stream position, and return the new stream position.\n" +"\n" +" cookie\n" +" Zero or an opaque number returned by tell().\n" +" whence\n" +" The relative position to seek from.\n" +"\n" +"Four operations are supported, given by the following argument\n" +"combinations:\n" +"\n" +"- seek(0, SEEK_SET): Rewind to the start of the stream.\n" +"- seek(cookie, SEEK_SET): Restore a previous position;\n" +" \'cookie\' must be a number returned by tell().\n" +"- seek(0, SEEK_END): Fast-forward to the end of the stream.\n" +"- seek(0, SEEK_CUR): Leave the current stream position unchanged.\n" +"\n" +"Any other argument combinations are invalid,\n" +"and may raise exceptions."); #define _IO_TEXTIOWRAPPER_SEEK_METHODDEF \ {"seek", _PyCFunction_CAST(_io_TextIOWrapper_seek), METH_FASTCALL, _io_TextIOWrapper_seek__doc__}, @@ -671,4 +689,4 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) { return _io_TextIOWrapper_close_impl(self); } -/*[clinic end generated code: output=bb78b568b24759d6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f9bda53adf576a8e input=a9049054013a1b77]*/ diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 6cb5a6861a158..403687dac0336 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2415,13 +2415,29 @@ _textiowrapper_encoder_setstate(textio *self, cookie_type *cookie) /*[clinic input] _io.TextIOWrapper.seek cookie as cookieObj: object - whence: int = 0 + Zero or an opaque number returned by tell(). + whence: int(c_default='0') = os.SEEK_SET + The relative position to seek from. / + +Set the stream position, and return the new stream position. + +Four operations are supported, given by the following argument +combinations: + +- seek(0, SEEK_SET): Rewind to the start of the stream. +- seek(cookie, SEEK_SET): Restore a previous position; + 'cookie' must be a number returned by tell(). +- seek(0, SEEK_END): Fast-forward to the end of the stream. +- seek(0, SEEK_CUR): Leave the current stream position unchanged. + +Any other argument combinations are invalid, +and may raise exceptions. [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence) -/*[clinic end generated code: output=0a15679764e2d04d input=0458abeb3d7842be]*/ +/*[clinic end generated code: output=0a15679764e2d04d input=0f68adcb02cf2823]*/ { PyObject *posobj; cookie_type cookie; From webhook-mailer at python.org Tue Aug 22 04:49:39 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Tue, 22 Aug 2023 08:49:39 -0000 Subject: [Python-checkins] [3.11] gh-102507 Remove invisible pagebreak characters (GH-102531) (#108266) Message-ID: https://github.com/python/cpython/commit/dd0a1f9da283bd784e2c88efec0a45cef978516a commit: dd0a1f9da283bd784e2c88efec0a45cef978516a branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: AlexWaygood date: 2023-08-22T08:49:35Z summary: [3.11] gh-102507 Remove invisible pagebreak characters (GH-102531) (#108266) gh-102507 Remove invisible pagebreak characters (GH-102531) (cherry picked from commit b097925858c6975c73e989226cf278cc382c0416) Co-authored-by: JosephSBoyle <48555120+JosephSBoyle at users.noreply.github.com> Co-authored-by: AlexWaygood files: M Lib/email/__init__.py M Lib/email/base64mime.py M Lib/email/charset.py M Lib/email/encoders.py M Lib/email/feedparser.py M Lib/email/generator.py M Lib/email/header.py M Lib/email/iterators.py M Lib/email/mime/base.py M Lib/email/mime/message.py M Lib/email/mime/multipart.py M Lib/email/mime/nonmultipart.py M Lib/email/mime/text.py M Lib/email/parser.py M Modules/_io/bufferedio.c M Tools/i18n/pygettext.py diff --git a/Lib/email/__init__.py b/Lib/email/__init__.py index fae872439edc6..9fa4778300418 100644 --- a/Lib/email/__init__.py +++ b/Lib/email/__init__.py @@ -25,7 +25,6 @@ ] - # Some convenience routines. Don't import Parser and Message as side-effects # of importing email since those cascadingly import most of the rest of the # email package. diff --git a/Lib/email/base64mime.py b/Lib/email/base64mime.py index a7cc37365c6f9..4cdf22666e301 100644 --- a/Lib/email/base64mime.py +++ b/Lib/email/base64mime.py @@ -45,7 +45,6 @@ MISC_LEN = 7 - # Helpers def header_length(bytearray): """Return the length of s when it is encoded with base64.""" @@ -57,7 +56,6 @@ def header_length(bytearray): return n - def header_encode(header_bytes, charset='iso-8859-1'): """Encode a single header line with Base64 encoding in a given charset. @@ -72,7 +70,6 @@ def header_encode(header_bytes, charset='iso-8859-1'): return '=?%s?b?%s?=' % (charset, encoded) - def body_encode(s, maxlinelen=76, eol=NL): r"""Encode a string with base64. @@ -98,7 +95,6 @@ def body_encode(s, maxlinelen=76, eol=NL): return EMPTYSTRING.join(encvec) - def decode(string): """Decode a raw base64 string, returning a bytes object. diff --git a/Lib/email/charset.py b/Lib/email/charset.py index d3d759ad9115f..1d4db7cd227ee 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -18,7 +18,6 @@ from email.encoders import encode_7or8bit - # Flags for types of header encodings QP = 1 # Quoted-Printable BASE64 = 2 # Base64 @@ -32,7 +31,6 @@ EMPTYSTRING = '' - # Defaults CHARSETS = { # input header enc body enc output conv @@ -104,7 +102,6 @@ } - # Convenience functions for extending the above mappings def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): """Add character set properties to the global registry. @@ -153,7 +150,6 @@ def add_codec(charset, codecname): CODEC_MAP[charset] = codecname - # Convenience function for encoding strings, taking into account # that they might be unknown-8bit (ie: have surrogate-escaped bytes) def _encode(string, codec): @@ -163,7 +159,6 @@ def _encode(string, codec): return string.encode(codec) - class Charset: """Map character sets to their email properties. diff --git a/Lib/email/encoders.py b/Lib/email/encoders.py index 0a66acb6240bd..17bd1ab7b19f3 100644 --- a/Lib/email/encoders.py +++ b/Lib/email/encoders.py @@ -16,7 +16,6 @@ from quopri import encodestring as _encodestring - def _qencode(s): enc = _encodestring(s, quotetabs=True) # Must encode spaces, which quopri.encodestring() doesn't do @@ -34,7 +33,6 @@ def encode_base64(msg): msg['Content-Transfer-Encoding'] = 'base64' - def encode_quopri(msg): """Encode the message's payload in quoted-printable. @@ -46,7 +44,6 @@ def encode_quopri(msg): msg['Content-Transfer-Encoding'] = 'quoted-printable' - def encode_7or8bit(msg): """Set the Content-Transfer-Encoding header to 7bit or 8bit.""" orig = msg.get_payload(decode=True) @@ -64,6 +61,5 @@ def encode_7or8bit(msg): msg['Content-Transfer-Encoding'] = '7bit' - def encode_noop(msg): """Do nothing.""" diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index e400dc7fb89b0..89c739183c1d0 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -41,7 +41,6 @@ NeedMoreData = object() - class BufferedSubFile(object): """A file-ish object that can have new data loaded into it. @@ -132,7 +131,6 @@ def __next__(self): return line - class FeedParser: """A feed-style parser of email.""" diff --git a/Lib/email/generator.py b/Lib/email/generator.py index c9b121624e08d..b8c10917a5d98 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -22,7 +22,6 @@ fcre = re.compile(r'^From ', re.MULTILINE) - class Generator: """Generates output from a Message object tree. @@ -392,7 +391,7 @@ def _make_boundary(cls, text=None): def _compile_re(cls, s, flags): return re.compile(s, flags) - + class BytesGenerator(Generator): """Generates a bytes version of a Message object tree. @@ -443,7 +442,6 @@ def _compile_re(cls, s, flags): return re.compile(s.encode('ascii'), flags) - _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' class DecodedGenerator(Generator): @@ -503,7 +501,6 @@ def _dispatch(self, msg): }, file=self) - # Helper used by Generator._make_boundary _width = len(repr(sys.maxsize-1)) _fmt = '%%0%dd' % _width diff --git a/Lib/email/header.py b/Lib/email/header.py index 4ab0032bc6612..984851a7d9a67 100644 --- a/Lib/email/header.py +++ b/Lib/email/header.py @@ -52,12 +52,10 @@ _embedded_header = re.compile(r'\n[^ \t]+:') - # Helpers _max_append = email.quoprimime._max_append - def decode_header(header): """Decode a message header value without converting charset. @@ -152,7 +150,6 @@ def decode_header(header): return collapsed - def make_header(decoded_seq, maxlinelen=None, header_name=None, continuation_ws=' '): """Create a Header from a sequence of pairs as returned by decode_header() @@ -175,7 +172,6 @@ def make_header(decoded_seq, maxlinelen=None, header_name=None, return h - class Header: def __init__(self, s=None, charset=None, maxlinelen=None, header_name=None, @@ -409,7 +405,6 @@ def _normalize(self): self._chunks = chunks - class _ValueFormatter: def __init__(self, headerlen, maxlen, continuation_ws, splitchars): self._maxlen = maxlen diff --git a/Lib/email/iterators.py b/Lib/email/iterators.py index b5502ee975266..3410935e38f47 100644 --- a/Lib/email/iterators.py +++ b/Lib/email/iterators.py @@ -15,7 +15,6 @@ from io import StringIO - # This function will become a method of the Message class def walk(self): """Walk over the message tree, yielding each subpart. @@ -29,7 +28,6 @@ def walk(self): yield from subpart.walk() - # These two functions are imported into the Iterators.py interface module. def body_line_iterator(msg, decode=False): """Iterate over the parts, returning string payloads line-by-line. @@ -55,7 +53,6 @@ def typed_subpart_iterator(msg, maintype='text', subtype=None): yield subpart - def _structure(msg, fp=None, level=0, include_default=False): """A handy debugging aid""" if fp is None: diff --git a/Lib/email/mime/base.py b/Lib/email/mime/base.py index 1a3f9b51f6c04..f601f621cec39 100644 --- a/Lib/email/mime/base.py +++ b/Lib/email/mime/base.py @@ -11,7 +11,6 @@ from email import message - class MIMEBase(message.Message): """Base class for MIME specializations.""" diff --git a/Lib/email/mime/message.py b/Lib/email/mime/message.py index 07e4f2d119615..61836b5a7861f 100644 --- a/Lib/email/mime/message.py +++ b/Lib/email/mime/message.py @@ -10,7 +10,6 @@ from email.mime.nonmultipart import MIMENonMultipart - class MIMEMessage(MIMENonMultipart): """Class representing message/* MIME documents.""" diff --git a/Lib/email/mime/multipart.py b/Lib/email/mime/multipart.py index 2d3f288810dd9..94d81c771a474 100644 --- a/Lib/email/mime/multipart.py +++ b/Lib/email/mime/multipart.py @@ -9,7 +9,6 @@ from email.mime.base import MIMEBase - class MIMEMultipart(MIMEBase): """Base class for MIME multipart/* type messages.""" diff --git a/Lib/email/mime/nonmultipart.py b/Lib/email/mime/nonmultipart.py index e1f51968b59eb..a41386eb148c0 100644 --- a/Lib/email/mime/nonmultipart.py +++ b/Lib/email/mime/nonmultipart.py @@ -10,7 +10,6 @@ from email.mime.base import MIMEBase - class MIMENonMultipart(MIMEBase): """Base class for MIME non-multipart type messages.""" diff --git a/Lib/email/mime/text.py b/Lib/email/mime/text.py index 35b442383002b..dfe53c426b2ac 100644 --- a/Lib/email/mime/text.py +++ b/Lib/email/mime/text.py @@ -10,7 +10,6 @@ from email.mime.nonmultipart import MIMENonMultipart - class MIMEText(MIMENonMultipart): """Class for generating text/* type MIME documents.""" diff --git a/Lib/email/parser.py b/Lib/email/parser.py index 7db4da1ff081c..b1ca08d8f6cb2 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -67,7 +67,6 @@ def parsestr(self, text, headersonly=False): return self.parse(StringIO(text), headersonly=headersonly) - class HeaderParser(Parser): def parse(self, fp, headersonly=True): return Parser.parse(self, fp, True) @@ -75,7 +74,7 @@ def parse(self, fp, headersonly=True): def parsestr(self, text, headersonly=True): return Parser.parsestr(self, text, True) - + class BytesParser: def __init__(self, *args, **kw): diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 4a4a1992dbbb7..fd1e638990658 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1740,7 +1740,6 @@ _bufferedreader_peek_unlocked(buffered *self) self->pos = 0; return PyBytes_FromStringAndSize(self->buffer, r); } - /* @@ -2047,7 +2046,6 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) LEAVE_BUFFERED(self) return res; } - /* @@ -2253,7 +2251,6 @@ bufferedrwpair_closed_get(rwpair *self, void *context) } return PyObject_GetAttr((PyObject *) self->writer, &_Py_ID(closed)); } - /* diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 7ada79105db1c..3a0b27ba420e7 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -174,7 +174,6 @@ EMPTYSTRING = '' - # The normal pot-file header. msgmerge and Emacs's po-mode work better if it's # there. pot_header = _('''\ @@ -196,7 +195,7 @@ ''') - + def usage(code, msg=''): print(__doc__ % globals(), file=sys.stderr) if msg: @@ -204,7 +203,6 @@ def usage(code, msg=''): sys.exit(code) - def make_escapes(pass_nonascii): global escapes, escape if pass_nonascii: @@ -258,7 +256,7 @@ def normalize(s, encoding): s = '""\n"' + lineterm.join(lines) + '"' return s - + def containsAny(str, set): """Check whether 'str' contains ANY of the chars in 'set'""" return 1 in [c in str for c in set] @@ -307,7 +305,7 @@ def getFilesForName(name): return [] - + class TokenEater: def __init__(self, options): self.__options = options @@ -515,7 +513,6 @@ def write(self, fp): print('msgstr ""\n', file=fp) - def main(): global default_keywords try: @@ -675,7 +672,7 @@ class Options: if closep: fp.close() - + if __name__ == '__main__': main() # some more test strings From webhook-mailer at python.org Tue Aug 22 04:52:02 2023 From: webhook-mailer at python.org (encukou) Date: Tue, 22 Aug 2023 08:52:02 -0000 Subject: [Python-checkins] [3.11] gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) (GH-108209) Message-ID: https://github.com/python/cpython/commit/8e837373edc7607d404f66df735da4e97e2bc4c5 commit: 8e837373edc7607d404f66df735da4e97e2bc4c5 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: encukou date: 2023-08-22T10:51:58+02:00 summary: [3.11] gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) (GH-108209) gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) (cherry picked from commit acbd3f9c5c5f23e95267714e41236140d84fe962) Co-authored-by: Petr Viktorin Co-authored-by: Victor Stinner Co-authored-by: Lum?r 'Frenzy' Balhar files: A Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst M Doc/library/tarfile.rst M Lib/tarfile.py M Lib/test/test_tarfile.py diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index b7b089a73e648..61a450cf88b0a 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -732,6 +732,11 @@ A ``TarInfo`` object has the following public data attributes: Name of the target file name, which is only present in :class:`TarInfo` objects of type :const:`LNKTYPE` and :const:`SYMTYPE`. + For symbolic links (``SYMTYPE``), the *linkname* is relative to the directory + that contains the link. + For hard links (``LNKTYPE``), the *linkname* is relative to the root of + the archive. + .. attribute:: TarInfo.uid :type: int diff --git a/Lib/tarfile.py b/Lib/tarfile.py index b7adff6e1723b..2808e7efc553a 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -741,7 +741,7 @@ def __init__(self, tarinfo): class AbsoluteLinkError(FilterError): def __init__(self, tarinfo): self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a symlink to an absolute path') + super().__init__(f'{tarinfo.name!r} is a link to an absolute path') class LinkOutsideDestinationError(FilterError): def __init__(self, tarinfo, path): @@ -801,7 +801,14 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) - target_path = os.path.realpath(os.path.join(dest_path, member.linkname)) + if member.issym(): + target_path = os.path.join(dest_path, + os.path.dirname(name), + member.linkname) + else: + target_path = os.path.join(dest_path, + member.linkname) + target_path = os.path.realpath(target_path) if os.path.commonpath([target_path, dest_path]) != dest_path: raise LinkOutsideDestinationError(member, target_path) return new_attrs diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index dc7ff852363cf..13a75f39f9ba5 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3256,10 +3256,12 @@ def __exit__(self, *exc): self.bio = None def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, - mode=None, **kwargs): + mode=None, size=None, **kwargs): """Add a member to the test archive. Call within `with`.""" name = str(name) tarinfo = tarfile.TarInfo(name).replace(**kwargs) + if size is not None: + tarinfo.size = size if mode: tarinfo.mode = _filemode_to_int(mode) if symlink_to is not None: @@ -3335,7 +3337,8 @@ def check_context(self, tar, filter): raise self.raised_exception self.assertEqual(self.expected_paths, set()) - def expect_file(self, name, type=None, symlink_to=None, mode=None): + def expect_file(self, name, type=None, symlink_to=None, mode=None, + size=None): """Check a single file. See check_context.""" if self.raised_exception: raise self.raised_exception @@ -3364,6 +3367,8 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None): self.assertTrue(path.is_fifo()) else: raise NotImplementedError(type) + if size is not None: + self.assertEqual(path.stat().st_size, size) for parent in path.parents: self.expected_paths.discard(parent) @@ -3410,8 +3415,15 @@ def test_parent_symlink(self): # Test interplaying symlinks # Inspired by 'dirsymlink2a' in jwilk/traversal-archives with ArchiveMaker() as arc: + + # `current` links to `.` which is both: + # - the destination directory + # - `current` itself arc.add('current', symlink_to='.') + + # effectively points to ./../ arc.add('parent', symlink_to='current/..') + arc.add('parent/evil') if os_helper.can_symlink(): @@ -3453,9 +3465,46 @@ def test_parent_symlink(self): def test_parent_symlink2(self): # Test interplaying symlinks # Inspired by 'dirsymlink2b' in jwilk/traversal-archives + + # Posix and Windows have different pathname resolution: + # either symlink or a '..' component resolve first. + # Let's see which we are on. + if os_helper.can_symlink(): + testpath = os.path.join(TEMPDIR, 'resolution_test') + os.mkdir(testpath) + + # testpath/current links to `.` which is all of: + # - `testpath` + # - `testpath/current` + # - `testpath/current/current` + # - etc. + os.symlink('.', os.path.join(testpath, 'current')) + + # we'll test where `testpath/current/../file` ends up + with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): + pass + + if os.path.exists(os.path.join(testpath, 'file')): + # Windows collapses 'current\..' to '.' first, leaving + # 'testpath\file' + dotdot_resolves_early = True + elif os.path.exists(os.path.join(testpath, '..', 'file')): + # Posix resolves 'current' to '.' first, leaving + # 'testpath/../file' + dotdot_resolves_early = False + else: + raise AssertionError('Could not determine link resolution') + with ArchiveMaker() as arc: + + # `current` links to `.` which is both the destination directory + # and `current` itself arc.add('current', symlink_to='.') + + # `current/parent` is also available as `./parent`, + # and effectively points to `./../` arc.add('current/parent', symlink_to='..') + arc.add('parent/evil') with self.check_context(arc.open(), 'fully_trusted'): @@ -3469,6 +3518,7 @@ def test_parent_symlink2(self): with self.check_context(arc.open(), 'tar'): if os_helper.can_symlink(): + # Fail when extracting a file outside destination self.expect_exception( tarfile.OutsideDestinationError, "'parent/evil' would be extracted to " @@ -3479,10 +3529,24 @@ def test_parent_symlink2(self): self.expect_file('parent/evil') with self.check_context(arc.open(), 'data'): - self.expect_exception( - tarfile.LinkOutsideDestinationError, - """'current/parent' would link to ['"].*['"], """ - + "which is outside the destination") + if os_helper.can_symlink(): + if dotdot_resolves_early: + # Fail when extracting a file outside destination + self.expect_exception( + tarfile.OutsideDestinationError, + "'parent/evil' would be extracted to " + + """['"].*evil['"], which is outside """ + + "the destination") + else: + # Fail as soon as we have a symlink outside the destination + self.expect_exception( + tarfile.LinkOutsideDestinationError, + "'current/parent' would link to " + + """['"].*outerdir['"], which is outside """ + + "the destination") + else: + self.expect_file('current/') + self.expect_file('parent/evil') @symlink_test def test_absolute_symlink(self): @@ -3512,12 +3576,30 @@ def test_absolute_symlink(self): with self.check_context(arc.open(), 'data'): self.expect_exception( tarfile.AbsoluteLinkError, - "'parent' is a symlink to an absolute path") + "'parent' is a link to an absolute path") + + def test_absolute_hardlink(self): + # Test hardlink to an absolute path + # Inspired by 'dirsymlink' in https://github.com/jwilk/traversal-archives + with ArchiveMaker() as arc: + arc.add('parent', hardlink_to=self.outerdir / 'foo') + + with self.check_context(arc.open(), 'fully_trusted'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'tar'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.AbsoluteLinkError, + "'parent' is a link to an absolute path") @symlink_test def test_sly_relative0(self): # Inspired by 'relative0' in jwilk/traversal-archives with ArchiveMaker() as arc: + # points to `../../tmp/moo` arc.add('../moo', symlink_to='..//tmp/moo') try: @@ -3568,6 +3650,56 @@ def test_sly_relative2(self): + """['"].*moo['"], which is outside the """ + "destination") + @symlink_test + def test_deep_symlink(self): + # Test that symlinks and hardlinks inside a directory + # point to the correct file (`target` of size 3). + # If links aren't supported we get a copy of the file. + with ArchiveMaker() as arc: + arc.add('targetdir/target', size=3) + # a hardlink's linkname is relative to the archive + arc.add('linkdir/hardlink', hardlink_to=os.path.join( + 'targetdir', 'target')) + # a symlink's linkname is relative to the link's directory + arc.add('linkdir/symlink', symlink_to=os.path.join( + '..', 'targetdir', 'target')) + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + if os_helper.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='../targetdir/target') + else: + self.expect_file('linkdir/symlink', size=3) + + @symlink_test + def test_chains(self): + # Test chaining of symlinks/hardlinks. + # Symlinks are created before the files they point to. + with ArchiveMaker() as arc: + arc.add('linkdir/symlink', symlink_to='hardlink') + arc.add('symlink2', symlink_to=os.path.join( + 'linkdir', 'hardlink2')) + arc.add('targetdir/target', size=3) + arc.add('linkdir/hardlink', hardlink_to='targetdir/target') + arc.add('linkdir/hardlink2', hardlink_to='linkdir/symlink') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + self.expect_file('linkdir/hardlink2', size=3) + if os_helper.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='hardlink') + self.expect_file('symlink2', size=3, + symlink_to='linkdir/hardlink2') + else: + self.expect_file('linkdir/symlink', size=3) + self.expect_file('symlink2', size=3) + def test_modes(self): # Test how file modes are extracted # (Note that the modes are ignored on platforms without working chmod) diff --git a/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst new file mode 100644 index 0000000000000..32c1fb93f4ab2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst @@ -0,0 +1,3 @@ +:func:`tarfile.data_filter` now takes the location of symlinks into account +when determining their target, so it will no longer reject some valid +tarballs with ``LinkOutsideDestinationError``. From webhook-mailer at python.org Tue Aug 22 05:57:34 2023 From: webhook-mailer at python.org (hugovk) Date: Tue, 22 Aug 2023 09:57:34 -0000 Subject: [Python-checkins] [3.11] Trim trailing whitespace and test on CI (GH-104275) (#108215) Message-ID: https://github.com/python/cpython/commit/d678ee771931a391212ef41e61e9717b43867e35 commit: d678ee771931a391212ef41e61e9717b43867e35 branch: 3.11 author: Hugo van Kemenade committer: hugovk date: 2023-08-22T12:57:31+03:00 summary: [3.11] Trim trailing whitespace and test on CI (GH-104275) (#108215) files: A .github/workflows/lint.yml A .pre-commit-config.yaml M .github/CODEOWNERS M Lib/test/test_asyncio/test_runners.py M Lib/test/test_isinstance.py M Modules/_blake2/blake2module.h M Modules/_blake2/impl/blake2b-round.h M Modules/_blake2/impl/blake2s-load-xop.h M Modules/_blake2/impl/blake2s-round.h M Modules/_ctypes/darwin/dlfcn.h M Modules/_ctypes/libffi_osx/ffi.c M Modules/_ctypes/libffi_osx/include/ffi.h M Modules/_ctypes/libffi_osx/include/ffi_common.h M Modules/_ctypes/libffi_osx/include/fficonfig.h M Modules/_ctypes/libffi_osx/include/x86-ffitarget.h M Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c M Modules/_ctypes/libffi_osx/types.c M Modules/_ctypes/libffi_osx/x86/x86-ffi64.c M Modules/_ctypes/libffi_osx/x86/x86-ffi_darwin.c M Modules/_io/_iomodule.c M Modules/termios.c M PC/winreg.c M Tools/msi/bundle/bootstrap/pch.h M Tools/msi/bundle/bootstrap/resource.h diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3d39e0c4ef041..99d701daa1d49 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,7 +5,10 @@ # https://git-scm.com/docs/gitignore#_pattern_format # GitHub -.github/** @ezio-melotti +.github/** @ezio-melotti @hugovk + +# pre-commit +.pre-commit-config.yaml @hugovk @AlexWaygood # asyncio **/*asyncio* @1st1 @asvetlov diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000000..4481ea80bfd93 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,22 @@ +name: Lint + +on: [push, pull_request, workflow_dispatch] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout at v3 + - uses: actions/setup-python at v4 + with: + python-version: "3.x" + - uses: pre-commit/action at v3.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000..808622f19a3db --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-yaml + - id: trailing-whitespace + types_or: [c, python, rst] diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 8a4aba6d470ba..ae823cc7bec33 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -441,7 +441,7 @@ async def coro(): with asyncio.Runner() as runner: with self.assertRaises(asyncio.CancelledError): runner.run(coro()) - + def test_signal_install_not_supported_ok(self): # signal.signal() can throw if the "main thread" doensn't have signals enabled assert threading.current_thread() is threading.main_thread() diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index a0974640bc114..a8315a4a9123d 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -8,7 +8,6 @@ from test import support - class TestIsInstanceExceptions(unittest.TestCase): # Test to make sure that an AttributeError when accessing the instance's # class's bases is masked. This was actually a bug in Python 2.2 and @@ -97,7 +96,7 @@ def getclass(self): class D: pass self.assertRaises(RuntimeError, isinstance, c, D) - + # These tests are similar to above, but tickle certain code paths in # issubclass() instead of isinstance() -- really PyObject_IsSubclass() # vs. PyObject_IsInstance(). @@ -147,7 +146,6 @@ def getbases(self): self.assertRaises(TypeError, issubclass, B, C()) - # meta classes for creating abstract classes and instances class AbstractClass(object): def __init__(self, bases): @@ -179,7 +177,7 @@ class Super: class Child(Super): pass - + class TestIsInstanceIsSubclass(unittest.TestCase): # Tests to ensure that isinstance and issubclass work on abstract # classes and instances. Before the 2.2 release, TypeErrors were @@ -357,6 +355,6 @@ def blowstack(fxn, arg, compare_to): tuple_arg = (tuple_arg,) fxn(arg, tuple_arg) - + if __name__ == '__main__': unittest.main() diff --git a/Modules/_blake2/blake2module.h b/Modules/_blake2/blake2module.h index aa8f281178ead..c8144ec9d48d2 100644 --- a/Modules/_blake2/blake2module.h +++ b/Modules/_blake2/blake2module.h @@ -38,6 +38,6 @@ #endif // HAVE_LIBB2 // for secure_zero_memory(), store32(), store48(), and store64() -#include "impl/blake2-impl.h" +#include "impl/blake2-impl.h" #endif // Py_BLAKE2MODULE_H diff --git a/Modules/_blake2/impl/blake2b-round.h b/Modules/_blake2/impl/blake2b-round.h index cebc22550da4c..5b452c4d63bab 100644 --- a/Modules/_blake2/impl/blake2b-round.h +++ b/Modules/_blake2/impl/blake2b-round.h @@ -62,7 +62,7 @@ \ row2l = _mm_roti_epi64(row2l, -24); \ row2h = _mm_roti_epi64(row2h, -24); \ - + #define G2(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h,b0,b1) \ row1l = _mm_add_epi64(_mm_add_epi64(row1l, b0), row2l); \ row1h = _mm_add_epi64(_mm_add_epi64(row1h, b1), row2h); \ @@ -81,7 +81,7 @@ \ row2l = _mm_roti_epi64(row2l, -63); \ row2h = _mm_roti_epi64(row2h, -63); \ - + #if defined(HAVE_SSSE3) #define DIAGONALIZE(row1l,row2l,row3l,row4l,row1h,row2h,row3h,row4h) \ t0 = _mm_alignr_epi8(row2h, row2l, 8); \ diff --git a/Modules/_blake2/impl/blake2s-load-xop.h b/Modules/_blake2/impl/blake2s-load-xop.h index ac591a77d191a..14d9e7f764067 100644 --- a/Modules/_blake2/impl/blake2s-load-xop.h +++ b/Modules/_blake2/impl/blake2s-load-xop.h @@ -166,7 +166,7 @@ buf = _mm_perm_epi8(t1, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(1),TOB(7)) ); #define LOAD_MSG_8_3(buf) \ t0 = _mm_perm_epi8(m0, m2, _mm_set_epi32(TOB(6),TOB(1),TOB(0),TOB(0)) ); \ buf = _mm_perm_epi8(t0, m3, _mm_set_epi32(TOB(3),TOB(2),TOB(5),TOB(4)) ); \ - + #define LOAD_MSG_8_4(buf) \ buf = _mm_perm_epi8(m0, m1, _mm_set_epi32(TOB(5),TOB(4),TOB(7),TOB(2)) ); diff --git a/Modules/_blake2/impl/blake2s-round.h b/Modules/_blake2/impl/blake2s-round.h index 1e2f2b7f59bd6..3af4be35bee5d 100644 --- a/Modules/_blake2/impl/blake2s-round.h +++ b/Modules/_blake2/impl/blake2s-round.h @@ -86,6 +86,6 @@ LOAD_MSG_ ##r ##_4(buf4); \ G2(row1,row2,row3,row4,buf4); \ UNDIAGONALIZE(row1,row2,row3,row4); \ - + #endif diff --git a/Modules/_ctypes/darwin/dlfcn.h b/Modules/_ctypes/darwin/dlfcn.h index a2afc3eeb8479..a9915c3115ceb 100644 --- a/Modules/_ctypes/darwin/dlfcn.h +++ b/Modules/_ctypes/darwin/dlfcn.h @@ -1,7 +1,7 @@ /* Copyright (c) 2002 Jorge Acereda & Peter O'Gorman - + Portions may be copyright others, see the AUTHORS file included with this distribution. diff --git a/Modules/_ctypes/libffi_osx/ffi.c b/Modules/_ctypes/libffi_osx/ffi.c index 1776b795e2f83..e16423aef19fc 100644 --- a/Modules/_ctypes/libffi_osx/ffi.c +++ b/Modules/_ctypes/libffi_osx/ffi.c @@ -65,12 +65,12 @@ initialize_aggregate( arg->size = ALIGN(arg->size, curalign); arg->size += (*ptr)->size; - arg->alignment = (arg->alignment > curalign) ? + arg->alignment = (arg->alignment > curalign) ? arg->alignment : curalign; #else arg->size = ALIGN(arg->size, (*ptr)->alignment); arg->size += (*ptr)->size; - arg->alignment = (arg->alignment > (*ptr)->alignment) ? + arg->alignment = (arg->alignment > (*ptr)->alignment) ? arg->alignment : (*ptr)->alignment; #endif @@ -130,10 +130,10 @@ struct_on_stack( // Arguments' ffi_type->alignment must be nonzero. ffi_status ffi_prep_cif( -/*@out@*/ /*@partial@*/ ffi_cif* cif, +/*@out@*/ /*@partial@*/ ffi_cif* cif, ffi_abi abi, - unsigned int nargs, -/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type* rtype, + unsigned int nargs, +/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type* rtype, /*@dependent@*/ ffi_type** atypes) { unsigned int bytes = 0; @@ -184,7 +184,7 @@ ffi_prep_cif( if ((*ptr)->alignment == 0) return FFI_BAD_TYPEDEF; - /* Perform a sanity check on the argument type, do this + /* Perform a sanity check on the argument type, do this check after the initialization. */ FFI_ASSERT_VALID_TYPE(*ptr); diff --git a/Modules/_ctypes/libffi_osx/include/ffi.h b/Modules/_ctypes/libffi_osx/include/ffi.h index c104a5c89350b..88c58fc43deb6 100644 --- a/Modules/_ctypes/libffi_osx/include/ffi.h +++ b/Modules/_ctypes/libffi_osx/include/ffi.h @@ -199,9 +199,9 @@ typedef union { void ffi_raw_call( -/*@dependent@*/ ffi_cif* cif, - void (*fn)(void), -/*@out@*/ void* rvalue, +/*@dependent@*/ ffi_cif* cif, + void (*fn)(void), +/*@out@*/ void* rvalue, /*@dependent@*/ ffi_raw* avalue); void @@ -225,9 +225,9 @@ ffi_raw_size( longs and doubles are followed by an empty 64-bit word. */ void ffi_java_raw_call( -/*@dependent@*/ ffi_cif* cif, - void (*fn)(void), -/*@out@*/ void* rvalue, +/*@dependent@*/ ffi_cif* cif, + void (*fn)(void), +/*@out@*/ void* rvalue, /*@dependent@*/ ffi_raw* avalue); void @@ -272,8 +272,8 @@ typedef struct ffi_raw_closure { ffi_cif* cif; #if !FFI_NATIVE_RAW_API - /* if this is enabled, then a raw closure has the same layout - as a regular closure. We use this to install an intermediate + /* if this is enabled, then a raw closure has the same layout + as a regular closure. We use this to install an intermediate handler to do the transaltion, void** -> ffi_raw*. */ void (*translate_args)(ffi_cif*,void*,void**,void*); void* this_closure; @@ -303,17 +303,17 @@ ffi_prep_java_raw_closure( ffi_status ffi_prep_cif( -/*@out@*/ /*@partial@*/ ffi_cif* cif, +/*@out@*/ /*@partial@*/ ffi_cif* cif, ffi_abi abi, - unsigned int nargs, -/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type* rtype, + unsigned int nargs, +/*@dependent@*/ /*@out@*/ /*@partial@*/ ffi_type* rtype, /*@dependent@*/ ffi_type** atypes); void ffi_call( -/*@dependent@*/ ffi_cif* cif, - void (*fn)(void), -/*@out@*/ void* rvalue, +/*@dependent@*/ ffi_cif* cif, + void (*fn)(void), +/*@out@*/ void* rvalue, /*@dependent@*/ void** avalue); /* Useful for eliminating compiler warnings */ diff --git a/Modules/_ctypes/libffi_osx/include/ffi_common.h b/Modules/_ctypes/libffi_osx/include/ffi_common.h index 685a3580f4fe0..02b53c8600e01 100644 --- a/Modules/_ctypes/libffi_osx/include/ffi_common.h +++ b/Modules/_ctypes/libffi_osx/include/ffi_common.h @@ -41,7 +41,7 @@ char* alloca(); # endif #endif -/*#if defined(FFI_DEBUG) +/*#if defined(FFI_DEBUG) #include #endif*/ @@ -65,7 +65,7 @@ ffi_type_test( # define FFI_ASSERT_AT(x, f, l) ((x) ? 0 : ffi_assert(#x, (f), (l))) # define FFI_ASSERT_VALID_TYPE(x) ffi_type_test(x, __FILE__, __LINE__) #else -# define FFI_ASSERT(x) +# define FFI_ASSERT(x) # define FFI_ASSERT_AT(x, f, l) # define FFI_ASSERT_VALID_TYPE(x) #endif // #ifdef FFI_DEBUG diff --git a/Modules/_ctypes/libffi_osx/include/fficonfig.h b/Modules/_ctypes/libffi_osx/include/fficonfig.h index 217249071dcf4..dc0f4ecb26ee0 100644 --- a/Modules/_ctypes/libffi_osx/include/fficonfig.h +++ b/Modules/_ctypes/libffi_osx/include/fficonfig.h @@ -1,4 +1,4 @@ -/* Manually created fficonfig.h for Darwin on PowerPC or Intel +/* Manually created fficonfig.h for Darwin on PowerPC or Intel This file is manually generated to do away with the need for autoconf and therefore make it easier to cross-compile and build fat binaries. @@ -33,10 +33,10 @@ # define SIZEOF_DOUBLE 8 # if __GNUC__ >= 4 # define HAVE_LONG_DOUBLE 1 -# define SIZEOF_LONG_DOUBLE 16 +# define SIZEOF_LONG_DOUBLE 16 # else # undef HAVE_LONG_DOUBLE -# define SIZEOF_LONG_DOUBLE 8 +# define SIZEOF_LONG_DOUBLE 8 # endif #elif defined(__ppc64__) diff --git a/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h b/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h index 55c2b6c50cd90..df149eb14d78c 100644 --- a/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h +++ b/Modules/_ctypes/libffi_osx/include/x86-ffitarget.h @@ -33,7 +33,7 @@ # define X86 #endif -#if defined(__x86_64__) +#if defined(__x86_64__) # ifndef X86_64 # define X86_64 # endif diff --git a/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c b/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c index 8953d5fda3581..875412a676695 100644 --- a/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c +++ b/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c @@ -891,7 +891,7 @@ ffi_closure_helper_DARWIN( avalue[i] = alloca(arg_types[i]->size); ffi64_struct_to_ram_form(arg_types[i], (const char*)pgr, &gprSize, (const char*)pfr, &fprSize, &nf, avalue[i], NULL); - + ng += gprSize / sizeof(long); pgr += gprSize / sizeof(long); pfr += (fprSize - savedFPRSize) / sizeof(double); @@ -1479,7 +1479,7 @@ ffi64_struct_to_reg_form( memcpy(&outGPRs[destGMarker], &inStruct[srcMarker], inType->size); } - + srcMarker += inType->size; destGMarker += inType->size; i += inType->size - 1; @@ -1561,7 +1561,7 @@ ffi64_struct_to_reg_form( case FFI_TYPE_STRUCT: recurseCount++; ffi64_struct_to_reg_form(inType->elements[i], - inStruct, &srcMarker, &fprsUsed, outGPRs, + inStruct, &srcMarker, &fprsUsed, outGPRs, &destGMarker, outFPRs, &destFMarker); recurseCount--; break; diff --git a/Modules/_ctypes/libffi_osx/types.c b/Modules/_ctypes/libffi_osx/types.c index 44806aeeb75d3..761d223ff1498 100644 --- a/Modules/_ctypes/libffi_osx/types.c +++ b/Modules/_ctypes/libffi_osx/types.c @@ -1,6 +1,6 @@ /* ----------------------------------------------------------------------- types.c - Copyright (c) 1996, 1998 Red Hat, Inc. - + Predefined ffi_types needed by libffi. Permission is hereby granted, free of charge, to any person obtaining @@ -85,7 +85,7 @@ FFI_INTEGRAL_TYPEDEF(sint64, 8, 8, FFI_TYPE_SINT64); FFI_INTEGRAL_TYPEDEF(longdouble, 12, 4, FFI_TYPE_LONGDOUBLE); # endif -#elif defined ARM || defined SH || defined POWERPC_AIX +#elif defined ARM || defined SH || defined POWERPC_AIX FFI_INTEGRAL_TYPEDEF(double, 8, 4, FFI_TYPE_DOUBLE); FFI_INTEGRAL_TYPEDEF(longdouble, 8, 4, FFI_TYPE_LONGDOUBLE); #elif defined POWERPC_DARWIN diff --git a/Modules/_ctypes/libffi_osx/x86/x86-ffi64.c b/Modules/_ctypes/libffi_osx/x86/x86-ffi64.c index 8e7d016488029..b8ae6e80231ec 100644 --- a/Modules/_ctypes/libffi_osx/x86/x86-ffi64.c +++ b/Modules/_ctypes/libffi_osx/x86/x86-ffi64.c @@ -2,8 +2,8 @@ /* ----------------------------------------------------------------------- x86-ffi64.c - Copyright (c) 2002 Bo Thorsen - - x86-64 Foreign Function Interface + + x86-64 Foreign Function Interface Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -208,7 +208,7 @@ classify_argument( case FFI_TYPE_STRUCT: { - ffi_type** ptr; + ffi_type** ptr; int i; enum x86_64_reg_class subclasses[MAX_CLASSES]; const int UNITS_PER_WORD = 8; diff --git a/Modules/_ctypes/libffi_osx/x86/x86-ffi_darwin.c b/Modules/_ctypes/libffi_osx/x86/x86-ffi_darwin.c index 706ea0f51206d..59e3615806739 100644 --- a/Modules/_ctypes/libffi_osx/x86/x86-ffi_darwin.c +++ b/Modules/_ctypes/libffi_osx/x86/x86-ffi_darwin.c @@ -4,8 +4,8 @@ Copyright (c) 2002 Ranjit Mathew Copyright (c) 2002 Bo Thorsen Copyright (c) 2002 Roger Sayle - - x86 Foreign Function Interface + + x86 Foreign Function Interface Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -43,27 +43,27 @@ void ffi_prep_args(char *stack, extended_cif *ecif) register void **p_argv; register char *argp; register ffi_type **p_arg; - + argp = stack; - + if (ecif->cif->flags == FFI_TYPE_STRUCT) { *(void **) argp = ecif->rvalue; argp += 4; } - + p_argv = ecif->avalue; - + for (i = ecif->cif->nargs, p_arg = ecif->cif->arg_types; i != 0; i--, p_arg++) { size_t z; - + /* Align if necessary */ if ((sizeof(int) - 1) & (unsigned) argp) argp = (char *) ALIGN(argp, sizeof(int)); - + z = (*p_arg)->size; if (z < sizeof(int)) { @@ -73,31 +73,31 @@ void ffi_prep_args(char *stack, extended_cif *ecif) case FFI_TYPE_SINT8: *(signed int *) argp = (signed int)*(SINT8 *)(* p_argv); break; - + case FFI_TYPE_UINT8: *(unsigned int *) argp = (unsigned int)*(UINT8 *)(* p_argv); break; - + case FFI_TYPE_SINT16: *(signed int *) argp = (signed int)*(SINT16 *)(* p_argv); break; - + case FFI_TYPE_UINT16: *(unsigned int *) argp = (unsigned int)*(UINT16 *)(* p_argv); break; - + case FFI_TYPE_SINT32: *(signed int *) argp = (signed int)*(SINT32 *)(* p_argv); break; - + case FFI_TYPE_UINT32: *(unsigned int *) argp = (unsigned int)*(UINT32 *)(* p_argv); break; - + case FFI_TYPE_STRUCT: *(unsigned int *) argp = (unsigned int)*(UINT32 *)(* p_argv); break; - + default: FFI_ASSERT(0); } @@ -109,7 +109,7 @@ void ffi_prep_args(char *stack, extended_cif *ecif) p_argv++; argp += z; } - + return; } @@ -127,18 +127,18 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) case FFI_TYPE_SINT8: case FFI_TYPE_SINT16: #endif - + case FFI_TYPE_SINT64: case FFI_TYPE_FLOAT: case FFI_TYPE_DOUBLE: case FFI_TYPE_LONGDOUBLE: cif->flags = (unsigned) cif->rtype->type; break; - + case FFI_TYPE_UINT64: cif->flags = FFI_TYPE_SINT64; break; - + #ifndef X86 case FFI_TYPE_STRUCT: if (cif->rtype->size == 1) @@ -163,16 +163,16 @@ ffi_status ffi_prep_cif_machdep(ffi_cif *cif) } break; #endif - + default: cif->flags = FFI_TYPE_INT; break; } - + #ifdef X86_DARWIN cif->bytes = (cif->bytes + 15) & ~0xF; #endif - + return FFI_OK; } @@ -188,23 +188,23 @@ extern void ffi_call_STDCALL(void (*)(char *, extended_cif *), extended_cif *, void ffi_call(ffi_cif *cif, void (*fn)(), void *rvalue, void **avalue) { extended_cif ecif; - + ecif.cif = cif; ecif.avalue = avalue; - + /* If the return value is a struct and we don't have a return */ /* value address then we need to make one */ - - if ((rvalue == NULL) && + + if ((rvalue == NULL) && (cif->flags == FFI_TYPE_STRUCT)) { ecif.rvalue = alloca(cif->rtype->size); } else ecif.rvalue = rvalue; - - - switch (cif->abi) + + + switch (cif->abi) { case FFI_SYSV: ffi_call_SYSV(ffi_prep_args, &ecif, cif->bytes, cif->flags, ecif.rvalue, @@ -245,20 +245,20 @@ void *args; // our various things... ffi_cif *cif; void **arg_area; - + cif = closure->cif; - arg_area = (void**) alloca (cif->nargs * sizeof (void*)); - + arg_area = (void**) alloca (cif->nargs * sizeof (void*)); + /* this call will initialize ARG_AREA, such that each - * element in that array points to the corresponding + * element in that array points to the corresponding * value on the stack; and if the function returns * a structure, it will re-set RESP to point to the * structure return address. */ - + ffi_prep_incoming_args_SYSV(args, respp, arg_area, cif); - + (closure->fun) (cif, *respp, arg_area, closure->user_data); - + return cif->flags; } @@ -270,35 +270,35 @@ ffi_prep_incoming_args_SYSV(char *stack, void **rvalue, void **avalue, register void **p_argv; register char *argp; register ffi_type **p_arg; - + argp = stack; - + if ( cif->flags == FFI_TYPE_STRUCT ) { *rvalue = *(void **) argp; argp += 4; } - + p_argv = avalue; - + for (i = cif->nargs, p_arg = cif->arg_types; (i != 0); i--, p_arg++) { size_t z; - + /* Align if necessary */ if ((sizeof(int) - 1) & (unsigned) argp) { argp = (char *) ALIGN(argp, sizeof(int)); } - + z = (*p_arg)->size; - + /* because we're little endian, this is what it turns into. */ - + *p_argv = (void*) argp; - + p_argv++; argp += z; } - + return; } @@ -325,15 +325,15 @@ ffi_prep_closure (ffi_closure* closure, { if (cif->abi != FFI_SYSV) return FFI_BAD_ABI; - + FFI_INIT_TRAMPOLINE (&closure->tramp[0], \ &ffi_closure_SYSV, \ (void*)closure); - + closure->cif = cif; closure->user_data = user_data; closure->fun = fun; - + return FFI_OK; } @@ -349,32 +349,32 @@ ffi_prep_raw_closure_loc (ffi_raw_closure* closure, void *codeloc) { int i; - + FFI_ASSERT (cif->abi == FFI_SYSV); - + // we currently don't support certain kinds of arguments for raw // closures. This should be implemented by a separate assembly language // routine, since it would require argument processing, something we // don't do now for performance. - + for (i = cif->nargs-1; i >= 0; i--) { FFI_ASSERT (cif->arg_types[i]->type != FFI_TYPE_STRUCT); FFI_ASSERT (cif->arg_types[i]->type != FFI_TYPE_LONGDOUBLE); } - - + + FFI_INIT_TRAMPOLINE (&closure->tramp[0], &ffi_closure_raw_SYSV, codeloc); - + closure->cif = cif; closure->user_data = user_data; closure->fun = fun; - + return FFI_OK; } -static void +static void ffi_prep_args_raw(char *stack, extended_cif *ecif) { memcpy (stack, ecif->avalue, ecif->cif->bytes); @@ -386,7 +386,7 @@ ffi_prep_args_raw(char *stack, extended_cif *ecif) */ extern void -ffi_call_SYSV(void (*)(char *, extended_cif *), extended_cif *, unsigned, +ffi_call_SYSV(void (*)(char *, extended_cif *), extended_cif *, unsigned, unsigned, unsigned *, void (*fn)()); #ifdef X86_WIN32 @@ -400,23 +400,23 @@ ffi_raw_call(ffi_cif *cif, void (*fn)(), void *rvalue, ffi_raw *fake_avalue) { extended_cif ecif; void **avalue = (void **)fake_avalue; - + ecif.cif = cif; ecif.avalue = avalue; - + /* If the return value is a struct and we don't have a return */ /* value address then we need to make one */ - - if ((rvalue == NULL) && + + if ((rvalue == NULL) && (cif->rtype->type == FFI_TYPE_STRUCT)) { ecif.rvalue = alloca(cif->rtype->size); } else ecif.rvalue = rvalue; - - - switch (cif->abi) + + + switch (cif->abi) { case FFI_SYSV: ffi_call_SYSV(ffi_prep_args_raw, &ecif, cif->bytes, cif->flags, diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index a7b2e984310d1..aa877174417cd 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -59,7 +59,7 @@ PyDoc_STRVAR(module_doc, " I/O classes. open() uses the file's blksize (as obtained by os.stat) if\n" " possible.\n" ); - + /* * The main open() function @@ -512,7 +512,7 @@ _io_open_code_impl(PyObject *module, PyObject *path) { return PyFile_OpenCodeObject(path); } - + /* * Private helpers for the io module. */ diff --git a/Modules/termios.c b/Modules/termios.c index fcc8f04267987..3900a6f0b8986 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -85,7 +85,7 @@ termios_tcgetattr_impl(PyObject *module, int fd) int r; Py_BEGIN_ALLOW_THREADS - r = tcgetattr(fd, &mode); + r = tcgetattr(fd, &mode); Py_END_ALLOW_THREADS if (r == -1) { return PyErr_SetFromErrno(state->TermiosError); @@ -372,7 +372,7 @@ termios_tcgetwinsize_impl(PyObject *module, int fd) #if defined(TIOCGWINSZ) termiosmodulestate *state = PyModule_GetState(module); struct winsize w; - int r; + int r; Py_BEGIN_ALLOW_THREADS r = ioctl(fd, TIOCGWINSZ, &w); diff --git a/PC/winreg.c b/PC/winreg.c index f668cf3c19cab..940278194f4cf 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -564,7 +564,7 @@ Py2Reg(PyObject *value, DWORD typ, BYTE **retDataBuf, DWORD *retDataSize) { Py_ssize_t i,j; switch (typ) { - case REG_DWORD: + case REG_DWORD: { if (value != Py_None && !PyLong_Check(value)) { return FALSE; @@ -588,7 +588,7 @@ Py2Reg(PyObject *value, DWORD typ, BYTE **retDataBuf, DWORD *retDataSize) *retDataSize = sizeof(DWORD); break; } - case REG_QWORD: + case REG_QWORD: { if (value != Py_None && !PyLong_Check(value)) { return FALSE; diff --git a/Tools/msi/bundle/bootstrap/pch.h b/Tools/msi/bundle/bootstrap/pch.h index b0aa5111dabd0..6d0974b34c61e 100644 --- a/Tools/msi/bundle/bootstrap/pch.h +++ b/Tools/msi/bundle/bootstrap/pch.h @@ -5,7 +5,7 @@ // The license and further copyright text can be found in the file // LICENSE.TXT at the root directory of the distribution. // -// +// // // Precompiled header for standard bootstrapper application. // diff --git a/Tools/msi/bundle/bootstrap/resource.h b/Tools/msi/bundle/bootstrap/resource.h index 53c03c319f091..d951e651f6d20 100644 --- a/Tools/msi/bundle/bootstrap/resource.h +++ b/Tools/msi/bundle/bootstrap/resource.h @@ -14,7 +14,7 @@ // Next default values for new objects -// +// #ifdef APSTUDIO_INVOKED #ifndef APSTUDIO_READONLY_SYMBOLS #define _APS_NEXT_RESOURCE_VALUE 102 From webhook-mailer at python.org Tue Aug 22 06:00:10 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 22 Aug 2023 10:00:10 -0000 Subject: [Python-checkins] [3.12] Fix test_generators: save/restore warnings filters (GH-108246) (#108249) Message-ID: https://github.com/python/cpython/commit/b26a7b78c9658a9b888f56ce3dc8cfae127d1eca commit: b26a7b78c9658a9b888f56ce3dc8cfae127d1eca branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-22T12:00:07+02:00 summary: [3.12] Fix test_generators: save/restore warnings filters (GH-108246) (#108249) Fix test_generators: save/restore warnings filters (GH-108246) Previously, depending on existing filters, the test could modify the warnings and so fail as "env changed". (cherry picked from commit 531930f47f6b2a548d31e62cb4ad3e215a24bf53) Co-authored-by: Victor Stinner files: M Lib/test/test_generators.py diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 31680b5a92e0f..1ee9958445bf1 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -2176,6 +2176,7 @@ def printsolution(self, x): caught ValueError (xyz) >>> import warnings +>>> old_filters = warnings.filters.copy() >>> warnings.filterwarnings("ignore", category=DeprecationWarning) # Filter DeprecationWarning: regarding the (type, val, tb) signature of throw(). @@ -2249,8 +2250,7 @@ def printsolution(self, x): ... ValueError: 7 ->>> warnings.filters.pop(0) -('ignore', None, , None, 0) +>>> warnings.filters[:] = old_filters # Re-enable DeprecationWarning: the (type, val, tb) exception representation is deprecated, # and may be removed in a future version of Python. From webhook-mailer at python.org Tue Aug 22 06:01:28 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 22 Aug 2023 10:01:28 -0000 Subject: [Python-checkins] [3.12] gh-106242: Make ntpath.realpath errors consistent with abspath when there are embedded nulls (GH-108248) (#108251) Message-ID: https://github.com/python/cpython/commit/dd73f2f76ab1eabe92a2ca820430be723df31cde commit: dd73f2f76ab1eabe92a2ca820430be723df31cde branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-22T12:01:24+02:00 summary: [3.12] gh-106242: Make ntpath.realpath errors consistent with abspath when there are embedded nulls (GH-108248) (#108251) gh-106242: Make ntpath.realpath errors consistent with abspath when there are embedded nulls (GH-108248) * gh-106242: Make ntpath.realpath errors consistent with abspath when there are embedded nulls * Update 2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst mention Windows and the former incorrect ValueError. --------- (cherry picked from commit de33b5c662ea8d35d81ed857c6a39e34ab94c510) Co-authored-by: Steve Dower Co-authored-by: Gregory P. Smith files: A Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst M Lib/ntpath.py M Lib/test/test_ntpath.py diff --git a/Lib/ntpath.py b/Lib/ntpath.py index dadcdc0c495da..df3402d46c9cc 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -721,6 +721,14 @@ def realpath(path, *, strict=False): try: path = _getfinalpathname(path) initial_winerror = 0 + except ValueError as ex: + # gh-106242: Raised for embedded null characters + # In strict mode, we convert into an OSError. + # Non-strict mode returns the path as-is, since we've already + # made it absolute. + if strict: + raise OSError(str(ex)) from None + path = normpath(path) except OSError as ex: if strict: raise @@ -740,6 +748,10 @@ def realpath(path, *, strict=False): try: if _getfinalpathname(spath) == path: path = spath + except ValueError as ex: + # Unexpected, as an invalid path should not have gained a prefix + # at any point, but we ignore this error just in case. + pass except OSError as ex: # If the path does not exist and originally did not exist, then # strip the prefix anyway. diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 78e1cb582512b..d91dcdfb0c5fa 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -394,6 +394,10 @@ def test_realpath_basic(self): d = drives.pop().encode() self.assertEqual(ntpath.realpath(d), d) + # gh-106242: Embedded nulls and non-strict fallback to abspath + self.assertEqual(ABSTFN + "\0spam", + ntpath.realpath(os_helper.TESTFN + "\0spam", strict=False)) + @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_strict(self): @@ -404,6 +408,8 @@ def test_realpath_strict(self): self.addCleanup(os_helper.unlink, ABSTFN) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN, strict=True) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN + "2", strict=True) + # gh-106242: Embedded nulls should raise OSError (not ValueError) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "\0spam", strict=True) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') diff --git a/Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst b/Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst new file mode 100644 index 0000000000000..ffe42ec5dc3fa --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst @@ -0,0 +1,4 @@ +Fixes :func:`~os.path.realpath` to behave consistently when passed a path +containing an embedded null character on Windows. In strict mode, it now +raises :exc:`OSError` instead of the unexpected :exc:`ValueError`, and in +non-strict mode will make the path absolute. From webhook-mailer at python.org Tue Aug 22 06:01:41 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 22 Aug 2023 10:01:41 -0000 Subject: [Python-checkins] [3.12] gh-107801: Improve the accuracy of io.TextIOWrapper.seek docs (#107933) (#108262) Message-ID: https://github.com/python/cpython/commit/ef4d427fd8ed89fcfb5a0fada4c24abec12991e5 commit: ef4d427fd8ed89fcfb5a0fada4c24abec12991e5 branch: 3.12 author: Erlend E. Aasland committer: Yhg1s date: 2023-08-22T12:01:38+02:00 summary: [3.12] gh-107801: Improve the accuracy of io.TextIOWrapper.seek docs (#107933) (#108262) (cherry picked from commit 7f87ebbc3f52680c939791f397b9a478edf0c8d4) Clearly document the supported seek() operations: - Rewind to the start of the stream - Restore a previous stream position (given by tell()) - Fast-forward to the end of the stream files: M Doc/library/io.rst M Modules/_io/clinic/textio.c.h M Modules/_io/textio.c diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 66273d9ed1ff0..792bf43d9811b 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -1044,6 +1044,22 @@ Text I/O .. versionchanged:: 3.11 The method supports ``encoding="locale"`` option. + .. method:: seek(cookie, whence=os.SEEK_SET, /) + + Set the stream position. + Return the new stream position as an :class:`int`. + + Four operations are supported, + given by the following argument combinations: + + * ``seek(0, SEEK_SET)``: Rewind to the start of the stream. + * ``seek(cookie, SEEK_SET)``: Restore a previous position; + *cookie* **must be** a number returned by :meth:`!tell`. + * ``seek(0, SEEK_END)``: Fast-forward to the end of the stream. + * ``seek(0, SEEK_CUR)``: Leave the current stream position unchanged. + + Any other argument combinations are invalid, + and may raise exceptions. .. class:: StringIO(initial_value='', newline='\n') diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 33fc23bd4c0c6..63ec56311e2a6 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -759,9 +759,27 @@ _io_TextIOWrapper_readline(textio *self, PyObject *const *args, Py_ssize_t nargs } PyDoc_STRVAR(_io_TextIOWrapper_seek__doc__, -"seek($self, cookie, whence=0, /)\n" +"seek($self, cookie, whence=os.SEEK_SET, /)\n" "--\n" -"\n"); +"\n" +"Set the stream position, and return the new stream position.\n" +"\n" +" cookie\n" +" Zero or an opaque number returned by tell().\n" +" whence\n" +" The relative position to seek from.\n" +"\n" +"Four operations are supported, given by the following argument\n" +"combinations:\n" +"\n" +"- seek(0, SEEK_SET): Rewind to the start of the stream.\n" +"- seek(cookie, SEEK_SET): Restore a previous position;\n" +" \'cookie\' must be a number returned by tell().\n" +"- seek(0, SEEK_END): Fast-forward to the end of the stream.\n" +"- seek(0, SEEK_CUR): Leave the current stream position unchanged.\n" +"\n" +"Any other argument combinations are invalid,\n" +"and may raise exceptions."); #define _IO_TEXTIOWRAPPER_SEEK_METHODDEF \ {"seek", _PyCFunction_CAST(_io_TextIOWrapper_seek), METH_FASTCALL, _io_TextIOWrapper_seek__doc__}, @@ -960,4 +978,4 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) { return _io_TextIOWrapper_close_impl(self); } -/*[clinic end generated code: output=42f592331302973f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fc02f9e59bfa9956 input=a9049054013a1b77]*/ diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 46411c70a9675..ff6023c8ef928 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2438,13 +2438,29 @@ _textiowrapper_encoder_setstate(textio *self, cookie_type *cookie) /*[clinic input] _io.TextIOWrapper.seek cookie as cookieObj: object - whence: int = 0 + Zero or an opaque number returned by tell(). + whence: int(c_default='0') = os.SEEK_SET + The relative position to seek from. / + +Set the stream position, and return the new stream position. + +Four operations are supported, given by the following argument +combinations: + +- seek(0, SEEK_SET): Rewind to the start of the stream. +- seek(cookie, SEEK_SET): Restore a previous position; + 'cookie' must be a number returned by tell(). +- seek(0, SEEK_END): Fast-forward to the end of the stream. +- seek(0, SEEK_CUR): Leave the current stream position unchanged. + +Any other argument combinations are invalid, +and may raise exceptions. [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence) -/*[clinic end generated code: output=0a15679764e2d04d input=0458abeb3d7842be]*/ +/*[clinic end generated code: output=0a15679764e2d04d input=0f68adcb02cf2823]*/ { PyObject *posobj; cookie_type cookie; From webhook-mailer at python.org Tue Aug 22 06:02:36 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 22 Aug 2023 10:02:36 -0000 Subject: [Python-checkins] [3.12] gh-108179: Add error message for parser stack overflows (GH-108256) (#108263) Message-ID: https://github.com/python/cpython/commit/149d70c2546d9615e6292135430795c3cf2b50fe commit: 149d70c2546d9615e6292135430795c3cf2b50fe branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-22T12:02:32+02:00 summary: [3.12] gh-108179: Add error message for parser stack overflows (GH-108256) (#108263) gh-108179: Add error message for parser stack overflows (GH-108256) (cherry picked from commit 86617518c4ac824e2b6dc20691ba5a08df04f285) Co-authored-by: Dennis Sweeney <36520290+sweeneyde at users.noreply.github.com> files: M Lib/test/test_syntax.py M Parser/parser.c M Parser/pegen.h M Parser/pegen_errors.c M Tools/peg_generator/pegen/c_generator.py diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index f3d6cd7bad0ee..4c988382f8b41 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -2335,7 +2335,7 @@ def test_error_on_parser_stack_overflow(self): source = "-" * 100000 + "4" for mode in ["exec", "eval", "single"]: with self.subTest(mode=mode): - with self.assertRaises(MemoryError): + with self.assertRaisesRegex(MemoryError, r"too complex"): compile(source, "", mode) @support.cpython_only diff --git a/Parser/parser.c b/Parser/parser.c index f2ea8f59b0056..860bbea4431c0 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -1126,8 +1126,7 @@ static mod_ty file_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1173,8 +1172,7 @@ static mod_ty interactive_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1217,8 +1215,7 @@ static mod_ty eval_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1267,8 +1264,7 @@ static mod_ty func_type_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1329,8 +1325,7 @@ static expr_ty fstring_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1379,8 +1374,7 @@ static asdl_stmt_seq* statements_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1423,8 +1417,7 @@ static asdl_stmt_seq* statement_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1491,8 +1484,7 @@ static asdl_stmt_seq* statement_newline_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1623,8 +1615,7 @@ static asdl_stmt_seq* simple_stmts_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -1717,8 +1708,7 @@ static stmt_ty simple_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2099,8 +2089,7 @@ static stmt_ty compound_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2290,8 +2279,7 @@ static stmt_ty assignment_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2509,8 +2497,7 @@ static expr_ty annotated_rhs_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2580,8 +2567,7 @@ static AugOperator* augassign_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2912,8 +2898,7 @@ static stmt_ty return_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -2977,8 +2962,7 @@ static stmt_ty raise_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3078,8 +3062,7 @@ static stmt_ty global_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3143,8 +3126,7 @@ static stmt_ty nonlocal_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3208,8 +3190,7 @@ static stmt_ty del_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3294,8 +3275,7 @@ static stmt_ty yield_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3356,8 +3336,7 @@ static stmt_ty assert_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3424,8 +3403,7 @@ static stmt_ty import_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3501,8 +3479,7 @@ static stmt_ty import_name_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3568,8 +3545,7 @@ static stmt_ty import_from_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3688,8 +3664,7 @@ static asdl_alias_seq* import_from_targets_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3824,8 +3799,7 @@ static asdl_alias_seq* import_from_as_names_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3868,8 +3842,7 @@ static alias_ty import_from_as_name_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3933,8 +3906,7 @@ static asdl_alias_seq* dotted_as_names_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -3977,8 +3949,7 @@ static alias_ty dotted_as_name_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4044,8 +4015,7 @@ static expr_ty dotted_name_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, dotted_name_type, &_res)) { @@ -4079,8 +4049,7 @@ static expr_ty dotted_name_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4148,8 +4117,7 @@ static asdl_stmt_seq* block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4244,8 +4212,7 @@ static asdl_expr_seq* decorators_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4288,8 +4255,7 @@ static stmt_ty class_def_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4356,8 +4322,7 @@ static stmt_ty class_def_raw_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4452,8 +4417,7 @@ static stmt_ty function_def_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4521,8 +4485,7 @@ static stmt_ty function_def_raw_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4692,8 +4655,7 @@ static arguments_ty params_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4755,8 +4717,7 @@ static arguments_ty parameters_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -4919,8 +4880,7 @@ static asdl_arg_seq* slash_no_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5000,8 +4960,7 @@ static SlashWithDefault* slash_with_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5090,8 +5049,7 @@ static StarEtc* star_etc_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5252,8 +5210,7 @@ static arg_ty kwds_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5318,8 +5275,7 @@ static arg_ty param_no_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5399,8 +5355,7 @@ static arg_ty param_no_default_star_annotation_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5478,8 +5433,7 @@ static NameDefaultPair* param_with_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5565,8 +5519,7 @@ static NameDefaultPair* param_maybe_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5650,8 +5603,7 @@ static arg_ty param_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5715,8 +5667,7 @@ static arg_ty param_star_annotation_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5780,8 +5731,7 @@ static expr_ty annotation_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5827,8 +5777,7 @@ static expr_ty star_annotation_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5874,8 +5823,7 @@ static expr_ty default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -5943,8 +5891,7 @@ static stmt_ty if_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6084,8 +6031,7 @@ static stmt_ty elif_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6222,8 +6168,7 @@ static asdl_stmt_seq* else_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6291,8 +6236,7 @@ static stmt_ty while_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6388,8 +6332,7 @@ static stmt_ty for_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6586,8 +6529,7 @@ static stmt_ty with_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6856,8 +6798,7 @@ static withitem_ty with_item_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -6955,8 +6896,7 @@ static stmt_ty try_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7145,8 +7085,7 @@ static excepthandler_ty except_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7299,8 +7238,7 @@ static excepthandler_ty except_star_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7414,8 +7352,7 @@ static asdl_stmt_seq* finally_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7485,8 +7422,7 @@ static stmt_ty match_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7584,8 +7520,7 @@ static expr_ty subject_expr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7671,8 +7606,7 @@ static match_case_ty case_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7746,8 +7680,7 @@ static expr_ty guard_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7793,8 +7726,7 @@ static pattern_ty patterns_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7874,8 +7806,7 @@ static pattern_ty pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -7932,8 +7863,7 @@ static pattern_ty as_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8019,8 +7949,7 @@ static pattern_ty or_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8089,8 +8018,7 @@ static pattern_ty closed_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8272,8 +8200,7 @@ static pattern_ty literal_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8507,8 +8434,7 @@ static expr_ty literal_expr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8696,8 +8622,7 @@ static expr_ty complex_number_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8803,8 +8728,7 @@ static expr_ty signed_number_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8887,8 +8811,7 @@ static expr_ty signed_real_number_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -8971,8 +8894,7 @@ static expr_ty real_number_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9015,8 +8937,7 @@ static expr_ty imaginary_number_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9059,8 +8980,7 @@ static pattern_ty capture_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9121,8 +9041,7 @@ static expr_ty pattern_capture_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9169,8 +9088,7 @@ static pattern_ty wildcard_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9231,8 +9149,7 @@ static pattern_ty value_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9297,8 +9214,7 @@ static expr_ty attr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, attr_type, &_res)) { @@ -9332,8 +9248,7 @@ static expr_ty attr_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9401,8 +9316,7 @@ static expr_ty name_or_attr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9459,8 +9373,7 @@ static pattern_ty group_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9509,8 +9422,7 @@ static pattern_ty sequence_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9616,8 +9528,7 @@ static asdl_seq* open_sequence_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9666,8 +9577,7 @@ static asdl_seq* maybe_sequence_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9714,8 +9624,7 @@ static pattern_ty maybe_star_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9772,8 +9681,7 @@ static pattern_ty star_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -9882,8 +9790,7 @@ static pattern_ty mapping_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10082,8 +9989,7 @@ static asdl_seq* items_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10121,8 +10027,7 @@ static KeyPatternPair* key_value_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10171,8 +10076,7 @@ static expr_ty double_star_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10223,8 +10127,7 @@ static pattern_ty class_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10454,8 +10357,7 @@ static asdl_pattern_seq* positional_patterns_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10498,8 +10400,7 @@ static asdl_seq* keyword_patterns_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10537,8 +10438,7 @@ static KeyPatternPair* keyword_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10587,8 +10487,7 @@ static stmt_ty type_alias_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10661,8 +10560,7 @@ static asdl_type_param_seq* type_params_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10711,8 +10609,7 @@ static asdl_type_param_seq* type_param_seq_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10764,8 +10661,7 @@ static type_param_ty type_param_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -10972,8 +10868,7 @@ static expr_ty type_param_bound_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11019,8 +10914,7 @@ static expr_ty expressions_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11148,8 +11042,7 @@ static expr_ty expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11303,8 +11196,7 @@ static expr_ty yield_expr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11410,8 +11302,7 @@ static expr_ty star_expressions_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11534,8 +11425,7 @@ static expr_ty star_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11623,8 +11513,7 @@ static asdl_expr_seq* star_named_expressions_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11671,8 +11560,7 @@ static expr_ty star_named_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11755,8 +11643,7 @@ static expr_ty assignment_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11830,8 +11717,7 @@ static expr_ty named_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11909,8 +11795,7 @@ static expr_ty disjunction_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -11998,8 +11883,7 @@ static expr_ty conjunction_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12087,8 +11971,7 @@ static expr_ty inversion_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12176,8 +12059,7 @@ static expr_ty comparison_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12270,8 +12152,7 @@ static CmpopExprPair* compare_op_bitwise_or_pair_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12480,8 +12361,7 @@ static CmpopExprPair* eq_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12527,8 +12407,7 @@ static CmpopExprPair* noteq_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12574,8 +12453,7 @@ static CmpopExprPair* lte_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12621,8 +12499,7 @@ static CmpopExprPair* lt_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12668,8 +12545,7 @@ static CmpopExprPair* gte_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12715,8 +12591,7 @@ static CmpopExprPair* gt_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12762,8 +12637,7 @@ static CmpopExprPair* notin_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12812,8 +12686,7 @@ static CmpopExprPair* in_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12859,8 +12732,7 @@ static CmpopExprPair* isnot_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12909,8 +12781,7 @@ static CmpopExprPair* is_bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -12958,8 +12829,7 @@ static expr_ty bitwise_or_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, bitwise_or_type, &_res)) { @@ -12993,8 +12863,7 @@ static expr_ty bitwise_or_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13082,8 +12951,7 @@ static expr_ty bitwise_xor_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, bitwise_xor_type, &_res)) { @@ -13117,8 +12985,7 @@ static expr_ty bitwise_xor_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13206,8 +13073,7 @@ static expr_ty bitwise_and_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, bitwise_and_type, &_res)) { @@ -13241,8 +13107,7 @@ static expr_ty bitwise_and_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13330,8 +13195,7 @@ static expr_ty shift_expr_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, shift_expr_type, &_res)) { @@ -13365,8 +13229,7 @@ static expr_ty shift_expr_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13493,8 +13356,7 @@ static expr_ty sum_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, sum_type, &_res)) { @@ -13528,8 +13390,7 @@ static expr_ty sum_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13662,8 +13523,7 @@ static expr_ty term_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, term_type, &_res)) { @@ -13697,8 +13557,7 @@ static expr_ty term_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -13940,8 +13799,7 @@ static expr_ty factor_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14101,8 +13959,7 @@ static expr_ty power_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14188,8 +14045,7 @@ static expr_ty await_primary_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14284,8 +14140,7 @@ static expr_ty primary_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, primary_type, &_res)) { @@ -14319,8 +14174,7 @@ static expr_ty primary_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14526,8 +14380,7 @@ static expr_ty slices_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14618,8 +14471,7 @@ static expr_ty slice_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -14723,8 +14575,7 @@ static expr_ty atom_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15006,8 +14857,7 @@ static expr_ty group_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15075,8 +14925,7 @@ static expr_ty lambdef_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15146,8 +14995,7 @@ static arguments_ty lambda_params_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15209,8 +15057,7 @@ static arguments_ty lambda_parameters_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15375,8 +15222,7 @@ static asdl_arg_seq* lambda_slash_no_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15456,8 +15302,7 @@ static SlashWithDefault* lambda_slash_with_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15545,8 +15390,7 @@ static StarEtc* lambda_star_etc_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15674,8 +15518,7 @@ static arg_ty lambda_kwds_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15740,8 +15583,7 @@ static arg_ty lambda_param_no_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15813,8 +15655,7 @@ static NameDefaultPair* lambda_param_with_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15892,8 +15733,7 @@ static NameDefaultPair* lambda_param_maybe_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -15971,8 +15811,7 @@ static arg_ty lambda_param_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16033,8 +15872,7 @@ static expr_ty fstring_middle_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16098,8 +15936,7 @@ static expr_ty fstring_replacement_field_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16194,8 +16031,7 @@ static ResultTokenWithMetadata* fstring_conversion_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16241,8 +16077,7 @@ static ResultTokenWithMetadata* fstring_full_format_spec_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16306,8 +16141,7 @@ static expr_ty fstring_format_spec_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16369,8 +16203,7 @@ static expr_ty string_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16413,8 +16246,7 @@ static expr_ty strings_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16480,8 +16312,7 @@ static expr_ty list_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16548,8 +16379,7 @@ static expr_ty tuple_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16616,8 +16446,7 @@ static expr_ty set_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16684,8 +16513,7 @@ static expr_ty dict_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16777,8 +16605,7 @@ static asdl_seq* double_starred_kvpairs_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16825,8 +16652,7 @@ static KeyValuePair* double_starred_kvpair_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16891,8 +16717,7 @@ static KeyValuePair* kvpair_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16941,8 +16766,7 @@ static asdl_comprehension_seq* for_if_clauses_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -16988,8 +16812,7 @@ static comprehension_ty for_if_clause_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17116,8 +16939,7 @@ static expr_ty listcomp_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17206,8 +17028,7 @@ static expr_ty setcomp_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17298,8 +17119,7 @@ static expr_ty genexp_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17388,8 +17208,7 @@ static expr_ty dictcomp_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17478,8 +17297,7 @@ static expr_ty arguments_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17554,8 +17372,7 @@ static expr_ty args_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17655,8 +17472,7 @@ static asdl_seq* kwargs_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17743,8 +17559,7 @@ static expr_ty starred_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17827,8 +17642,7 @@ static KeywordOrStarred* kwarg_or_starred_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -17938,8 +17752,7 @@ static KeywordOrStarred* kwarg_or_double_starred_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18061,8 +17874,7 @@ static expr_ty star_targets_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18156,8 +17968,7 @@ static asdl_expr_seq* star_targets_list_seq_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18204,8 +18015,7 @@ static asdl_expr_seq* star_targets_tuple_seq_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18282,8 +18092,7 @@ static expr_ty star_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18374,8 +18183,7 @@ static expr_ty target_with_star_atom_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18516,8 +18324,7 @@ static expr_ty star_atom_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18677,8 +18484,7 @@ static expr_ty single_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18772,8 +18578,7 @@ static expr_ty single_subscript_attribute_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -18893,8 +18698,7 @@ static expr_ty t_primary_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } expr_ty _res = NULL; if (_PyPegen_is_memoized(p, t_primary_type, &_res)) { @@ -18928,8 +18732,7 @@ static expr_ty t_primary_raw(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19150,8 +18953,7 @@ static void * t_lookahead_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19227,8 +19029,7 @@ static asdl_expr_seq* del_targets_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19278,8 +19079,7 @@ static expr_ty del_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19416,8 +19216,7 @@ static expr_ty del_t_atom_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19584,8 +19383,7 @@ static asdl_expr_seq* type_expressions_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19829,8 +19627,7 @@ static Token* func_type_comment_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -19923,8 +19720,7 @@ static void * invalid_arguments_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20167,8 +19963,7 @@ static void * invalid_kwarg_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20314,8 +20109,7 @@ expression_without_invalid_rule(Parser *p) int _prev_call_invalid = p->call_invalid_rules; p->call_invalid_rules = 0; if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->call_invalid_rules = _prev_call_invalid; @@ -20434,8 +20228,7 @@ static void * invalid_legacy_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20486,8 +20279,7 @@ static void * invalid_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20603,8 +20395,7 @@ static void * invalid_named_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20730,8 +20521,7 @@ static void * invalid_assignment_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -20936,8 +20726,7 @@ static expr_ty invalid_ann_assign_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21024,8 +20813,7 @@ static void * invalid_del_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21071,8 +20859,7 @@ static void * invalid_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21120,8 +20907,7 @@ static void * invalid_comprehension_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21239,8 +21025,7 @@ static void * invalid_dict_comprehension_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21301,8 +21086,7 @@ static void * invalid_parameters_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21519,8 +21303,7 @@ static void * invalid_default_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21569,8 +21352,7 @@ static void * invalid_star_etc_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21712,8 +21494,7 @@ static void * invalid_kwds_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21828,8 +21609,7 @@ static void * invalid_parameters_helper_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -21897,8 +21677,7 @@ static void * invalid_lambda_parameters_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22117,8 +21896,7 @@ static void * invalid_lambda_parameters_helper_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22183,8 +21961,7 @@ static void * invalid_lambda_star_etc_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22299,8 +22076,7 @@ static void * invalid_lambda_kwds_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22415,8 +22191,7 @@ static void * invalid_double_type_comments_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22471,8 +22246,7 @@ static void * invalid_with_item_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22523,8 +22297,7 @@ static void * invalid_for_target_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22574,8 +22347,7 @@ static void * invalid_group_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22657,8 +22429,7 @@ static void * invalid_import_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22710,8 +22481,7 @@ static void * invalid_import_from_targets_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22762,8 +22532,7 @@ static void * invalid_with_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22862,8 +22631,7 @@ static void * invalid_with_stmt_indent_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -22974,8 +22742,7 @@ static void * invalid_try_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23154,8 +22921,7 @@ static void * invalid_except_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23313,8 +23079,7 @@ static void * invalid_finally_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23367,8 +23132,7 @@ static void * invalid_except_stmt_indent_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23459,8 +23223,7 @@ static void * invalid_except_star_stmt_indent_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23523,8 +23286,7 @@ static void * invalid_match_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23610,8 +23372,7 @@ static void * invalid_case_block_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23703,8 +23464,7 @@ static void * invalid_as_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23785,8 +23545,7 @@ static void * invalid_class_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23836,8 +23595,7 @@ static asdl_pattern_seq* invalid_class_argument_pattern_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23892,8 +23650,7 @@ static void * invalid_if_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -23979,8 +23736,7 @@ static void * invalid_elif_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24064,8 +23820,7 @@ static void * invalid_else_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24118,8 +23873,7 @@ static void * invalid_while_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24205,8 +23959,7 @@ static void * invalid_for_stmt_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24311,8 +24064,7 @@ static void * invalid_def_raw_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24386,8 +24138,7 @@ static void * invalid_class_def_raw_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24482,8 +24233,7 @@ static void * invalid_double_starred_kvpairs_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24592,8 +24342,7 @@ static void * invalid_kvpair_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24700,8 +24449,7 @@ static void * invalid_starred_expression_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -24764,8 +24512,7 @@ static void * invalid_replacement_field_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25125,8 +24872,7 @@ static void * invalid_conversion_character_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25197,8 +24943,7 @@ static asdl_seq * _loop0_1_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25265,8 +25010,7 @@ static asdl_seq * _loop0_2_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25333,8 +25077,7 @@ static asdl_seq * _loop0_3_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25401,8 +25144,7 @@ static asdl_seq * _loop1_4_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25474,8 +25216,7 @@ static asdl_seq * _loop0_6_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25551,8 +25292,7 @@ static asdl_seq * _gather_5_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25593,8 +25333,7 @@ static void * _tmp_7_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25651,8 +25390,7 @@ static void * _tmp_8_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25728,8 +25466,7 @@ static void * _tmp_9_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25786,8 +25523,7 @@ static void * _tmp_10_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25844,8 +25580,7 @@ static void * _tmp_11_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25902,8 +25637,7 @@ static void * _tmp_12_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -25949,8 +25683,7 @@ static void * _tmp_13_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26018,8 +25751,7 @@ static void * _tmp_14_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26065,8 +25797,7 @@ static asdl_seq * _loop1_15_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26138,8 +25869,7 @@ static void * _tmp_16_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26196,8 +25926,7 @@ static void * _tmp_17_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26254,8 +25983,7 @@ static void * _tmp_18_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26301,8 +26029,7 @@ static asdl_seq * _loop0_20_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26378,8 +26105,7 @@ static asdl_seq * _gather_19_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26420,8 +26146,7 @@ static asdl_seq * _loop0_22_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26497,8 +26222,7 @@ static asdl_seq * _gather_21_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26539,8 +26263,7 @@ static void * _tmp_23_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26597,8 +26320,7 @@ static void * _tmp_24_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26644,8 +26366,7 @@ static asdl_seq * _loop0_25_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26712,8 +26433,7 @@ static asdl_seq * _loop1_26_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26785,8 +26505,7 @@ static asdl_seq * _loop0_28_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26862,8 +26581,7 @@ static asdl_seq * _gather_27_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26904,8 +26622,7 @@ static void * _tmp_29_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -26951,8 +26668,7 @@ static asdl_seq * _loop0_31_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27028,8 +26744,7 @@ static asdl_seq * _gather_30_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27070,8 +26785,7 @@ static void * _tmp_32_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27117,8 +26831,7 @@ static asdl_seq * _loop1_33_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27190,8 +26903,7 @@ static void * _tmp_34_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27240,8 +26952,7 @@ static void * _tmp_35_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27287,8 +26998,7 @@ static void * _tmp_36_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27334,8 +27044,7 @@ static asdl_seq * _loop0_37_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27402,8 +27111,7 @@ static asdl_seq * _loop0_38_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27470,8 +27178,7 @@ static asdl_seq * _loop0_39_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27538,8 +27245,7 @@ static asdl_seq * _loop1_40_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27611,8 +27317,7 @@ static asdl_seq * _loop0_41_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27679,8 +27384,7 @@ static asdl_seq * _loop1_42_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27752,8 +27456,7 @@ static asdl_seq * _loop1_43_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27825,8 +27528,7 @@ static asdl_seq * _loop1_44_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27898,8 +27600,7 @@ static asdl_seq * _loop0_45_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -27966,8 +27667,7 @@ static asdl_seq * _loop1_46_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28039,8 +27739,7 @@ static asdl_seq * _loop0_47_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28107,8 +27806,7 @@ static asdl_seq * _loop1_48_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28180,8 +27878,7 @@ static asdl_seq * _loop0_49_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28248,8 +27945,7 @@ static asdl_seq * _loop0_50_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28316,8 +28012,7 @@ static asdl_seq * _loop1_51_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28389,8 +28084,7 @@ static asdl_seq * _loop0_53_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28466,8 +28160,7 @@ static asdl_seq * _gather_52_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28508,8 +28201,7 @@ static asdl_seq * _loop0_55_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28585,8 +28277,7 @@ static asdl_seq * _gather_54_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28627,8 +28318,7 @@ static asdl_seq * _loop0_57_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28704,8 +28394,7 @@ static asdl_seq * _gather_56_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28746,8 +28435,7 @@ static asdl_seq * _loop0_59_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28823,8 +28511,7 @@ static asdl_seq * _gather_58_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28865,8 +28552,7 @@ static void * _tmp_60_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -28942,8 +28628,7 @@ static asdl_seq * _loop1_61_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29015,8 +28700,7 @@ static asdl_seq * _loop1_62_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29088,8 +28772,7 @@ static void * _tmp_63_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29135,8 +28818,7 @@ static void * _tmp_64_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29182,8 +28864,7 @@ static asdl_seq * _loop1_65_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29255,8 +28936,7 @@ static asdl_seq * _loop0_67_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29332,8 +29012,7 @@ static asdl_seq * _gather_66_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29374,8 +29053,7 @@ static void * _tmp_68_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29432,8 +29110,7 @@ static void * _tmp_69_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29490,8 +29167,7 @@ static void * _tmp_70_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29567,8 +29243,7 @@ static void * _tmp_71_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29644,8 +29319,7 @@ static asdl_seq * _loop0_73_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29721,8 +29395,7 @@ static asdl_seq * _gather_72_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29763,8 +29436,7 @@ static asdl_seq * _loop0_75_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29840,8 +29512,7 @@ static asdl_seq * _gather_74_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29882,8 +29553,7 @@ static void * _tmp_76_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -29940,8 +29610,7 @@ static asdl_seq * _loop0_78_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30017,8 +29686,7 @@ static asdl_seq * _gather_77_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30059,8 +29727,7 @@ static asdl_seq * _loop0_80_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30136,8 +29803,7 @@ static asdl_seq * _gather_79_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30178,8 +29844,7 @@ static asdl_seq * _loop0_82_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30255,8 +29920,7 @@ static asdl_seq * _gather_81_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30297,8 +29961,7 @@ static asdl_seq * _loop1_83_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30370,8 +30033,7 @@ static asdl_seq * _loop1_84_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30443,8 +30105,7 @@ static asdl_seq * _loop0_86_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30520,8 +30181,7 @@ static asdl_seq * _gather_85_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30562,8 +30222,7 @@ static asdl_seq * _loop1_87_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30635,8 +30294,7 @@ static asdl_seq * _loop1_88_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30708,8 +30366,7 @@ static asdl_seq * _loop1_89_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30781,8 +30438,7 @@ static void * _tmp_90_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30825,8 +30481,7 @@ static asdl_seq * _loop0_92_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30902,8 +30557,7 @@ static asdl_seq * _gather_91_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30944,8 +30598,7 @@ static void * _tmp_93_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -30991,8 +30644,7 @@ static void * _tmp_94_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31049,8 +30701,7 @@ static void * _tmp_95_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31126,8 +30777,7 @@ static void * _tmp_96_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31184,8 +30834,7 @@ static void * _tmp_97_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31280,8 +30929,7 @@ static void * _tmp_98_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31338,8 +30986,7 @@ static asdl_seq * _loop0_99_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31406,8 +31053,7 @@ static asdl_seq * _loop0_100_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31474,8 +31120,7 @@ static asdl_seq * _loop0_101_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31542,8 +31187,7 @@ static asdl_seq * _loop1_102_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31615,8 +31259,7 @@ static asdl_seq * _loop0_103_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31683,8 +31326,7 @@ static asdl_seq * _loop1_104_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31756,8 +31398,7 @@ static asdl_seq * _loop1_105_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31829,8 +31470,7 @@ static asdl_seq * _loop1_106_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31902,8 +31542,7 @@ static asdl_seq * _loop0_107_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -31970,8 +31609,7 @@ static asdl_seq * _loop1_108_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32043,8 +31681,7 @@ static asdl_seq * _loop0_109_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32111,8 +31748,7 @@ static asdl_seq * _loop1_110_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32184,8 +31820,7 @@ static asdl_seq * _loop0_111_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32252,8 +31887,7 @@ static asdl_seq * _loop1_112_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32325,8 +31959,7 @@ static void * _tmp_113_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32383,8 +32016,7 @@ static asdl_seq * _loop0_114_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32451,8 +32083,7 @@ static asdl_seq * _loop1_115_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32524,8 +32155,7 @@ static void * _tmp_116_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32574,8 +32204,7 @@ static asdl_seq * _loop0_118_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32651,8 +32280,7 @@ static asdl_seq * _gather_117_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32693,8 +32321,7 @@ static asdl_seq * _loop1_119_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32766,8 +32393,7 @@ static asdl_seq * _loop0_120_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32834,8 +32460,7 @@ static asdl_seq * _loop0_121_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32902,8 +32527,7 @@ static void * _tmp_122_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -32962,8 +32586,7 @@ static asdl_seq * _loop0_124_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33040,8 +32663,7 @@ static asdl_seq * _gather_123_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33082,8 +32704,7 @@ static void * _tmp_125_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33129,8 +32750,7 @@ static asdl_seq * _loop0_127_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33206,8 +32826,7 @@ static asdl_seq * _gather_126_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33248,8 +32867,7 @@ static asdl_seq * _loop0_129_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33325,8 +32943,7 @@ static asdl_seq * _gather_128_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33367,8 +32984,7 @@ static asdl_seq * _loop0_131_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33444,8 +33060,7 @@ static asdl_seq * _gather_130_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33486,8 +33101,7 @@ static asdl_seq * _loop0_133_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33563,8 +33177,7 @@ static asdl_seq * _gather_132_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33605,8 +33218,7 @@ static asdl_seq * _loop0_134_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33673,8 +33285,7 @@ static asdl_seq * _loop0_136_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33750,8 +33361,7 @@ static asdl_seq * _gather_135_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33792,8 +33402,7 @@ static asdl_seq * _loop1_137_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33865,8 +33474,7 @@ static void * _tmp_138_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33906,8 +33514,7 @@ static asdl_seq * _loop0_140_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -33983,8 +33590,7 @@ static asdl_seq * _gather_139_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34025,8 +33631,7 @@ static asdl_seq * _loop0_142_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34102,8 +33707,7 @@ static asdl_seq * _gather_141_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34144,8 +33748,7 @@ static asdl_seq * _loop0_144_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34221,8 +33824,7 @@ static asdl_seq * _gather_143_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34263,8 +33865,7 @@ static asdl_seq * _loop0_146_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34340,8 +33941,7 @@ static asdl_seq * _gather_145_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34382,8 +33982,7 @@ static asdl_seq * _loop0_148_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34459,8 +34058,7 @@ static asdl_seq * _gather_147_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34501,8 +34099,7 @@ static void * _tmp_149_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34543,8 +34140,7 @@ static void * _tmp_150_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34604,8 +34200,7 @@ static void * _tmp_151_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34646,8 +34241,7 @@ static void * _tmp_152_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34704,8 +34298,7 @@ static void * _tmp_153_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34781,8 +34374,7 @@ static void * _tmp_154_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34823,8 +34415,7 @@ static void * _tmp_155_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34884,8 +34475,7 @@ static void * _tmp_156_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -34942,8 +34532,7 @@ static void * _tmp_157_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35000,8 +34589,7 @@ static void * _tmp_158_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35058,8 +34646,7 @@ static void * _tmp_159_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35192,8 +34779,7 @@ static void * _tmp_160_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35250,8 +34836,7 @@ static asdl_seq * _loop0_161_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35318,8 +34903,7 @@ static asdl_seq * _loop0_162_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35386,8 +34970,7 @@ static asdl_seq * _loop0_163_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35454,8 +35037,7 @@ static void * _tmp_164_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35512,8 +35094,7 @@ static void * _tmp_165_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35589,8 +35170,7 @@ static void * _tmp_166_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35647,8 +35227,7 @@ static void * _tmp_167_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35705,8 +35284,7 @@ static void * _tmp_168_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35763,8 +35341,7 @@ static asdl_seq * _loop0_169_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35831,8 +35408,7 @@ static asdl_seq * _loop0_170_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35899,8 +35475,7 @@ static asdl_seq * _loop0_171_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -35967,8 +35542,7 @@ static asdl_seq * _loop1_172_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36040,8 +35614,7 @@ static void * _tmp_173_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36098,8 +35671,7 @@ static asdl_seq * _loop0_174_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36166,8 +35738,7 @@ static void * _tmp_175_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36224,8 +35795,7 @@ static asdl_seq * _loop0_176_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36292,8 +35862,7 @@ static asdl_seq * _loop1_177_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36365,8 +35934,7 @@ static void * _tmp_178_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36423,8 +35991,7 @@ static void * _tmp_179_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36484,8 +36051,7 @@ static void * _tmp_180_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36542,8 +36108,7 @@ static asdl_seq * _loop0_181_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36610,8 +36175,7 @@ static void * _tmp_182_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36668,8 +36232,7 @@ static void * _tmp_183_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36745,8 +36308,7 @@ static asdl_seq * _loop1_184_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36818,8 +36380,7 @@ static void * _tmp_185_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36876,8 +36437,7 @@ static asdl_seq * _loop0_186_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -36944,8 +36504,7 @@ static asdl_seq * _loop0_187_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37012,8 +36571,7 @@ static asdl_seq * _loop0_188_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37080,8 +36638,7 @@ static asdl_seq * _loop0_190_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37157,8 +36714,7 @@ static asdl_seq * _gather_189_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37199,8 +36755,7 @@ static void * _tmp_191_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37257,8 +36812,7 @@ static asdl_seq * _loop0_192_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37325,8 +36879,7 @@ static void * _tmp_193_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37383,8 +36936,7 @@ static asdl_seq * _loop0_194_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37451,8 +37003,7 @@ static asdl_seq * _loop1_195_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37524,8 +37075,7 @@ static asdl_seq * _loop1_196_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37597,8 +37147,7 @@ static void * _tmp_197_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37658,8 +37207,7 @@ static void * _tmp_198_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37716,8 +37264,7 @@ static asdl_seq * _loop0_199_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37784,8 +37331,7 @@ static void * _tmp_200_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37842,8 +37388,7 @@ static void * _tmp_201_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37919,8 +37464,7 @@ static void * _tmp_202_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -37996,8 +37540,7 @@ static asdl_seq * _loop0_204_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38073,8 +37616,7 @@ static asdl_seq * _gather_203_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38115,8 +37657,7 @@ static asdl_seq * _loop0_206_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38192,8 +37733,7 @@ static asdl_seq * _gather_205_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38234,8 +37774,7 @@ static asdl_seq * _loop0_208_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38311,8 +37850,7 @@ static asdl_seq * _gather_207_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38353,8 +37891,7 @@ static asdl_seq * _loop0_210_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38430,8 +37967,7 @@ static asdl_seq * _gather_209_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38472,8 +38008,7 @@ static asdl_seq * _loop0_212_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38549,8 +38084,7 @@ static asdl_seq * _gather_211_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38591,8 +38125,7 @@ static void * _tmp_213_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38649,8 +38182,7 @@ static asdl_seq * _loop0_214_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38717,8 +38249,7 @@ static asdl_seq * _loop1_215_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38790,8 +38321,7 @@ static void * _tmp_216_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38832,8 +38362,7 @@ static asdl_seq * _loop0_217_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38900,8 +38429,7 @@ static asdl_seq * _loop1_218_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -38973,8 +38501,7 @@ static void * _tmp_219_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39016,8 +38543,7 @@ static void * _tmp_220_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39058,8 +38584,7 @@ static void * _tmp_221_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39100,8 +38625,7 @@ static void * _tmp_222_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39158,8 +38682,7 @@ static void * _tmp_223_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39200,8 +38723,7 @@ static void * _tmp_224_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39242,8 +38764,7 @@ static void * _tmp_225_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39284,8 +38805,7 @@ static void * _tmp_226_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39326,8 +38846,7 @@ static void * _tmp_227_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39372,8 +38891,7 @@ static void * _tmp_228_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39418,8 +38936,7 @@ static asdl_seq * _loop0_230_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39495,8 +39012,7 @@ static asdl_seq * _gather_229_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39537,8 +39053,7 @@ static void * _tmp_231_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39595,8 +39110,7 @@ static void * _tmp_232_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39653,8 +39167,7 @@ static void * _tmp_233_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39711,8 +39224,7 @@ static void * _tmp_234_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39769,8 +39281,7 @@ static void * _tmp_235_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39865,8 +39376,7 @@ static void * _tmp_236_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -39923,8 +39433,7 @@ static void * _tmp_237_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40000,8 +39509,7 @@ static void * _tmp_238_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40058,8 +39566,7 @@ static void * _tmp_239_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40116,8 +39623,7 @@ static void * _tmp_240_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40158,8 +39664,7 @@ static void * _tmp_241_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40216,8 +39721,7 @@ static void * _tmp_242_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40274,8 +39778,7 @@ static void * _tmp_243_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40316,8 +39819,7 @@ static asdl_seq * _loop0_244_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40384,8 +39886,7 @@ static void * _tmp_245_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40442,8 +39943,7 @@ static void * _tmp_246_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40484,8 +39984,7 @@ static void * _tmp_247_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40542,8 +40041,7 @@ static void * _tmp_248_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40589,8 +40087,7 @@ static void * _tmp_249_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40647,8 +40144,7 @@ static void * _tmp_250_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40705,8 +40201,7 @@ static void * _tmp_251_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40755,8 +40250,7 @@ static void * _tmp_252_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40802,8 +40296,7 @@ static void * _tmp_253_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40849,8 +40342,7 @@ static void * _tmp_254_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40896,8 +40388,7 @@ static void * _tmp_255_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -40943,8 +40434,7 @@ static void * _tmp_256_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41001,8 +40491,7 @@ static void * _tmp_257_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41059,8 +40548,7 @@ static void * _tmp_258_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41106,8 +40594,7 @@ static void * _tmp_259_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41153,8 +40640,7 @@ static void * _tmp_260_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41213,8 +40699,7 @@ static void * _tmp_261_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41260,8 +40745,7 @@ static void * _tmp_262_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41307,8 +40791,7 @@ static void * _tmp_263_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41349,8 +40832,7 @@ static void * _tmp_264_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41391,8 +40873,7 @@ static void * _tmp_265_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41449,8 +40930,7 @@ static void * _tmp_266_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41507,8 +40987,7 @@ static void * _tmp_267_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41550,8 +41029,7 @@ static void * _tmp_268_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41593,8 +41071,7 @@ static void * _tmp_269_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41636,8 +41113,7 @@ static void * _tmp_270_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41679,8 +41155,7 @@ static void * _tmp_271_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41721,8 +41196,7 @@ static void * _tmp_272_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41781,8 +41255,7 @@ static void * _tmp_273_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41823,8 +41296,7 @@ static void * _tmp_274_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41865,8 +41337,7 @@ static void * _tmp_275_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; @@ -41907,8 +41378,7 @@ static void * _tmp_276_rule(Parser *p) { if (p->level++ == MAXSTACK) { - p->error_indicator = 1; - PyErr_NoMemory(); + _Pypegen_stack_overflow(p); } if (p->error_indicator) { p->level--; diff --git a/Parser/pegen.h b/Parser/pegen.h index fe13d10e6b83e..a8bfa786b9611 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -168,6 +168,8 @@ void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, Py_ssize_t end_lineno, Py_ssize_t end_col_offset, const char *errmsg, va_list va); void _Pypegen_set_syntax_error(Parser* p, Token* last_token); +void _Pypegen_stack_overflow(Parser *p); + Py_LOCAL_INLINE(void *) RAISE_ERROR_KNOWN_LOCATION(Parser *p, PyObject *errtype, Py_ssize_t lineno, Py_ssize_t col_offset, diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index af529057f50e7..b2fca019f6819 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -453,3 +453,11 @@ _Pypegen_set_syntax_error(Parser* p, Token* last_token) { // generic SyntaxError we just raised if errors are found. _PyPegen_tokenize_full_source_to_check_for_errors(p); } + +void +_Pypegen_stack_overflow(Parser *p) +{ + p->error_indicator = 1; + PyErr_SetString(PyExc_MemoryError, + "Parser stack overflowed - Python source too complex to parse"); +} diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index f57b6275f671d..301949bdae96f 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -375,8 +375,7 @@ def __init__( def add_level(self) -> None: self.print("if (p->level++ == MAXSTACK) {") with self.indent(): - self.print("p->error_indicator = 1;") - self.print("PyErr_NoMemory();") + self.print("_Pypegen_stack_overflow(p);") self.print("}") def remove_level(self) -> None: From webhook-mailer at python.org Tue Aug 22 07:10:32 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 22 Aug 2023 11:10:32 -0000 Subject: [Python-checkins] gh-105539: Emit ResourceWarning if sqlite3 database is not closed explicitly (#108015) Message-ID: https://github.com/python/cpython/commit/1a1bfc28912a39b500c578e9f10a8a222638d411 commit: 1a1bfc28912a39b500c578e9f10a8a222638d411 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-22T13:10:29+02:00 summary: gh-105539: Emit ResourceWarning if sqlite3 database is not closed explicitly (#108015) files: A Misc/NEWS.d/next/Library/2023-08-16-14-30-13.gh-issue-105539.29lA6c.rst M Doc/library/sqlite3.rst M Doc/whatsnew/3.13.rst M Lib/test/test_sqlite3/test_dbapi.py M Modules/_sqlite/connection.c diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 5f1676e5ae2f5..d2745b0dbceb2 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -630,6 +630,12 @@ Connection objects * :ref:`sqlite3-connection-shortcuts` * :ref:`sqlite3-connection-context-manager` + + .. versionchanged:: 3.13 + + A :exc:`ResourceWarning` is emitted if :meth:`close` is not called before + a :class:`!Connection` object is deleted. + An SQLite database connection has the following attributes and methods: .. method:: cursor(factory=Cursor) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index bfab868d1c5b6..8509e18a7d792 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -158,6 +158,13 @@ pathlib :meth:`~pathlib.Path.is_dir`. (Contributed by Barney Gale in :gh:`77609` and :gh:`105793`.) +sqlite3 +------- + +* A :exc:`ResourceWarning` is now emitted if a :class:`sqlite3.Connection` + object is not :meth:`closed ` explicitly. + (Contributed by Erlend E. Aasland in :gh:`105539`.) + tkinter ------- diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index df3c2ea8d1dbd..d80ad7af3200f 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -583,6 +583,12 @@ def test_connect_positional_arguments(self): cx.close() self.assertEqual(cm.filename, __file__) + def test_connection_resource_warning(self): + with self.assertWarns(ResourceWarning): + cx = sqlite.connect(":memory:") + del cx + gc_collect() + class UninitialisedConnectionTests(unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS.d/next/Library/2023-08-16-14-30-13.gh-issue-105539.29lA6c.rst b/Misc/NEWS.d/next/Library/2023-08-16-14-30-13.gh-issue-105539.29lA6c.rst new file mode 100644 index 0000000000000..0098c7f2438e9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-16-14-30-13.gh-issue-105539.29lA6c.rst @@ -0,0 +1,3 @@ +:mod:`sqlite3` now emits an :exc:`ResourceWarning` if a +:class:`sqlite3.Connection` object is not :meth:`closed +` explicitly. Patch by Erlend E. Aasland. diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 282855f886594..e133977b28c37 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -493,6 +493,14 @@ connection_finalize(PyObject *self) } /* Clean up if user has not called .close() explicitly. */ + if (con->db) { + if (PyErr_ResourceWarning(self, 1, "unclosed database in %R", self)) { + /* Spurious errors can appear at shutdown */ + if (PyErr_ExceptionMatches(PyExc_Warning)) { + PyErr_WriteUnraisable(self); + } + } + } if (connection_close(con) < 0) { if (teardown) { PyErr_Clear(); From webhook-mailer at python.org Tue Aug 22 08:18:43 2023 From: webhook-mailer at python.org (hugovk) Date: Tue, 22 Aug 2023 12:18:43 -0000 Subject: [Python-checkins] [3.11] Docs: move sphinx-lint to pre-commit (GH-105750) (#108276) Message-ID: https://github.com/python/cpython/commit/9d00379e2c8dbf527cc113054b82f2fbde3e731c commit: 9d00379e2c8dbf527cc113054b82f2fbde3e731c branch: 3.11 author: Hugo van Kemenade committer: hugovk date: 2023-08-22T15:18:39+03:00 summary: [3.11] Docs: move sphinx-lint to pre-commit (GH-105750) (#108276) files: M .github/workflows/reusable-docs.yml M .pre-commit-config.yaml M Doc/Makefile M Doc/constraints.txt M Doc/requirements.txt diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 9c1ed2788080d..f8e166b780b3e 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -26,8 +26,6 @@ jobs: cache-dependency-path: 'Doc/requirements.txt' - name: 'Install build dependencies' run: make -C Doc/ venv - - name: 'Check documentation' - run: make -C Doc/ check - name: 'Build HTML documentation' run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html - name: 'Upload' @@ -37,8 +35,6 @@ jobs: path: Doc/build/html # This build doesn't use problem matchers or check annotations - # It also does not run 'make check', as sphinx-lint is not installed into the - # environment. build_doc_oldest_supported_sphinx: name: 'Docs (Oldest Sphinx)' runs-on: ubuntu-latest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 808622f19a3db..889bd8502a10e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,3 +5,11 @@ repos: - id: check-yaml - id: trailing-whitespace types_or: [c, python, rst] + + - repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v0.6.8 + hooks: + - id: sphinx-lint + args: [--enable=default-role] + files: ^Doc/|^Misc/NEWS.d/next/ + types: [rst] diff --git a/Doc/Makefile b/Doc/Makefile index 918814140f45c..55be2b2dc5c1e 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -214,11 +214,9 @@ dist: rm -r dist/python-$(DISTVERSION)-docs-texinfo rm dist/python-$(DISTVERSION)-docs-texinfo.tar -check: - # Check the docs and NEWS files with sphinx-lint. - # Ignore the tools and venv dirs and check that the default role is not used. - $(SPHINXLINT) -i tools -i $(VENVDIR) --enable default-role - $(SPHINXLINT) --enable default-role ../Misc/NEWS.d/next/ +check: venv + $(VENVDIR)/bin/python3 -m pre_commit --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install pre-commit + $(VENVDIR)/bin/python3 -m pre_commit run --all-files serve: @echo "The serve target was removed, use htmlview instead (see bpo-36329)" diff --git a/Doc/constraints.txt b/Doc/constraints.txt index 66c748eb092d8..54888eaab242e 100644 --- a/Doc/constraints.txt +++ b/Doc/constraints.txt @@ -23,7 +23,3 @@ sphinxcontrib-serializinghtml<1.2 # Direct dependencies of Jinja2 (Jinja is a dependency of Sphinx, see above) MarkupSafe<2.2 - -# Direct dependencies of sphinx-lint -polib<1.3 -regex<2024 diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 3c0e0a7733673..df79ae6384047 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -10,7 +10,6 @@ sphinx==4.5.0 blurb -sphinx-lint==0.6.7 sphinxext-opengraph>=0.7.1 # The theme used by the documentation is stored separately, so we need From webhook-mailer at python.org Tue Aug 22 08:38:04 2023 From: webhook-mailer at python.org (hugovk) Date: Tue, 22 Aug 2023 12:38:04 -0000 Subject: [Python-checkins] gh-106971: Docs: Add missing issue reference (#106992) Message-ID: https://github.com/python/cpython/commit/c556f9a3c9af48c9af9e1f298be638553a6c886e commit: c556f9a3c9af48c9af9e1f298be638553a6c886e branch: main author: Junya Fukuda committer: hugovk date: 2023-08-22T15:38:01+03:00 summary: gh-106971: Docs: Add missing issue reference (#106992) Co-authored-by: Hugo van Kemenade files: M Doc/whatsnew/3.12.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 65ca4c332ce35..ff651bcd04728 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1201,7 +1201,7 @@ Pending Removal in Python 3.14 is deprecated and will raise an exception in Python 3.14. * Creating immutable types (:c:macro:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable - bases using the C API. + bases using the C API (:gh:`95388`). * ``__package__`` and ``__cached__`` will cease to be set or taken into consideration by the import system (:gh:`97879`). From webhook-mailer at python.org Tue Aug 22 08:50:35 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Tue, 22 Aug 2023 12:50:35 -0000 Subject: [Python-checkins] gh-107298: Fix numerous ref errors and typos in the C API docs (GH-108258) Message-ID: https://github.com/python/cpython/commit/d7202e4879bf4e7e00a69500ddcb3143864139b4 commit: d7202e4879bf4e7e00a69500ddcb3143864139b4 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-22T15:50:30+03:00 summary: gh-107298: Fix numerous ref errors and typos in the C API docs (GH-108258) files: M Doc/c-api/exceptions.rst M Doc/c-api/init_config.rst M Doc/c-api/module.rst M Doc/c-api/unicode.rst M Doc/extending/newtypes.rst M Doc/tools/.nitignore M Doc/whatsnew/2.2.rst M Doc/whatsnew/2.3.rst M Doc/whatsnew/2.4.rst M Doc/whatsnew/2.5.rst M Doc/whatsnew/2.6.rst M Doc/whatsnew/2.7.rst M Doc/whatsnew/3.0.rst M Doc/whatsnew/3.1.rst M Doc/whatsnew/3.10.rst M Doc/whatsnew/3.11.rst M Doc/whatsnew/3.2.rst M Doc/whatsnew/3.3.rst M Doc/whatsnew/3.5.rst M Doc/whatsnew/3.8.rst M Doc/whatsnew/3.9.rst M Misc/NEWS.d/3.11.0a1.rst M Misc/NEWS.d/3.8.0a1.rst M Misc/NEWS.d/3.8.0a4.rst M Misc/NEWS.d/3.8.0b1.rst M Misc/NEWS.d/3.9.0a1.rst M Misc/NEWS.d/3.9.0a5.rst diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index f1d6c995188ab..6e2ac0a40a5f1 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -222,17 +222,21 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the - filename is given as a C string. *filename* is decoded from the filesystem - encoding (:func:`os.fsdecode`). + Similar to :c:func:`PyErr_SetFromWindowsErr`, with the additional behavior + that if *filename* is not ``NULL``, it is decoded from the filesystem + encoding (:func:`os.fsdecode`) and passed to the constructor of + :exc:`OSError` as a third parameter to be used to define the + :attr:`!filename` attribute of the exception instance. .. availability:: Windows. .. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an - additional parameter specifying the exception type to be raised. + Similar to :c:func:`PyErr_SetExcFromWindowsErr`, with the additional behavior + that if *filename* is not ``NULL``, it is passed to the constructor of + :exc:`OSError` as a third parameter to be used to define the + :attr:`!filename` attribute of the exception instance. .. availability:: Windows. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 3ad2d435665f8..56b4ee03c1a8f 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -82,6 +82,8 @@ PyWideStringList If *length* is non-zero, *items* must be non-``NULL`` and all strings must be non-``NULL``. + .. c:namespace:: NULL + Methods: .. c:function:: PyStatus PyWideStringList_Append(PyWideStringList *list, const wchar_t *item) @@ -101,6 +103,8 @@ PyWideStringList Python must be preinitialized to call this function. + .. c:namespace:: PyWideStringList + Structure fields: .. c:member:: Py_ssize_t length diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 187f8419d4ee4..979b22261efa3 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -338,6 +338,7 @@ The available slot types are: The *value* pointer of this slot must point to a function of the signature: .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) + :noindex: The function receives a :py:class:`~importlib.machinery.ModuleSpec` instance, as defined in :PEP:`451`, and the module definition. @@ -372,6 +373,7 @@ The available slot types are: The signature of the function is: .. c:function:: int exec_module(PyObject* module) + :noindex: If multiple ``Py_mod_exec`` slots are specified, they are processed in the order they appear in the *m_slots* array. @@ -380,6 +382,8 @@ The available slot types are: Specifies one of the following values: + .. c:namespace:: NULL + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED The module does not support being imported in subinterpreters. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index cf965fa167652..2a2cb1b8c458e 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1292,7 +1292,7 @@ the user settings on the machine running the codec. Encode the Unicode object using the specified code page and return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. Use - :c:macro:`CP_ACP` code page to get the MBCS encoder. + :c:macro:`!CP_ACP` code page to get the MBCS encoder. .. versionadded:: 3.3 diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 386b3c8f4452c..9f166eb8a4c3f 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -298,7 +298,7 @@ have an associated doc string simply by providing the text in the table. An application can use the introspection API to retrieve the descriptor from the class object, and get the doc string using its :attr:`__doc__` attribute. -As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.name` value +As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.ml_name` value of ``NULL`` is required. .. XXX Descriptors need to be explained in more detail somewhere, but not here. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 09514c89c117a..bb5515b561095 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -20,7 +20,6 @@ Doc/c-api/structures.rst Doc/c-api/sys.rst Doc/c-api/type.rst Doc/c-api/typeobj.rst -Doc/c-api/unicode.rst Doc/extending/extending.rst Doc/extending/newtypes.rst Doc/glossary.rst diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 7de48a4026303..d9ead57413cbb 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -1078,17 +1078,17 @@ code, none of the changes described here will affect you very much. To upgrade an extension module to the new API, perform the following steps: -* Rename :c:func:`Py_TPFLAGS_GC` to :c:func:`PyTPFLAGS_HAVE_GC`. +* Rename :c:macro:`!Py_TPFLAGS_GC` to :c:macro:`Py_TPFLAGS_HAVE_GC`. * Use :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar` to allocate objects, and :c:func:`PyObject_GC_Del` to deallocate them. -* Rename :c:func:`PyObject_GC_Init` to :c:func:`PyObject_GC_Track` and - :c:func:`PyObject_GC_Fini` to :c:func:`PyObject_GC_UnTrack`. +* Rename :c:func:`!PyObject_GC_Init` to :c:func:`PyObject_GC_Track` and + :c:func:`!PyObject_GC_Fini` to :c:func:`PyObject_GC_UnTrack`. -* Remove :c:func:`PyGC_HEAD_SIZE` from object size calculations. +* Remove :c:macro:`!PyGC_HEAD_SIZE` from object size calculations. -* Remove calls to :c:func:`PyObject_AS_GC` and :c:func:`PyObject_FROM_GC`. +* Remove calls to :c:func:`!PyObject_AS_GC` and :c:func:`!PyObject_FROM_GC`. * A new ``et`` format sequence was added to :c:func:`PyArg_ParseTuple`; ``et`` takes both a parameter and an encoding name, and converts the parameter to the @@ -1219,7 +1219,7 @@ Some of the more notable changes are: operator, but these features were rarely used and therefore buggy. The :meth:`tolist` method and the :attr:`start`, :attr:`stop`, and :attr:`step` attributes are also being deprecated. At the C level, the fourth argument to - the :c:func:`PyRange_New` function, ``repeat``, has also been deprecated. + the :c:func:`!PyRange_New` function, ``repeat``, has also been deprecated. * There were a bunch of patches to the dictionary implementation, mostly to fix potential core dumps if a dictionary contains objects that sneakily changed diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index 2120ee49f7d6b..ba18ce343c35d 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1897,7 +1897,7 @@ Changes to Python's build process and to the C API include: but will also mean that you can't get help for Python's built-ins. (Contributed by Gustavo Niemeyer.) -* The :c:func:`PyArg_NoArgs` macro is now deprecated, and code that uses it +* The :c:func:`!PyArg_NoArgs` macro is now deprecated, and code that uses it should be changed. For Python 2.2 and later, the method definition table can specify the :c:macro:`METH_NOARGS` flag, signalling that there are no arguments, and the argument checking can then be removed. If compatibility with pre-2.2 diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 40b404f87e515..cab321c3e54d1 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -1468,7 +1468,7 @@ Some of the changes to Python's build process and to the C API are: *X* is a NaN. (Contributed by Tim Peters.) * C code can avoid unnecessary locking by using the new - :c:func:`PyEval_ThreadsInitialized` function to tell if any thread operations + :c:func:`!PyEval_ThreadsInitialized` function to tell if any thread operations have been performed. If this function returns false, no lock operations are needed. (Contributed by Nick Coghlan.) diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index a47327d15fd79..2df6d603207a1 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -2119,9 +2119,9 @@ Changes to Python's build process and to the C API include: the various AST nodes in :file:`Parser/Python.asdl`. A Python script reads this file and generates a set of C structure definitions in :file:`Include/Python-ast.h`. The :c:func:`PyParser_ASTFromString` and - :c:func:`PyParser_ASTFromFile`, defined in :file:`Include/pythonrun.h`, take + :c:func:`!PyParser_ASTFromFile`, defined in :file:`Include/pythonrun.h`, take Python source as input and return the root of an AST representing the contents. - This AST can then be turned into a code object by :c:func:`PyAST_Compile`. For + This AST can then be turned into a code object by :c:func:`!PyAST_Compile`. For more information, read the source code, and then ask questions on python-dev. The AST code was developed under Jeremy Hylton's management, and implemented by @@ -2172,7 +2172,7 @@ Changes to Python's build process and to the C API include: ``Py_LOCAL(type)`` declares the function as returning a value of the specified *type* and uses a fast-calling qualifier. ``Py_LOCAL_INLINE(type)`` does the same thing and also requests the - function be inlined. If :c:func:`PY_LOCAL_AGGRESSIVE` is defined before + function be inlined. If macro :c:macro:`!PY_LOCAL_AGGRESSIVE` is defined before :file:`python.h` is included, a set of more aggressive optimizations are enabled for the module; you should benchmark the results to find out if these optimizations actually make the code faster. (Contributed by Fredrik Lundh at @@ -2181,7 +2181,7 @@ Changes to Python's build process and to the C API include: * ``PyErr_NewException(name, base, dict)`` can now accept a tuple of base classes as its *base* argument. (Contributed by Georg Brandl.) -* The :c:func:`PyErr_Warn` function for issuing warnings is now deprecated in +* The :c:func:`!PyErr_Warn` function for issuing warnings is now deprecated in favour of ``PyErr_WarnEx(category, message, stacklevel)`` which lets you specify the number of stack frames separating this function and the caller. A *stacklevel* of 1 is the function calling :c:func:`PyErr_WarnEx`, 2 is the @@ -2191,7 +2191,7 @@ Changes to Python's build process and to the C API include: compiled with a C++ compiler without errors. (Implemented by Anthony Baxter, Martin von L?wis, Skip Montanaro.) -* The :c:func:`PyRange_New` function was removed. It was never documented, never +* The :c:func:`!PyRange_New` function was removed. It was never documented, never used in the core code, and had dangerously lax error checking. In the unlikely case that your extensions were using it, you can replace it by something like the following:: diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 5c46dc8952154..8b3d3a324f68f 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -977,7 +977,7 @@ can be used to include Unicode characters:: print len(s) # 12 Unicode characters At the C level, Python 3.0 will rename the existing 8-bit -string type, called :c:type:`PyStringObject` in Python 2.x, +string type, called :c:type:`!PyStringObject` in Python 2.x, to :c:type:`PyBytesObject`. Python 2.6 uses ``#define`` to support using the names :c:func:`PyBytesObject`, :c:func:`PyBytes_Check`, :c:func:`PyBytes_FromStringAndSize`, @@ -3012,11 +3012,11 @@ Changes to Python's build process and to the C API include: bug occurred if one thread closed a file object while another thread was reading from or writing to the object. In 2.6 file objects have a reference count, manipulated by the - :c:func:`PyFile_IncUseCount` and :c:func:`PyFile_DecUseCount` + :c:func:`!PyFile_IncUseCount` and :c:func:`!PyFile_DecUseCount` functions. File objects can't be closed unless the reference count - is zero. :c:func:`PyFile_IncUseCount` should be called while the GIL + is zero. :c:func:`!PyFile_IncUseCount` should be called while the GIL is still held, before carrying out an I/O operation using the - ``FILE *`` pointer, and :c:func:`PyFile_DecUseCount` should be called + ``FILE *`` pointer, and :c:func:`!PyFile_DecUseCount` should be called immediately after the GIL is re-acquired. (Contributed by Antoine Pitrou and Gregory P. Smith.) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 00001f92b51c8..37e2d04d516ab 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2152,7 +2152,7 @@ Changes to Python's build process and to the C API include: * New function: stemming from the rewrite of string-to-float conversion, a new :c:func:`PyOS_string_to_double` function was added. The old - :c:func:`PyOS_ascii_strtod` and :c:func:`PyOS_ascii_atof` functions + :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof` functions are now deprecated. * New function: :c:func:`PySys_SetArgvEx` sets the value of @@ -2195,13 +2195,13 @@ Changes to Python's build process and to the C API include: .. XXX these macros don't seem to be described in the c-api docs. -* Removed function: :c:macro:`PyEval_CallObject` is now only available +* Removed function: :c:func:`!PyEval_CallObject` is now only available as a macro. A function version was being kept around to preserve ABI linking compatibility, but that was in 1997; it can certainly be deleted by now. (Removed by Antoine Pitrou; :issue:`8276`.) -* New format codes: the :c:func:`PyFormat_FromString`, - :c:func:`PyFormat_FromStringV`, and :c:func:`PyErr_Format` functions now +* New format codes: the :c:func:`!PyString_FromFormat`, + :c:func:`!PyString_FromFormatV`, and :c:func:`PyErr_Format` functions now accept ``%lld`` and ``%llu`` format codes for displaying C's :c:expr:`long long` types. (Contributed by Mark Dickinson; :issue:`7228`.) @@ -2540,7 +2540,7 @@ For C extensions: instead of triggering a :exc:`DeprecationWarning` (:issue:`5080`). * Use the new :c:func:`PyOS_string_to_double` function instead of the old - :c:func:`PyOS_ascii_strtod` and :c:func:`PyOS_ascii_atof` functions, + :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof` functions, which are now deprecated. For applications that embed Python: diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 58d42bd94cb61..b0c2529e78021 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -865,8 +865,8 @@ to the C API. * No more C API support for restricted execution. -* :c:func:`PyNumber_Coerce`, :c:func:`PyNumber_CoerceEx`, - :c:func:`PyMember_Get`, and :c:func:`PyMember_Set` C APIs are removed. +* :c:func:`!PyNumber_Coerce`, :c:func:`!PyNumber_CoerceEx`, + :c:func:`!PyMember_Get`, and :c:func:`!PyMember_Set` C APIs are removed. * New C API :c:func:`PyImport_ImportModuleNoBlock`, works like :c:func:`PyImport_ImportModule` but won't block on the import lock diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index 3c1c9c3c4bc60..e237179f4b182 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -501,12 +501,12 @@ Changes to Python's build process and to the C API include: (Contributed by Mark Dickinson and Lisandro Dalcrin; :issue:`5175`.) -* Deprecated :c:func:`PyNumber_Int`. Use :c:func:`PyNumber_Long` instead. +* Deprecated :c:func:`!PyNumber_Int`. Use :c:func:`PyNumber_Long` instead. (Contributed by Mark Dickinson; :issue:`4910`.) * Added a new :c:func:`PyOS_string_to_double` function to replace the - deprecated functions :c:func:`PyOS_ascii_strtod` and :c:func:`PyOS_ascii_atof`. + deprecated functions :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof`. (Contributed by Mark Dickinson; :issue:`5914`.) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index e4ca3c4c81ba5..1e6e0befa9819 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1819,8 +1819,8 @@ Removed into their code. (Contributed by Dong-hee Na and Terry J. Reedy in :issue:`42299`.) -* Removed the :c:func:`PyModule_GetWarningsModule` function that was useless - now due to the _warnings module was converted to a builtin module in 2.6. +* Removed the :c:func:`!PyModule_GetWarningsModule` function that was useless + now due to the :mod:`!_warnings` module was converted to a builtin module in 2.6. (Contributed by Hai Shi in :issue:`42599`.) * Remove deprecated aliases to :ref:`collections-abstract-base-classes` from diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index fce00d37555c2..58ce881e9787f 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2216,7 +2216,7 @@ New Features * :c:func:`PyBuffer_SizeFromFormat` * :c:func:`PyBuffer_ToContiguous` * :c:func:`PyBuffer_FromContiguous` - * :c:func:`PyBuffer_CopyData` + * :c:func:`PyObject_CopyData` * :c:func:`PyBuffer_IsContiguous` * :c:func:`PyBuffer_FillContiguousStrides` * :c:func:`PyBuffer_FillInfo` @@ -2562,18 +2562,18 @@ Deprecated * Deprecate the following functions to configure the Python initialization: - * :c:func:`PySys_AddWarnOptionUnicode` - * :c:func:`PySys_AddWarnOption` - * :c:func:`PySys_AddXOption` - * :c:func:`PySys_HasWarnOptions` - * :c:func:`PySys_SetArgvEx` - * :c:func:`PySys_SetArgv` - * :c:func:`PySys_SetPath` - * :c:func:`Py_SetPath` - * :c:func:`Py_SetProgramName` - * :c:func:`Py_SetPythonHome` - * :c:func:`Py_SetStandardStreamEncoding` - * :c:func:`_Py_SetProgramFullPath` + * :c:func:`!PySys_AddWarnOptionUnicode` + * :c:func:`!PySys_AddWarnOption` + * :c:func:`!PySys_AddXOption` + * :c:func:`!PySys_HasWarnOptions` + * :c:func:`!PySys_SetArgvEx` + * :c:func:`!PySys_SetArgv` + * :c:func:`!PySys_SetPath` + * :c:func:`!Py_SetPath` + * :c:func:`!Py_SetProgramName` + * :c:func:`!Py_SetPythonHome` + * :c:func:`!Py_SetStandardStreamEncoding` + * :c:func:`!_Py_SetProgramFullPath` Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization Configuration ` instead (:pep:`587`). diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 56ac5c4c0a1c1..df32b76b6d7b0 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -2569,7 +2569,7 @@ Changes to Python's build process and to the C API include: to set :data:`sys.argv` without also modifying :data:`sys.path` (:issue:`5753`). -* :c:macro:`PyEval_CallObject` is now only available in macro form. The +* :c:func:`!PyEval_CallObject` is now only available in macro form. The function declaration, which was kept for backwards compatibility reasons, is now removed -- the macro was introduced in 1997 (:issue:`8276`). @@ -2731,15 +2731,15 @@ require changes to your code: (Contributed by Antoine Pitrou, :issue:`10272`.) -* The misleading functions :c:func:`PyEval_AcquireLock()` and - :c:func:`PyEval_ReleaseLock()` have been officially deprecated. The - thread-state aware APIs (such as :c:func:`PyEval_SaveThread()` - and :c:func:`PyEval_RestoreThread()`) should be used instead. +* The misleading functions :c:func:`!PyEval_AcquireLock` and + :c:func:`!PyEval_ReleaseLock` have been officially deprecated. The + thread-state aware APIs (such as :c:func:`PyEval_SaveThread` + and :c:func:`PyEval_RestoreThread`) should be used instead. * Due to security risks, :func:`asyncore.handle_accept` has been deprecated, and a new function, :func:`asyncore.handle_accepted`, was added to replace it. (Contributed by Giampaolo Rodola in :issue:`6706`.) -* Due to the new :term:`GIL` implementation, :c:func:`PyEval_InitThreads()` - cannot be called before :c:func:`Py_Initialize()` anymore. +* Due to the new :term:`GIL` implementation, :c:func:`!PyEval_InitThreads` + cannot be called before :c:func:`Py_Initialize` anymore. diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 7e12c27906567..3f98c82c2fa55 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -2304,7 +2304,7 @@ Functions and macros manipulating Py_UNICODE* strings: Encoders: -* :c:func:`!PyUnicode_Encode`: use :c:func:`PyUnicode_AsEncodedObject` +* :c:func:`!PyUnicode_Encode`: use :c:func:`!PyUnicode_AsEncodedObject` * :c:func:`!PyUnicode_EncodeUTF7` * :c:func:`!PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or :c:func:`PyUnicode_AsUTF8String` @@ -2462,7 +2462,7 @@ Porting C code -------------- * In the course of changes to the buffer API the undocumented - :c:member:`~Py_buffer.smalltable` member of the + :c:member:`!smalltable` member of the :c:type:`Py_buffer` structure has been removed and the layout of the :c:type:`PyMemoryViewObject` has changed. diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 86bfdc4d478a0..0c45a42d1a7c1 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -2512,7 +2512,7 @@ Changes in the Python API Changes in the C API -------------------- -* The undocumented :c:member:`~PyMemoryViewObject.format` member of the +* The undocumented :c:member:`!format` member of the (non-public) :c:type:`PyMemoryViewObject` structure has been removed. All extensions relying on the relevant parts in ``memoryobject.h`` must be rebuilt. @@ -2520,7 +2520,7 @@ Changes in the C API * The :c:type:`PyMemAllocator` structure was renamed to :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. -* Removed non-documented macro :c:macro:`PyObject_REPR` which leaked references. +* Removed non-documented macro :c:macro:`!PyObject_REPR()` which leaked references. Use format character ``%R`` in :c:func:`PyUnicode_FromFormat`-like functions to format the :func:`repr` of the object. (Contributed by Serhiy Storchaka in :issue:`22453`.) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index c5fb5c53dfe35..7946bf910af2c 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -1574,12 +1574,12 @@ Build and C API Changes * :c:func:`Py_INCREF`, :c:func:`Py_DECREF` * :c:func:`Py_XINCREF`, :c:func:`Py_XDECREF` * :c:func:`PyObject_INIT`, :c:func:`PyObject_INIT_VAR` - * Private functions: :c:func:`_PyObject_GC_TRACK`, - :c:func:`_PyObject_GC_UNTRACK`, :c:func:`_Py_Dealloc` + * Private functions: :c:func:`!_PyObject_GC_TRACK`, + :c:func:`!_PyObject_GC_UNTRACK`, :c:func:`!_Py_Dealloc` (Contributed by Victor Stinner in :issue:`35059`.) -* The :c:func:`PyByteArray_Init` and :c:func:`PyByteArray_Fini` functions have +* The :c:func:`!PyByteArray_Init` and :c:func:`!PyByteArray_Fini` functions have been removed. They did nothing since Python 2.7.4 and Python 3.2.0, were excluded from the limited API (stable ABI), and were not documented. (Contributed by Victor Stinner in :issue:`35713`.) @@ -1628,7 +1628,7 @@ Build and C API Changes parameter for indicating the number of positional-only arguments. (Contributed by Pablo Galindo in :issue:`37221`.) -* :c:func:`Py_SetPath` now sets :data:`sys.executable` to the program full +* :c:func:`!Py_SetPath` now sets :data:`sys.executable` to the program full path (:c:func:`Py_GetProgramFullPath`) rather than to the program name (:c:func:`Py_GetProgramName`). (Contributed by Victor Stinner in :issue:`38234`.) @@ -1845,11 +1845,11 @@ Changes in Python behavior always use ``sys.platform.startswith('aix')``. (Contributed by M. Felt in :issue:`36588`.) -* :c:func:`PyEval_AcquireLock` and :c:func:`PyEval_AcquireThread` now +* :c:func:`!PyEval_AcquireLock` and :c:func:`!PyEval_AcquireThread` now terminate the current thread if called while the interpreter is finalizing, making them consistent with :c:func:`PyEval_RestoreThread`, :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`. If this - behavior is not desired, guard the call by checking :c:func:`_Py_IsFinalizing` + behavior is not desired, guard the call by checking :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing`. (Contributed by Joannah Nanjekye in :issue:`36475`.) @@ -2021,7 +2021,7 @@ Changes in the C API *cf_flags*. (Contributed by Guido van Rossum in :issue:`35766`.) -* The :c:func:`PyEval_ReInitThreads` function has been removed from the C API. +* The :c:func:`!PyEval_ReInitThreads` function has been removed from the C API. It should not be called explicitly: use :c:func:`PyOS_AfterFork_Child` instead. (Contributed by Victor Stinner in :issue:`36728`.) @@ -2121,7 +2121,7 @@ Changes in the C API (Contributed by Antoine Pitrou in :issue:`32388`.) -* The functions :c:func:`PyNode_AddChild` and :c:func:`PyParser_AddToken` now accept +* The functions :c:func:`!PyNode_AddChild` and :c:func:`!PyParser_AddToken` now accept two additional ``int`` arguments *end_lineno* and *end_col_offset*. * The :file:`libpython38.a` file to allow MinGW tools to link directly against diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 5891d9a55534f..8e2df19419bfc 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -870,9 +870,9 @@ Deprecated users can leverage the Abstract Syntax Tree (AST) generation and compilation stage, using the :mod:`ast` module. -* The Public C API functions :c:func:`PyParser_SimpleParseStringFlags`, - :c:func:`PyParser_SimpleParseStringFlagsFilename`, - :c:func:`PyParser_SimpleParseFileFlags` and :c:func:`PyNode_Compile` +* The Public C API functions :c:func:`!PyParser_SimpleParseStringFlags`, + :c:func:`!PyParser_SimpleParseStringFlagsFilename`, + :c:func:`!PyParser_SimpleParseFileFlags` and :c:func:`!PyNode_Compile` are deprecated and will be removed in Python 3.10 together with the old parser. * Using :data:`NotImplemented` in a boolean context has been deprecated, @@ -923,10 +923,10 @@ Deprecated (Contributed by Batuhan Taskaya in :issue:`39639` and :issue:`39969` and Serhiy Storchaka in :issue:`39988`.) -* The :c:func:`PyEval_InitThreads` and :c:func:`PyEval_ThreadsInitialized` +* The :c:func:`!PyEval_InitThreads` and :c:func:`!PyEval_ThreadsInitialized` functions are now deprecated and will be removed in Python 3.11. Calling - :c:func:`PyEval_InitThreads` now does nothing. The :term:`GIL` is initialized - by :c:func:`Py_Initialize()` since Python 3.7. + :c:func:`!PyEval_InitThreads` now does nothing. The :term:`GIL` is initialized + by :c:func:`Py_Initialize` since Python 3.7. (Contributed by Victor Stinner in :issue:`39877`.) * Passing ``None`` as the first argument to the :func:`shlex.split` function diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 17ee5138dbf90..e1d0adc478029 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -5036,15 +5036,15 @@ Limited API. Deprecate the following functions to configure the Python initialization: -* :c:func:`PySys_AddWarnOptionUnicode` -* :c:func:`PySys_AddWarnOption` -* :c:func:`PySys_AddXOption` -* :c:func:`PySys_HasWarnOptions` -* :c:func:`Py_SetPath` -* :c:func:`Py_SetProgramName` -* :c:func:`Py_SetPythonHome` -* :c:func:`Py_SetStandardStreamEncoding` -* :c:func:`_Py_SetProgramFullPath` +* :c:func:`!PySys_AddWarnOptionUnicode` +* :c:func:`!PySys_AddWarnOption` +* :c:func:`!PySys_AddXOption` +* :c:func:`!PySys_HasWarnOptions` +* :c:func:`!Py_SetPath` +* :c:func:`!Py_SetProgramName` +* :c:func:`!Py_SetPythonHome` +* :c:func:`!Py_SetStandardStreamEncoding` +* :c:func:`!_Py_SetProgramFullPath` Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization Configuration ` instead (:pep:`587`). diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 467e992780382..dbbfb6e8b0d68 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -8765,7 +8765,7 @@ for relative path to files in current directory. .. nonce: fmehdG .. section: C API -The :c:func:`PyByteArray_Init` and :c:func:`PyByteArray_Fini` functions have +The :c:func:`!PyByteArray_Init` and :c:func:`!PyByteArray_Fini` functions have been removed. They did nothing since Python 2.7.4 and Python 3.2.0, were excluded from the limited API (stable ABI), and were not documented. @@ -8836,7 +8836,7 @@ Py_LIMITED_API. Patch by Arthur Neufeld. .. nonce: gFd85N .. section: C API -The :c:func:`_PyObject_GC_TRACK` and :c:func:`_PyObject_GC_UNTRACK` macros +The :c:func:`!_PyObject_GC_TRACK` and :c:func:`!_PyObject_GC_UNTRACK` macros have been removed from the public C API. .. diff --git a/Misc/NEWS.d/3.8.0a4.rst b/Misc/NEWS.d/3.8.0a4.rst index 524a05a7ae970..2ce60f39539e8 100644 --- a/Misc/NEWS.d/3.8.0a4.rst +++ b/Misc/NEWS.d/3.8.0a4.rst @@ -124,7 +124,7 @@ Galindo. .. nonce: CjRps3 .. section: Core and Builtins -:c:func:`PyEval_AcquireLock` and :c:func:`PyEval_AcquireThread` now +:c:func:`!PyEval_AcquireLock` and :c:func:`!PyEval_AcquireThread` now terminate the current thread if called while the interpreter is finalizing, making them consistent with :c:func:`PyEval_RestoreThread`, :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`. diff --git a/Misc/NEWS.d/3.8.0b1.rst b/Misc/NEWS.d/3.8.0b1.rst index 72e6b31874e37..c5e27b04ef8e8 100644 --- a/Misc/NEWS.d/3.8.0b1.rst +++ b/Misc/NEWS.d/3.8.0b1.rst @@ -2047,6 +2047,6 @@ unbound methods. These are objects supporting the optimization given by the .. nonce: FR-dMP .. section: C API -The :c:func:`PyEval_ReInitThreads` function has been removed from the C API. +The :c:func:`!PyEval_ReInitThreads` function has been removed from the C API. It should not be called explicitly: use :c:func:`PyOS_AfterFork_Child` instead. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index 43676836478c9..5a4431b0fcf1c 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -5535,7 +5535,7 @@ Tyler Kieft. .. nonce: d0bhEA .. section: C API -:c:func:`Py_SetPath` now sets :data:`sys.executable` to the program full +:c:func:`!Py_SetPath` now sets :data:`sys.executable` to the program full path (:c:func:`Py_GetProgramFullPath`) rather than to the program name (:c:func:`Py_GetProgramName`). @@ -5546,8 +5546,8 @@ path (:c:func:`Py_GetProgramFullPath`) rather than to the program name .. nonce: ZbquVK .. section: C API -Python ignored arguments passed to :c:func:`Py_SetPath`, -:c:func:`Py_SetPythonHome` and :c:func:`Py_SetProgramName`: fix Python +Python ignored arguments passed to :c:func:`!Py_SetPath`, +:c:func:`!Py_SetPythonHome` and :c:func:`!Py_SetProgramName`: fix Python initialization to use specified arguments. .. diff --git a/Misc/NEWS.d/3.9.0a5.rst b/Misc/NEWS.d/3.9.0a5.rst index 8a1219501e81b..19ad20ad3db04 100644 --- a/Misc/NEWS.d/3.9.0a5.rst +++ b/Misc/NEWS.d/3.9.0a5.rst @@ -1234,8 +1234,8 @@ method name in the SystemError "bad call flags" error message to ease debug. .. nonce: GOYtIm .. section: C API -Deprecated :c:func:`PyEval_InitThreads` and -:c:func:`PyEval_ThreadsInitialized`. Calling :c:func:`PyEval_InitThreads` +Deprecated :c:func:`!PyEval_InitThreads` and +:c:func:`!PyEval_ThreadsInitialized`. Calling :c:func:`!PyEval_InitThreads` now does nothing. .. From webhook-mailer at python.org Tue Aug 22 09:21:07 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 22 Aug 2023 13:21:07 -0000 Subject: [Python-checkins] Docs: align the param spec of sqlite3.Connection methods with the implementation (#108285) Message-ID: https://github.com/python/cpython/commit/893215a4e7f59eabb8ccdf188c4b9b1de5bd8966 commit: 893215a4e7f59eabb8ccdf188c4b9b1de5bd8966 branch: main author: Erlend E. Aasland committer: erlend-aasland date: 2023-08-22T13:20:58Z summary: Docs: align the param spec of sqlite3.Connection methods with the implementation (#108285) - no parameters of create_aggregate() are positional-only - all parameters of create_collation() are positional-only files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index d2745b0dbceb2..0b53f97b284f0 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -764,7 +764,7 @@ Connection objects ('acbd18db4cc2f85cedef654fccc4a4d8',) - .. method:: create_aggregate(name, /, n_arg, aggregate_class) + .. method:: create_aggregate(name, n_arg, aggregate_class) Create or remove a user-defined SQL aggregate function. @@ -904,7 +904,7 @@ Connection objects [('a', 9), ('b', 12), ('c', 16), ('d', 12), ('e', 9)] - .. method:: create_collation(name, callable) + .. method:: create_collation(name, callable, /) Create a collation named *name* using the collating function *callable*. *callable* is passed two :class:`string ` arguments, From webhook-mailer at python.org Tue Aug 22 09:28:42 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Tue, 22 Aug 2023 13:28:42 -0000 Subject: [Python-checkins] [3.11] Docs: align the param spec of sqlite3.Connection methods with the implementation (GH-108285) (#108288) Message-ID: https://github.com/python/cpython/commit/d22ac0c6052c4e568c316ce3ce194fdf86d9af87 commit: d22ac0c6052c4e568c316ce3ce194fdf86d9af87 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: erlend-aasland date: 2023-08-22T13:28:38Z summary: [3.11] Docs: align the param spec of sqlite3.Connection methods with the implementation (GH-108285) (#108288) - no parameters of create_aggregate() are positional-only - all parameters of create_collation() are positional-only (cherry picked from commit 893215a4e7f59eabb8ccdf188c4b9b1de5bd8966) Co-authored-by: Erlend E. Aasland files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 13e61b3d68006..53c21aed2d609 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -679,7 +679,7 @@ Connection objects ('acbd18db4cc2f85cedef654fccc4a4d8',) - .. method:: create_aggregate(name, /, n_arg, aggregate_class) + .. method:: create_aggregate(name, n_arg, aggregate_class) Create or remove a user-defined SQL aggregate function. @@ -819,7 +819,7 @@ Connection objects [('a', 9), ('b', 12), ('c', 16), ('d', 12), ('e', 9)] - .. method:: create_collation(name, callable) + .. method:: create_collation(name, callable, /) Create a collation named *name* using the collating function *callable*. *callable* is passed two :class:`string ` arguments, From webhook-mailer at python.org Tue Aug 22 09:52:37 2023 From: webhook-mailer at python.org (vstinner) Date: Tue, 22 Aug 2023 13:52:37 -0000 Subject: [Python-checkins] gh-90791: Enable test___all__ on ASAN build (#108286) Message-ID: https://github.com/python/cpython/commit/a541e01537cc07b326cc37cc29412b816bc2b4fc commit: a541e01537cc07b326cc37cc29412b816bc2b4fc branch: main author: Victor Stinner committer: vstinner date: 2023-08-22T15:52:32+02:00 summary: gh-90791: Enable test___all__ on ASAN build (#108286) * Only skip modules and tests related to X11 on ASAN builds: run other tests with ASAN. * Use print(flush=True) to see output earlier when it's redirected to a pipe. * Update issue reference: replace bpo-46633 with gh-90791. files: M Lib/test/test___all__.py M Lib/test/test_concurrent_futures.py M Lib/test/test_idle.py M Lib/test/test_peg_generator/__init__.py M Lib/test/test_tkinter/__init__.py M Lib/test/test_tools/__init__.py diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index 2163f1636566b..c87cde4b3d1fa 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -12,10 +12,19 @@ if support.check_sanitizer(address=True, memory=True): - # bpo-46633: test___all__ is skipped because importing some modules - # directly can trigger known problems with ASAN (like tk). - raise unittest.SkipTest("workaround ASAN build issues on loading tests " - "like tk") + SKIP_MODULES = frozenset(( + # gh-90791: Tests involving libX11 can SEGFAULT on ASAN/MSAN builds. + # Skip modules, packages and tests using '_tkinter'. + '_tkinter', + 'tkinter', + 'test_tkinter', + 'test_ttk', + 'test_ttk_textonly', + 'idlelib', + 'test_idle', + )) +else: + SKIP_MODULES = () class NoAll(RuntimeError): @@ -83,15 +92,23 @@ def walk_modules(self, basedir, modpath): for fn in sorted(os.listdir(basedir)): path = os.path.join(basedir, fn) if os.path.isdir(path): + if fn in SKIP_MODULES: + continue pkg_init = os.path.join(path, '__init__.py') if os.path.exists(pkg_init): yield pkg_init, modpath + fn for p, m in self.walk_modules(path, modpath + fn + "."): yield p, m continue - if not fn.endswith('.py') or fn == '__init__.py': + + if fn == '__init__.py': + continue + if not fn.endswith('.py'): + continue + modname = fn.removesuffix('.py') + if modname in SKIP_MODULES: continue - yield path, modpath + fn[:-3] + yield path, modpath + modname def test_all(self): # List of denied modules and packages @@ -119,14 +136,14 @@ def test_all(self): if denied: continue if support.verbose: - print(modname) + print(f"Check {modname}", flush=True) try: # This heuristic speeds up the process by removing, de facto, # most test modules (and avoiding the auto-executing ones). with open(path, "rb") as f: if b"__all__" not in f.read(): raise NoAll(modname) - self.check_all(modname) + self.check_all(modname) except NoAll: ignored.append(modname) except FailedImport: diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index 39dbe234e765e..0f0ea6190def0 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -34,7 +34,7 @@ if support.check_sanitizer(address=True, memory=True): - # bpo-46633: Skip the test because it is too slow when Python is built + # gh-90791: Skip the test because it is too slow when Python is built # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. raise unittest.SkipTest("test too slow on ASAN/MSAN build") diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index ebb1e8eb823b5..3d8b7ecc0ecb6 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -3,6 +3,7 @@ from test.support import check_sanitizer if check_sanitizer(address=True, memory=True): + # See gh-90791 for details raise unittest.SkipTest("Tests involving libX11 can SEGFAULT on ASAN/MSAN builds") # Skip test_idle if _tkinter, tkinter, or idlelib are missing. diff --git a/Lib/test/test_peg_generator/__init__.py b/Lib/test/test_peg_generator/__init__.py index 77f72fcc7c6e3..87281eb5e03c7 100644 --- a/Lib/test/test_peg_generator/__init__.py +++ b/Lib/test/test_peg_generator/__init__.py @@ -5,7 +5,7 @@ if support.check_sanitizer(address=True, memory=True): - # bpo-46633: Skip the test because it is too slow when Python is built + # gh-90791: Skip the test because it is too slow when Python is built # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. raise unittest.SkipTest("test too slow on ASAN/MSAN build") diff --git a/Lib/test/test_tkinter/__init__.py b/Lib/test/test_tkinter/__init__.py index b1181bc04b795..aa196c12c804a 100644 --- a/Lib/test/test_tkinter/__init__.py +++ b/Lib/test/test_tkinter/__init__.py @@ -10,6 +10,7 @@ if check_sanitizer(address=True, memory=True): + # See gh-90791 for details raise unittest.SkipTest("Tests involving libX11 can SEGFAULT on ASAN/MSAN builds") # Skip test if _tkinter wasn't built. diff --git a/Lib/test/test_tools/__init__.py b/Lib/test/test_tools/__init__.py index 2102b9fceff56..dde5d84e9edc6 100644 --- a/Lib/test/test_tools/__init__.py +++ b/Lib/test/test_tools/__init__.py @@ -8,7 +8,7 @@ if support.check_sanitizer(address=True, memory=True): - # bpo-46633: Skip the test because it is too slow when Python is built + # gh-90791: Skip the test because it is too slow when Python is built # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. raise unittest.SkipTest("test too slow on ASAN/MSAN build") From webhook-mailer at python.org Tue Aug 22 10:15:41 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 22 Aug 2023 14:15:41 -0000 Subject: [Python-checkins] [3.12] gh-107901: Fix missing line number on BACKWARD_JUMP at the end of a for loop (GH-108242) (#108275) Message-ID: https://github.com/python/cpython/commit/4ee945479243125ea5c50be526e754df4ab9c749 commit: 4ee945479243125ea5c50be526e754df4ab9c749 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-22T16:15:34+02:00 summary: [3.12] gh-107901: Fix missing line number on BACKWARD_JUMP at the end of a for loop (GH-108242) (#108275) gh-107901: Fix missing line number on BACKWARD_JUMP at the end of a for loop (GH-108242) (cherry picked from commit a1cc74c4eebc55795877eb3be019a1bec34402f8) Co-authored-by: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> files: A Misc/NEWS.d/next/Core and Builtins/2023-08-21-21-13-30.gh-issue-107901.hszvdk.rst M Lib/test/test_compile.py M Python/flowgraph.c diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 142d3f5c125b6..55e55c7d2884d 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1031,6 +1031,20 @@ async def test(aseq): code_lines = self.get_code_lines(test.__code__) self.assertEqual(expected_lines, code_lines) + def test_lineno_of_backward_jump(self): + # Issue gh-107901 + def f(): + for i in x: + if y: + pass + + linenos = list(inst.positions.lineno + for inst in dis.get_instructions(f.__code__) + if inst.opname == 'JUMP_BACKWARD') + + self.assertTrue(len(linenos) > 0) + self.assertTrue(all(l is not None for l in linenos)) + def test_big_dict_literal(self): # The compiler has a flushing point in "compiler_dict" that calls compiles # a portion of the dictionary literal when the loop that iterates over the items diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-21-21-13-30.gh-issue-107901.hszvdk.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-21-21-13-30.gh-issue-107901.hszvdk.rst new file mode 100644 index 0000000000000..112e093736dd5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-21-21-13-30.gh-issue-107901.hszvdk.rst @@ -0,0 +1 @@ +Fix missing line number on :opcode:`JUMP_BACKWARD` at the end of a for loop. diff --git a/Python/flowgraph.c b/Python/flowgraph.c index f8039a4985d94..4fe581beb5fc6 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -450,7 +450,7 @@ normalize_jumps_in_block(cfg_builder *g, basicblock *b) { if (backwards_jump == NULL) { return ERROR; } - basicblock_addop(backwards_jump, JUMP, target->b_label.id, NO_LOCATION); + basicblock_addop(backwards_jump, JUMP, target->b_label.id, last->i_loc); backwards_jump->b_instr[0].i_target = target; last->i_opcode = reversed_opcode; last->i_target = b->b_next; From webhook-mailer at python.org Tue Aug 22 10:16:17 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 22 Aug 2023 14:16:17 -0000 Subject: [Python-checkins] [3.12] gh-106971: Docs: Add missing issue reference (GH-106992) (#108283) Message-ID: https://github.com/python/cpython/commit/a2b680d9e994973e7a5a53671e1a733882d29f28 commit: a2b680d9e994973e7a5a53671e1a733882d29f28 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-22T16:15:59+02:00 summary: [3.12] gh-106971: Docs: Add missing issue reference (GH-106992) (#108283) gh-106971: Docs: Add missing issue reference (GH-106992) (cherry picked from commit c556f9a3c9af48c9af9e1f298be638553a6c886e) Co-authored-by: Junya Fukuda Co-authored-by: Hugo van Kemenade files: M Doc/whatsnew/3.12.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index bf9eadcaefc85..2084f4ed0b9bc 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1201,7 +1201,7 @@ Pending Removal in Python 3.14 is deprecated and will raise an exception in Python 3.14. * Creating immutable types (:c:macro:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable - bases using the C API. + bases using the C API (:gh:`95388`). * ``__package__`` and ``__cached__`` will cease to be set or taken into consideration by the import system (:gh:`97879`). From webhook-mailer at python.org Tue Aug 22 10:16:35 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 22 Aug 2023 14:16:35 -0000 Subject: [Python-checkins] [3.12] gh-107298: Fix numerous ref errors and typos in the C API docs (GH-108258) (#108284) Message-ID: https://github.com/python/cpython/commit/e5d779c0e296ed6247d911231dad39c16dec0172 commit: e5d779c0e296ed6247d911231dad39c16dec0172 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-22T16:16:31+02:00 summary: [3.12] gh-107298: Fix numerous ref errors and typos in the C API docs (GH-108258) (#108284) gh-107298: Fix numerous ref errors and typos in the C API docs (GH-108258) (cherry picked from commit d7202e4879bf4e7e00a69500ddcb3143864139b4) Co-authored-by: Serhiy Storchaka files: M Doc/c-api/exceptions.rst M Doc/c-api/init_config.rst M Doc/c-api/module.rst M Doc/c-api/unicode.rst M Doc/extending/newtypes.rst M Doc/tools/.nitignore M Doc/whatsnew/2.2.rst M Doc/whatsnew/2.3.rst M Doc/whatsnew/2.4.rst M Doc/whatsnew/2.5.rst M Doc/whatsnew/2.6.rst M Doc/whatsnew/2.7.rst M Doc/whatsnew/3.0.rst M Doc/whatsnew/3.1.rst M Doc/whatsnew/3.10.rst M Doc/whatsnew/3.11.rst M Doc/whatsnew/3.2.rst M Doc/whatsnew/3.3.rst M Doc/whatsnew/3.5.rst M Doc/whatsnew/3.8.rst M Doc/whatsnew/3.9.rst M Misc/NEWS.d/3.11.0a1.rst M Misc/NEWS.d/3.8.0a1.rst M Misc/NEWS.d/3.8.0a4.rst M Misc/NEWS.d/3.8.0b1.rst M Misc/NEWS.d/3.9.0a1.rst M Misc/NEWS.d/3.9.0a5.rst diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index f1d6c995188ab..6e2ac0a40a5f1 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -222,17 +222,21 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the - filename is given as a C string. *filename* is decoded from the filesystem - encoding (:func:`os.fsdecode`). + Similar to :c:func:`PyErr_SetFromWindowsErr`, with the additional behavior + that if *filename* is not ``NULL``, it is decoded from the filesystem + encoding (:func:`os.fsdecode`) and passed to the constructor of + :exc:`OSError` as a third parameter to be used to define the + :attr:`!filename` attribute of the exception instance. .. availability:: Windows. .. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an - additional parameter specifying the exception type to be raised. + Similar to :c:func:`PyErr_SetExcFromWindowsErr`, with the additional behavior + that if *filename* is not ``NULL``, it is passed to the constructor of + :exc:`OSError` as a third parameter to be used to define the + :attr:`!filename` attribute of the exception instance. .. availability:: Windows. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 96705c804f11d..7c5465b5bfa20 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -82,6 +82,8 @@ PyWideStringList If *length* is non-zero, *items* must be non-``NULL`` and all strings must be non-``NULL``. + .. c:namespace:: NULL + Methods: .. c:function:: PyStatus PyWideStringList_Append(PyWideStringList *list, const wchar_t *item) @@ -101,6 +103,8 @@ PyWideStringList Python must be preinitialized to call this function. + .. c:namespace:: PyWideStringList + Structure fields: .. c:member:: Py_ssize_t length diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index cb6b0a47681e9..f941f0c7d42b2 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -338,6 +338,7 @@ The available slot types are: The *value* pointer of this slot must point to a function of the signature: .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) + :noindex: The function receives a :py:class:`~importlib.machinery.ModuleSpec` instance, as defined in :PEP:`451`, and the module definition. @@ -372,6 +373,7 @@ The available slot types are: The signature of the function is: .. c:function:: int exec_module(PyObject* module) + :noindex: If multiple ``Py_mod_exec`` slots are specified, they are processed in the order they appear in the *m_slots* array. @@ -380,6 +382,8 @@ The available slot types are: Specifies one of the following values: + .. c:namespace:: NULL + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED The module does not support being imported in subinterpreters. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 586ad182e3a45..c10c15111cd90 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1299,7 +1299,7 @@ the user settings on the machine running the codec. Encode the Unicode object using the specified code page and return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. Use - :c:macro:`CP_ACP` code page to get the MBCS encoder. + :c:macro:`!CP_ACP` code page to get the MBCS encoder. .. versionadded:: 3.3 diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 386b3c8f4452c..9f166eb8a4c3f 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -298,7 +298,7 @@ have an associated doc string simply by providing the text in the table. An application can use the introspection API to retrieve the descriptor from the class object, and get the doc string using its :attr:`__doc__` attribute. -As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.name` value +As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.ml_name` value of ``NULL`` is required. .. XXX Descriptors need to be explained in more detail somewhere, but not here. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 4eb466a909a49..2ba307df21d85 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -21,7 +21,6 @@ Doc/c-api/structures.rst Doc/c-api/sys.rst Doc/c-api/type.rst Doc/c-api/typeobj.rst -Doc/c-api/unicode.rst Doc/extending/extending.rst Doc/extending/newtypes.rst Doc/faq/gui.rst diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 7de48a4026303..d9ead57413cbb 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -1078,17 +1078,17 @@ code, none of the changes described here will affect you very much. To upgrade an extension module to the new API, perform the following steps: -* Rename :c:func:`Py_TPFLAGS_GC` to :c:func:`PyTPFLAGS_HAVE_GC`. +* Rename :c:macro:`!Py_TPFLAGS_GC` to :c:macro:`Py_TPFLAGS_HAVE_GC`. * Use :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar` to allocate objects, and :c:func:`PyObject_GC_Del` to deallocate them. -* Rename :c:func:`PyObject_GC_Init` to :c:func:`PyObject_GC_Track` and - :c:func:`PyObject_GC_Fini` to :c:func:`PyObject_GC_UnTrack`. +* Rename :c:func:`!PyObject_GC_Init` to :c:func:`PyObject_GC_Track` and + :c:func:`!PyObject_GC_Fini` to :c:func:`PyObject_GC_UnTrack`. -* Remove :c:func:`PyGC_HEAD_SIZE` from object size calculations. +* Remove :c:macro:`!PyGC_HEAD_SIZE` from object size calculations. -* Remove calls to :c:func:`PyObject_AS_GC` and :c:func:`PyObject_FROM_GC`. +* Remove calls to :c:func:`!PyObject_AS_GC` and :c:func:`!PyObject_FROM_GC`. * A new ``et`` format sequence was added to :c:func:`PyArg_ParseTuple`; ``et`` takes both a parameter and an encoding name, and converts the parameter to the @@ -1219,7 +1219,7 @@ Some of the more notable changes are: operator, but these features were rarely used and therefore buggy. The :meth:`tolist` method and the :attr:`start`, :attr:`stop`, and :attr:`step` attributes are also being deprecated. At the C level, the fourth argument to - the :c:func:`PyRange_New` function, ``repeat``, has also been deprecated. + the :c:func:`!PyRange_New` function, ``repeat``, has also been deprecated. * There were a bunch of patches to the dictionary implementation, mostly to fix potential core dumps if a dictionary contains objects that sneakily changed diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index a9c44d501701c..c06fd9582627e 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1897,7 +1897,7 @@ Changes to Python's build process and to the C API include: but will also mean that you can't get help for Python's built-ins. (Contributed by Gustavo Niemeyer.) -* The :c:func:`PyArg_NoArgs` macro is now deprecated, and code that uses it +* The :c:func:`!PyArg_NoArgs` macro is now deprecated, and code that uses it should be changed. For Python 2.2 and later, the method definition table can specify the :c:macro:`METH_NOARGS` flag, signalling that there are no arguments, and the argument checking can then be removed. If compatibility with pre-2.2 diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 4272928fb6789..ceda03afd5f4f 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -1468,7 +1468,7 @@ Some of the changes to Python's build process and to the C API are: *X* is a NaN. (Contributed by Tim Peters.) * C code can avoid unnecessary locking by using the new - :c:func:`PyEval_ThreadsInitialized` function to tell if any thread operations + :c:func:`!PyEval_ThreadsInitialized` function to tell if any thread operations have been performed. If this function returns false, no lock operations are needed. (Contributed by Nick Coghlan.) diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index a15eefc2953f4..f58b3ede27fac 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -2119,9 +2119,9 @@ Changes to Python's build process and to the C API include: the various AST nodes in :file:`Parser/Python.asdl`. A Python script reads this file and generates a set of C structure definitions in :file:`Include/Python-ast.h`. The :c:func:`PyParser_ASTFromString` and - :c:func:`PyParser_ASTFromFile`, defined in :file:`Include/pythonrun.h`, take + :c:func:`!PyParser_ASTFromFile`, defined in :file:`Include/pythonrun.h`, take Python source as input and return the root of an AST representing the contents. - This AST can then be turned into a code object by :c:func:`PyAST_Compile`. For + This AST can then be turned into a code object by :c:func:`!PyAST_Compile`. For more information, read the source code, and then ask questions on python-dev. The AST code was developed under Jeremy Hylton's management, and implemented by @@ -2172,7 +2172,7 @@ Changes to Python's build process and to the C API include: ``Py_LOCAL(type)`` declares the function as returning a value of the specified *type* and uses a fast-calling qualifier. ``Py_LOCAL_INLINE(type)`` does the same thing and also requests the - function be inlined. If :c:func:`PY_LOCAL_AGGRESSIVE` is defined before + function be inlined. If macro :c:macro:`!PY_LOCAL_AGGRESSIVE` is defined before :file:`python.h` is included, a set of more aggressive optimizations are enabled for the module; you should benchmark the results to find out if these optimizations actually make the code faster. (Contributed by Fredrik Lundh at @@ -2181,7 +2181,7 @@ Changes to Python's build process and to the C API include: * ``PyErr_NewException(name, base, dict)`` can now accept a tuple of base classes as its *base* argument. (Contributed by Georg Brandl.) -* The :c:func:`PyErr_Warn` function for issuing warnings is now deprecated in +* The :c:func:`!PyErr_Warn` function for issuing warnings is now deprecated in favour of ``PyErr_WarnEx(category, message, stacklevel)`` which lets you specify the number of stack frames separating this function and the caller. A *stacklevel* of 1 is the function calling :c:func:`PyErr_WarnEx`, 2 is the @@ -2191,7 +2191,7 @@ Changes to Python's build process and to the C API include: compiled with a C++ compiler without errors. (Implemented by Anthony Baxter, Martin von L?wis, Skip Montanaro.) -* The :c:func:`PyRange_New` function was removed. It was never documented, never +* The :c:func:`!PyRange_New` function was removed. It was never documented, never used in the core code, and had dangerously lax error checking. In the unlikely case that your extensions were using it, you can replace it by something like the following:: diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 563fa461d2a86..b975aea7d6894 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -977,7 +977,7 @@ can be used to include Unicode characters:: print len(s) # 12 Unicode characters At the C level, Python 3.0 will rename the existing 8-bit -string type, called :c:type:`PyStringObject` in Python 2.x, +string type, called :c:type:`!PyStringObject` in Python 2.x, to :c:type:`PyBytesObject`. Python 2.6 uses ``#define`` to support using the names :c:func:`PyBytesObject`, :c:func:`PyBytes_Check`, :c:func:`PyBytes_FromStringAndSize`, @@ -3012,11 +3012,11 @@ Changes to Python's build process and to the C API include: bug occurred if one thread closed a file object while another thread was reading from or writing to the object. In 2.6 file objects have a reference count, manipulated by the - :c:func:`PyFile_IncUseCount` and :c:func:`PyFile_DecUseCount` + :c:func:`!PyFile_IncUseCount` and :c:func:`!PyFile_DecUseCount` functions. File objects can't be closed unless the reference count - is zero. :c:func:`PyFile_IncUseCount` should be called while the GIL + is zero. :c:func:`!PyFile_IncUseCount` should be called while the GIL is still held, before carrying out an I/O operation using the - ``FILE *`` pointer, and :c:func:`PyFile_DecUseCount` should be called + ``FILE *`` pointer, and :c:func:`!PyFile_DecUseCount` should be called immediately after the GIL is re-acquired. (Contributed by Antoine Pitrou and Gregory P. Smith.) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 76beb4d746a19..eda6c8be38ad0 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2152,7 +2152,7 @@ Changes to Python's build process and to the C API include: * New function: stemming from the rewrite of string-to-float conversion, a new :c:func:`PyOS_string_to_double` function was added. The old - :c:func:`PyOS_ascii_strtod` and :c:func:`PyOS_ascii_atof` functions + :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof` functions are now deprecated. * New function: :c:func:`PySys_SetArgvEx` sets the value of @@ -2195,13 +2195,13 @@ Changes to Python's build process and to the C API include: .. XXX these macros don't seem to be described in the c-api docs. -* Removed function: :c:macro:`PyEval_CallObject` is now only available +* Removed function: :c:func:`!PyEval_CallObject` is now only available as a macro. A function version was being kept around to preserve ABI linking compatibility, but that was in 1997; it can certainly be deleted by now. (Removed by Antoine Pitrou; :issue:`8276`.) -* New format codes: the :c:func:`PyFormat_FromString`, - :c:func:`PyFormat_FromStringV`, and :c:func:`PyErr_Format` functions now +* New format codes: the :c:func:`!PyString_FromFormat`, + :c:func:`!PyString_FromFormatV`, and :c:func:`PyErr_Format` functions now accept ``%lld`` and ``%llu`` format codes for displaying C's :c:expr:`long long` types. (Contributed by Mark Dickinson; :issue:`7228`.) @@ -2540,7 +2540,7 @@ For C extensions: instead of triggering a :exc:`DeprecationWarning` (:issue:`5080`). * Use the new :c:func:`PyOS_string_to_double` function instead of the old - :c:func:`PyOS_ascii_strtod` and :c:func:`PyOS_ascii_atof` functions, + :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof` functions, which are now deprecated. For applications that embed Python: diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 4f8b05efd4464..4bd53fb14ccb5 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -865,8 +865,8 @@ to the C API. * No more C API support for restricted execution. -* :c:func:`PyNumber_Coerce`, :c:func:`PyNumber_CoerceEx`, - :c:func:`PyMember_Get`, and :c:func:`PyMember_Set` C APIs are removed. +* :c:func:`!PyNumber_Coerce`, :c:func:`!PyNumber_CoerceEx`, + :c:func:`!PyMember_Get`, and :c:func:`!PyMember_Set` C APIs are removed. * New C API :c:func:`PyImport_ImportModuleNoBlock`, works like :c:func:`PyImport_ImportModule` but won't block on the import lock diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index ceef622f9f8da..cc6619c53cd62 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -501,12 +501,12 @@ Changes to Python's build process and to the C API include: (Contributed by Mark Dickinson and Lisandro Dalcrin; :issue:`5175`.) -* Deprecated :c:func:`PyNumber_Int`. Use :c:func:`PyNumber_Long` instead. +* Deprecated :c:func:`!PyNumber_Int`. Use :c:func:`PyNumber_Long` instead. (Contributed by Mark Dickinson; :issue:`4910`.) * Added a new :c:func:`PyOS_string_to_double` function to replace the - deprecated functions :c:func:`PyOS_ascii_strtod` and :c:func:`PyOS_ascii_atof`. + deprecated functions :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof`. (Contributed by Mark Dickinson; :issue:`5914`.) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 92d23462acc57..47a6f3e1c0839 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1819,8 +1819,8 @@ Removed into their code. (Contributed by Dong-hee Na and Terry J. Reedy in :issue:`42299`.) -* Removed the :c:func:`PyModule_GetWarningsModule` function that was useless - now due to the _warnings module was converted to a builtin module in 2.6. +* Removed the :c:func:`!PyModule_GetWarningsModule` function that was useless + now due to the :mod:`!_warnings` module was converted to a builtin module in 2.6. (Contributed by Hai Shi in :issue:`42599`.) * Remove deprecated aliases to :ref:`collections-abstract-base-classes` from diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 0ac8d02ab15c8..6112e807a1225 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2216,7 +2216,7 @@ New Features * :c:func:`PyBuffer_SizeFromFormat` * :c:func:`PyBuffer_ToContiguous` * :c:func:`PyBuffer_FromContiguous` - * :c:func:`PyBuffer_CopyData` + * :c:func:`PyObject_CopyData` * :c:func:`PyBuffer_IsContiguous` * :c:func:`PyBuffer_FillContiguousStrides` * :c:func:`PyBuffer_FillInfo` @@ -2562,18 +2562,18 @@ Deprecated * Deprecate the following functions to configure the Python initialization: - * :c:func:`PySys_AddWarnOptionUnicode` - * :c:func:`PySys_AddWarnOption` - * :c:func:`PySys_AddXOption` - * :c:func:`PySys_HasWarnOptions` - * :c:func:`PySys_SetArgvEx` - * :c:func:`PySys_SetArgv` - * :c:func:`PySys_SetPath` - * :c:func:`Py_SetPath` - * :c:func:`Py_SetProgramName` - * :c:func:`Py_SetPythonHome` - * :c:func:`Py_SetStandardStreamEncoding` - * :c:func:`_Py_SetProgramFullPath` + * :c:func:`!PySys_AddWarnOptionUnicode` + * :c:func:`!PySys_AddWarnOption` + * :c:func:`!PySys_AddXOption` + * :c:func:`!PySys_HasWarnOptions` + * :c:func:`!PySys_SetArgvEx` + * :c:func:`!PySys_SetArgv` + * :c:func:`!PySys_SetPath` + * :c:func:`!Py_SetPath` + * :c:func:`!Py_SetProgramName` + * :c:func:`!Py_SetPythonHome` + * :c:func:`!Py_SetStandardStreamEncoding` + * :c:func:`!_Py_SetProgramFullPath` Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization Configuration ` instead (:pep:`587`). diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index ec554aacb096c..ba1cb4ae00623 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -2569,7 +2569,7 @@ Changes to Python's build process and to the C API include: to set :data:`sys.argv` without also modifying :data:`sys.path` (:issue:`5753`). -* :c:macro:`PyEval_CallObject` is now only available in macro form. The +* :c:func:`!PyEval_CallObject` is now only available in macro form. The function declaration, which was kept for backwards compatibility reasons, is now removed -- the macro was introduced in 1997 (:issue:`8276`). @@ -2731,15 +2731,15 @@ require changes to your code: (Contributed by Antoine Pitrou, :issue:`10272`.) -* The misleading functions :c:func:`PyEval_AcquireLock()` and - :c:func:`PyEval_ReleaseLock()` have been officially deprecated. The - thread-state aware APIs (such as :c:func:`PyEval_SaveThread()` - and :c:func:`PyEval_RestoreThread()`) should be used instead. +* The misleading functions :c:func:`!PyEval_AcquireLock` and + :c:func:`!PyEval_ReleaseLock` have been officially deprecated. The + thread-state aware APIs (such as :c:func:`PyEval_SaveThread` + and :c:func:`PyEval_RestoreThread`) should be used instead. * Due to security risks, :func:`asyncore.handle_accept` has been deprecated, and a new function, :func:`asyncore.handle_accepted`, was added to replace it. (Contributed by Giampaolo Rodola in :issue:`6706`.) -* Due to the new :term:`GIL` implementation, :c:func:`PyEval_InitThreads()` - cannot be called before :c:func:`Py_Initialize()` anymore. +* Due to the new :term:`GIL` implementation, :c:func:`!PyEval_InitThreads` + cannot be called before :c:func:`Py_Initialize` anymore. diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 8cf7ad57475b2..8d2bd0a2ae8da 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -2304,7 +2304,7 @@ Functions and macros manipulating Py_UNICODE* strings: Encoders: -* :c:func:`!PyUnicode_Encode`: use :c:func:`PyUnicode_AsEncodedObject` +* :c:func:`!PyUnicode_Encode`: use :c:func:`!PyUnicode_AsEncodedObject` * :c:func:`!PyUnicode_EncodeUTF7` * :c:func:`!PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or :c:func:`PyUnicode_AsUTF8String` @@ -2462,7 +2462,7 @@ Porting C code -------------- * In the course of changes to the buffer API the undocumented - :c:member:`~Py_buffer.smalltable` member of the + :c:member:`!smalltable` member of the :c:type:`Py_buffer` structure has been removed and the layout of the :c:type:`PyMemoryViewObject` has changed. diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 002d829e78bb3..3c0d8d665c394 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -2512,7 +2512,7 @@ Changes in the Python API Changes in the C API -------------------- -* The undocumented :c:member:`~PyMemoryViewObject.format` member of the +* The undocumented :c:member:`!format` member of the (non-public) :c:type:`PyMemoryViewObject` structure has been removed. All extensions relying on the relevant parts in ``memoryobject.h`` must be rebuilt. @@ -2520,7 +2520,7 @@ Changes in the C API * The :c:type:`PyMemAllocator` structure was renamed to :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. -* Removed non-documented macro :c:macro:`PyObject_REPR` which leaked references. +* Removed non-documented macro :c:macro:`!PyObject_REPR()` which leaked references. Use format character ``%R`` in :c:func:`PyUnicode_FromFormat`-like functions to format the :func:`repr` of the object. (Contributed by Serhiy Storchaka in :issue:`22453`.) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 3397de5d3e8c4..329fe8cb410da 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -1574,12 +1574,12 @@ Build and C API Changes * :c:func:`Py_INCREF`, :c:func:`Py_DECREF` * :c:func:`Py_XINCREF`, :c:func:`Py_XDECREF` * :c:func:`PyObject_INIT`, :c:func:`PyObject_INIT_VAR` - * Private functions: :c:func:`_PyObject_GC_TRACK`, - :c:func:`_PyObject_GC_UNTRACK`, :c:func:`_Py_Dealloc` + * Private functions: :c:func:`!_PyObject_GC_TRACK`, + :c:func:`!_PyObject_GC_UNTRACK`, :c:func:`!_Py_Dealloc` (Contributed by Victor Stinner in :issue:`35059`.) -* The :c:func:`PyByteArray_Init` and :c:func:`PyByteArray_Fini` functions have +* The :c:func:`!PyByteArray_Init` and :c:func:`!PyByteArray_Fini` functions have been removed. They did nothing since Python 2.7.4 and Python 3.2.0, were excluded from the limited API (stable ABI), and were not documented. (Contributed by Victor Stinner in :issue:`35713`.) @@ -1628,7 +1628,7 @@ Build and C API Changes parameter for indicating the number of positional-only arguments. (Contributed by Pablo Galindo in :issue:`37221`.) -* :c:func:`Py_SetPath` now sets :data:`sys.executable` to the program full +* :c:func:`!Py_SetPath` now sets :data:`sys.executable` to the program full path (:c:func:`Py_GetProgramFullPath`) rather than to the program name (:c:func:`Py_GetProgramName`). (Contributed by Victor Stinner in :issue:`38234`.) @@ -1845,11 +1845,11 @@ Changes in Python behavior always use ``sys.platform.startswith('aix')``. (Contributed by M. Felt in :issue:`36588`.) -* :c:func:`PyEval_AcquireLock` and :c:func:`PyEval_AcquireThread` now +* :c:func:`!PyEval_AcquireLock` and :c:func:`!PyEval_AcquireThread` now terminate the current thread if called while the interpreter is finalizing, making them consistent with :c:func:`PyEval_RestoreThread`, :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`. If this - behavior is not desired, guard the call by checking :c:func:`_Py_IsFinalizing` + behavior is not desired, guard the call by checking :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing`. (Contributed by Joannah Nanjekye in :issue:`36475`.) @@ -2021,7 +2021,7 @@ Changes in the C API *cf_flags*. (Contributed by Guido van Rossum in :issue:`35766`.) -* The :c:func:`PyEval_ReInitThreads` function has been removed from the C API. +* The :c:func:`!PyEval_ReInitThreads` function has been removed from the C API. It should not be called explicitly: use :c:func:`PyOS_AfterFork_Child` instead. (Contributed by Victor Stinner in :issue:`36728`.) @@ -2121,7 +2121,7 @@ Changes in the C API (Contributed by Antoine Pitrou in :issue:`32388`.) -* The functions :c:func:`PyNode_AddChild` and :c:func:`PyParser_AddToken` now accept +* The functions :c:func:`!PyNode_AddChild` and :c:func:`!PyParser_AddToken` now accept two additional ``int`` arguments *end_lineno* and *end_col_offset*. * The :file:`libpython38.a` file to allow MinGW tools to link directly against diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 664c8d8a545db..6829970900930 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -870,9 +870,9 @@ Deprecated users can leverage the Abstract Syntax Tree (AST) generation and compilation stage, using the :mod:`ast` module. -* The Public C API functions :c:func:`PyParser_SimpleParseStringFlags`, - :c:func:`PyParser_SimpleParseStringFlagsFilename`, - :c:func:`PyParser_SimpleParseFileFlags` and :c:func:`PyNode_Compile` +* The Public C API functions :c:func:`!PyParser_SimpleParseStringFlags`, + :c:func:`!PyParser_SimpleParseStringFlagsFilename`, + :c:func:`!PyParser_SimpleParseFileFlags` and :c:func:`!PyNode_Compile` are deprecated and will be removed in Python 3.10 together with the old parser. * Using :data:`NotImplemented` in a boolean context has been deprecated, @@ -923,10 +923,10 @@ Deprecated (Contributed by Batuhan Taskaya in :issue:`39639` and :issue:`39969` and Serhiy Storchaka in :issue:`39988`.) -* The :c:func:`PyEval_InitThreads` and :c:func:`PyEval_ThreadsInitialized` +* The :c:func:`!PyEval_InitThreads` and :c:func:`!PyEval_ThreadsInitialized` functions are now deprecated and will be removed in Python 3.11. Calling - :c:func:`PyEval_InitThreads` now does nothing. The :term:`GIL` is initialized - by :c:func:`Py_Initialize()` since Python 3.7. + :c:func:`!PyEval_InitThreads` now does nothing. The :term:`GIL` is initialized + by :c:func:`Py_Initialize` since Python 3.7. (Contributed by Victor Stinner in :issue:`39877`.) * Passing ``None`` as the first argument to the :func:`shlex.split` function diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index a9ff21149a87d..d4d6be870f170 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -5036,15 +5036,15 @@ Limited API. Deprecate the following functions to configure the Python initialization: -* :c:func:`PySys_AddWarnOptionUnicode` -* :c:func:`PySys_AddWarnOption` -* :c:func:`PySys_AddXOption` -* :c:func:`PySys_HasWarnOptions` -* :c:func:`Py_SetPath` -* :c:func:`Py_SetProgramName` -* :c:func:`Py_SetPythonHome` -* :c:func:`Py_SetStandardStreamEncoding` -* :c:func:`_Py_SetProgramFullPath` +* :c:func:`!PySys_AddWarnOptionUnicode` +* :c:func:`!PySys_AddWarnOption` +* :c:func:`!PySys_AddXOption` +* :c:func:`!PySys_HasWarnOptions` +* :c:func:`!Py_SetPath` +* :c:func:`!Py_SetProgramName` +* :c:func:`!Py_SetPythonHome` +* :c:func:`!Py_SetStandardStreamEncoding` +* :c:func:`!_Py_SetProgramFullPath` Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization Configuration ` instead (:pep:`587`). diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index a3bf6ff3c54b4..c5a4e407e85c9 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -8765,7 +8765,7 @@ for relative path to files in current directory. .. nonce: fmehdG .. section: C API -The :c:func:`PyByteArray_Init` and :c:func:`PyByteArray_Fini` functions have +The :c:func:`!PyByteArray_Init` and :c:func:`!PyByteArray_Fini` functions have been removed. They did nothing since Python 2.7.4 and Python 3.2.0, were excluded from the limited API (stable ABI), and were not documented. @@ -8836,7 +8836,7 @@ Py_LIMITED_API. Patch by Arthur Neufeld. .. nonce: gFd85N .. section: C API -The :c:func:`_PyObject_GC_TRACK` and :c:func:`_PyObject_GC_UNTRACK` macros +The :c:func:`!_PyObject_GC_TRACK` and :c:func:`!_PyObject_GC_UNTRACK` macros have been removed from the public C API. .. diff --git a/Misc/NEWS.d/3.8.0a4.rst b/Misc/NEWS.d/3.8.0a4.rst index 524a05a7ae970..2ce60f39539e8 100644 --- a/Misc/NEWS.d/3.8.0a4.rst +++ b/Misc/NEWS.d/3.8.0a4.rst @@ -124,7 +124,7 @@ Galindo. .. nonce: CjRps3 .. section: Core and Builtins -:c:func:`PyEval_AcquireLock` and :c:func:`PyEval_AcquireThread` now +:c:func:`!PyEval_AcquireLock` and :c:func:`!PyEval_AcquireThread` now terminate the current thread if called while the interpreter is finalizing, making them consistent with :c:func:`PyEval_RestoreThread`, :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`. diff --git a/Misc/NEWS.d/3.8.0b1.rst b/Misc/NEWS.d/3.8.0b1.rst index 5285770de1318..f8f180bc731fd 100644 --- a/Misc/NEWS.d/3.8.0b1.rst +++ b/Misc/NEWS.d/3.8.0b1.rst @@ -2047,6 +2047,6 @@ unbound methods. These are objects supporting the optimization given by the .. nonce: FR-dMP .. section: C API -The :c:func:`PyEval_ReInitThreads` function has been removed from the C API. +The :c:func:`!PyEval_ReInitThreads` function has been removed from the C API. It should not be called explicitly: use :c:func:`PyOS_AfterFork_Child` instead. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index 85473da78bd43..367c194c1ea18 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -5535,7 +5535,7 @@ Tyler Kieft. .. nonce: d0bhEA .. section: C API -:c:func:`Py_SetPath` now sets :data:`sys.executable` to the program full +:c:func:`!Py_SetPath` now sets :data:`sys.executable` to the program full path (:c:func:`Py_GetProgramFullPath`) rather than to the program name (:c:func:`Py_GetProgramName`). @@ -5546,8 +5546,8 @@ path (:c:func:`Py_GetProgramFullPath`) rather than to the program name .. nonce: ZbquVK .. section: C API -Python ignored arguments passed to :c:func:`Py_SetPath`, -:c:func:`Py_SetPythonHome` and :c:func:`Py_SetProgramName`: fix Python +Python ignored arguments passed to :c:func:`!Py_SetPath`, +:c:func:`!Py_SetPythonHome` and :c:func:`!Py_SetProgramName`: fix Python initialization to use specified arguments. .. diff --git a/Misc/NEWS.d/3.9.0a5.rst b/Misc/NEWS.d/3.9.0a5.rst index 8a1219501e81b..19ad20ad3db04 100644 --- a/Misc/NEWS.d/3.9.0a5.rst +++ b/Misc/NEWS.d/3.9.0a5.rst @@ -1234,8 +1234,8 @@ method name in the SystemError "bad call flags" error message to ease debug. .. nonce: GOYtIm .. section: C API -Deprecated :c:func:`PyEval_InitThreads` and -:c:func:`PyEval_ThreadsInitialized`. Calling :c:func:`PyEval_InitThreads` +Deprecated :c:func:`!PyEval_InitThreads` and +:c:func:`!PyEval_ThreadsInitialized`. Calling :c:func:`!PyEval_InitThreads` now does nothing. .. From webhook-mailer at python.org Tue Aug 22 10:17:16 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 22 Aug 2023 14:17:16 -0000 Subject: [Python-checkins] [3.12] Docs: align the param spec of sqlite3.Connection methods with the implementation (GH-108285) (#108287) Message-ID: https://github.com/python/cpython/commit/68fd6db9817d8b0b083a8f9372da5feba2f24db1 commit: 68fd6db9817d8b0b083a8f9372da5feba2f24db1 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-22T16:17:03+02:00 summary: [3.12] Docs: align the param spec of sqlite3.Connection methods with the implementation (GH-108285) (#108287) Docs: align the param spec of sqlite3.Connection methods with the implementation (GH-108285) - no parameters of create_aggregate() are positional-only - all parameters of create_collation() are positional-only (cherry picked from commit 893215a4e7f59eabb8ccdf188c4b9b1de5bd8966) Co-authored-by: Erlend E. Aasland files: M Doc/library/sqlite3.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 23a22e637c0a9..93ca46811f1c0 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -755,7 +755,7 @@ Connection objects ('acbd18db4cc2f85cedef654fccc4a4d8',) - .. method:: create_aggregate(name, /, n_arg, aggregate_class) + .. method:: create_aggregate(name, n_arg, aggregate_class) Create or remove a user-defined SQL aggregate function. @@ -895,7 +895,7 @@ Connection objects [('a', 9), ('b', 12), ('c', 16), ('d', 12), ('e', 9)] - .. method:: create_collation(name, callable) + .. method:: create_collation(name, callable, /) Create a collation named *name* using the collating function *callable*. *callable* is passed two :class:`string ` arguments, From webhook-mailer at python.org Tue Aug 22 10:28:25 2023 From: webhook-mailer at python.org (vstinner) Date: Tue, 22 Aug 2023 14:28:25 -0000 Subject: [Python-checkins] Ignore _Py_write_noraise() result: cast to (void) (#108291) Message-ID: https://github.com/python/cpython/commit/6541fe4ad7b96ab96ee5c596b60814a93346dd27 commit: 6541fe4ad7b96ab96ee5c596b60814a93346dd27 branch: main author: Victor Stinner committer: vstinner date: 2023-08-22T14:28:20Z summary: Ignore _Py_write_noraise() result: cast to (void) (#108291) Code using _Py_write_noraise() usually cannot report. Ignore errors is the least surprising behavior for users. files: M Modules/faulthandler.c M Python/pylifecycle.c M Python/traceback.c M Python/tracemalloc.c diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 428b090193f09..d8cfc13a7dc6d 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -26,7 +26,7 @@ /* Allocate at maximum 100 MiB of the stack to raise the stack overflow */ #define STACK_OVERFLOW_MAX_SIZE (100 * 1024 * 1024) -#define PUTS(fd, str) _Py_write_noraise(fd, str, strlen(str)) +#define PUTS(fd, str) (void)_Py_write_noraise(fd, str, strlen(str)) // clang uses __attribute__((no_sanitize("undefined"))) @@ -576,7 +576,7 @@ faulthandler_thread(void *unused) /* Timeout => dump traceback */ assert(st == PY_LOCK_FAILURE); - _Py_write_noraise(thread.fd, thread.header, (int)thread.header_len); + (void)_Py_write_noraise(thread.fd, thread.header, (int)thread.header_len); errmsg = _Py_DumpTracebackThreads(thread.fd, thread.interp, NULL); ok = (errmsg == NULL); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 263ead39fa472..9837e0f0d52e6 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -57,7 +57,7 @@ # undef BYTE #endif -#define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) +#define PUTS(fd, str) (void)_Py_write_noraise(fd, str, (int)strlen(str)) #ifdef __cplusplus diff --git a/Python/traceback.c b/Python/traceback.c index bddb8763a2f9b..61ace38275355 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -25,7 +25,7 @@ #define OFF(x) offsetof(PyTracebackObject, x) -#define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) +#define PUTS(fd, str) (void)_Py_write_noraise(fd, str, (int)strlen(str)) #define MAX_STRING_LENGTH 500 #define MAX_FRAME_DEPTH 100 #define MAX_NTHREADS 100 @@ -1047,7 +1047,7 @@ _Py_DumpDecimal(int fd, size_t value) value /= 10; } while (value); - _Py_write_noraise(fd, ptr, end - ptr); + (void)_Py_write_noraise(fd, ptr, end - ptr); } /* Format an integer as hexadecimal with width digits into fd file descriptor. @@ -1072,7 +1072,7 @@ _Py_DumpHexadecimal(int fd, uintptr_t value, Py_ssize_t width) value >>= 4; } while ((end - ptr) < width || value); - _Py_write_noraise(fd, ptr, end - ptr); + (void)_Py_write_noraise(fd, ptr, end - ptr); } void @@ -1125,7 +1125,7 @@ _Py_DumpASCII(int fd, PyObject *text) } if (!need_escape) { // The string can be written with a single write() syscall - _Py_write_noraise(fd, str, size); + (void)_Py_write_noraise(fd, str, size); goto done; } } @@ -1135,7 +1135,7 @@ _Py_DumpASCII(int fd, PyObject *text) if (' ' <= ch && ch <= 126) { /* printable ASCII character */ char c = (char)ch; - _Py_write_noraise(fd, &c, 1); + (void)_Py_write_noraise(fd, &c, 1); } else if (ch <= 0xff) { PUTS(fd, "\\x"); diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index f8ad939dccacc..7d294ea5fe744 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -1247,7 +1247,7 @@ tracemalloc_get_traceback(unsigned int domain, uintptr_t ptr) } -#define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) +#define PUTS(fd, str) (void)_Py_write_noraise(fd, str, (int)strlen(str)) static void _PyMem_DumpFrame(int fd, frame_t * frame) From webhook-mailer at python.org Tue Aug 22 10:34:23 2023 From: webhook-mailer at python.org (ethanfurman) Date: Tue, 22 Aug 2023 14:34:23 -0000 Subject: [Python-checkins] gh-107700: [Enum] Document that `EnumType` was added in 3.11 (GH-108260) Message-ID: https://github.com/python/cpython/commit/e8ef0bdd8c613a722bf7965bf1da912882141a52 commit: e8ef0bdd8c613a722bf7965bf1da912882141a52 branch: main author: Nikita Sobolev committer: ethanfurman date: 2023-08-22T07:34:18-07:00 summary: gh-107700: [Enum] Document that `EnumType` was added in 3.11 (GH-108260) files: M Doc/library/enum.rst diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index e9c4f0e2c5f59..7653865f0b9b3 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -241,6 +241,10 @@ Data Types >>> list(reversed(Color)) [, , ] + .. versionadded:: 3.11 + + Before 3.11 ``enum`` used ``EnumMeta`` type, which is kept as an alias. + .. class:: Enum From webhook-mailer at python.org Tue Aug 22 10:35:19 2023 From: webhook-mailer at python.org (zooba) Date: Tue, 22 Aug 2023 14:35:19 -0000 Subject: [Python-checkins] [3.11] gh-106242: Make ntpath.realpath errors consistent with abspath when there are embedded nulls (GH-108248) Message-ID: https://github.com/python/cpython/commit/8927cf0200e56ebf74ed43f8c3fd724cd699ec22 commit: 8927cf0200e56ebf74ed43f8c3fd724cd699ec22 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: zooba date: 2023-08-22T15:35:16+01:00 summary: [3.11] gh-106242: Make ntpath.realpath errors consistent with abspath when there are embedded nulls (GH-108248) gh-106242: Make ntpath.realpath errors consistent with abspath when there are embedded nulls (GH-108248) --------- (cherry picked from commit de33b5c662ea8d35d81ed857c6a39e34ab94c510) Co-authored-by: Steve Dower Co-authored-by: Gregory P. Smith files: A Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst M Lib/ntpath.py M Lib/test/test_ntpath.py diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 0444b0f65d10b..0246419485da0 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -695,6 +695,14 @@ def realpath(path, *, strict=False): try: path = _getfinalpathname(path) initial_winerror = 0 + except ValueError as ex: + # gh-106242: Raised for embedded null characters + # In strict mode, we convert into an OSError. + # Non-strict mode returns the path as-is, since we've already + # made it absolute. + if strict: + raise OSError(str(ex)) from None + path = normpath(path) except OSError as ex: if strict: raise @@ -714,6 +722,10 @@ def realpath(path, *, strict=False): try: if _getfinalpathname(spath) == path: path = spath + except ValueError as ex: + # Unexpected, as an invalid path should not have gained a prefix + # at any point, but we ignore this error just in case. + pass except OSError as ex: # If the path does not exist and originally did not exist, then # strip the prefix anyway. diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 75e50d92ed1eb..88660fc05a1b1 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -332,6 +332,10 @@ def test_realpath_basic(self): raise OSError("No free drive letters available") self.assertEqual(ntpath.realpath(d), d) + # gh-106242: Embedded nulls and non-strict fallback to abspath + self.assertEqual(ABSTFN + "\0spam", + ntpath.realpath(os_helper.TESTFN + "\0spam", strict=False)) + @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_strict(self): @@ -342,6 +346,8 @@ def test_realpath_strict(self): self.addCleanup(os_helper.unlink, ABSTFN) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN, strict=True) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN + "2", strict=True) + # gh-106242: Embedded nulls should raise OSError (not ValueError) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "\0spam", strict=True) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') diff --git a/Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst b/Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst new file mode 100644 index 0000000000000..ffe42ec5dc3fa --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-08-22-00-36-57.gh-issue-106242.q24ITw.rst @@ -0,0 +1,4 @@ +Fixes :func:`~os.path.realpath` to behave consistently when passed a path +containing an embedded null character on Windows. In strict mode, it now +raises :exc:`OSError` instead of the unexpected :exc:`ValueError`, and in +non-strict mode will make the path absolute. From webhook-mailer at python.org Tue Aug 22 10:53:53 2023 From: webhook-mailer at python.org (vstinner) Date: Tue, 22 Aug 2023 14:53:53 -0000 Subject: [Python-checkins] gh-106016: Add Lib/test/test_module/ directory (#108293) Message-ID: https://github.com/python/cpython/commit/adfc118fdab66882599e01a84c22bd897055f3f1 commit: adfc118fdab66882599e01a84c22bd897055f3f1 branch: main author: Victor Stinner committer: vstinner date: 2023-08-22T16:53:49+02:00 summary: gh-106016: Add Lib/test/test_module/ directory (#108293) * Move Python scripts related to test_module to this new directory: good_getattr.py and bad_getattrX.py scripts. * Move Lib/test/test_module.py to Lib/test/test_module/__init__.py. files: A Lib/test/test_module/__init__.py A Lib/test/test_module/bad_getattr.py A Lib/test/test_module/bad_getattr2.py A Lib/test/test_module/bad_getattr3.py A Lib/test/test_module/good_getattr.py D Lib/test/bad_getattr.py D Lib/test/bad_getattr2.py D Lib/test/bad_getattr3.py D Lib/test/good_getattr.py D Lib/test/test_module.py M Makefile.pre.in diff --git a/Lib/test/test_module.py b/Lib/test/test_module/__init__.py similarity index 92% rename from Lib/test/test_module.py rename to Lib/test/test_module/__init__.py index c7eb92290e1b6..cfc4d9ccf1cc8 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module/__init__.py @@ -126,8 +126,8 @@ def test_weakref(self): self.assertIs(wr(), None) def test_module_getattr(self): - import test.good_getattr as gga - from test.good_getattr import test + import test.test_module.good_getattr as gga + from test.test_module.good_getattr import test self.assertEqual(test, "There is test") self.assertEqual(gga.x, 1) self.assertEqual(gga.y, 2) @@ -135,46 +135,46 @@ def test_module_getattr(self): "Deprecated, use whatever instead"): gga.yolo self.assertEqual(gga.whatever, "There is whatever") - del sys.modules['test.good_getattr'] + del sys.modules['test.test_module.good_getattr'] def test_module_getattr_errors(self): - import test.bad_getattr as bga - from test import bad_getattr2 + import test.test_module.bad_getattr as bga + from test.test_module import bad_getattr2 self.assertEqual(bga.x, 1) self.assertEqual(bad_getattr2.x, 1) with self.assertRaises(TypeError): bga.nope with self.assertRaises(TypeError): bad_getattr2.nope - del sys.modules['test.bad_getattr'] - if 'test.bad_getattr2' in sys.modules: - del sys.modules['test.bad_getattr2'] + del sys.modules['test.test_module.bad_getattr'] + if 'test.test_module.bad_getattr2' in sys.modules: + del sys.modules['test.test_module.bad_getattr2'] def test_module_dir(self): - import test.good_getattr as gga + import test.test_module.good_getattr as gga self.assertEqual(dir(gga), ['a', 'b', 'c']) - del sys.modules['test.good_getattr'] + del sys.modules['test.test_module.good_getattr'] def test_module_dir_errors(self): - import test.bad_getattr as bga - from test import bad_getattr2 + import test.test_module.bad_getattr as bga + from test.test_module import bad_getattr2 with self.assertRaises(TypeError): dir(bga) with self.assertRaises(TypeError): dir(bad_getattr2) - del sys.modules['test.bad_getattr'] - if 'test.bad_getattr2' in sys.modules: - del sys.modules['test.bad_getattr2'] + del sys.modules['test.test_module.bad_getattr'] + if 'test.test_module.bad_getattr2' in sys.modules: + del sys.modules['test.test_module.bad_getattr2'] def test_module_getattr_tricky(self): - from test import bad_getattr3 + from test.test_module import bad_getattr3 # these lookups should not crash with self.assertRaises(AttributeError): bad_getattr3.one with self.assertRaises(AttributeError): bad_getattr3.delgetattr - if 'test.bad_getattr3' in sys.modules: - del sys.modules['test.bad_getattr3'] + if 'test.test_module.bad_getattr3' in sys.modules: + del sys.modules['test.test_module.bad_getattr3'] def test_module_repr_minimal(self): # reprs when modules have no __file__, __name__, or __loader__ diff --git a/Lib/test/bad_getattr.py b/Lib/test/test_module/bad_getattr.py similarity index 100% rename from Lib/test/bad_getattr.py rename to Lib/test/test_module/bad_getattr.py diff --git a/Lib/test/bad_getattr2.py b/Lib/test/test_module/bad_getattr2.py similarity index 100% rename from Lib/test/bad_getattr2.py rename to Lib/test/test_module/bad_getattr2.py diff --git a/Lib/test/bad_getattr3.py b/Lib/test/test_module/bad_getattr3.py similarity index 100% rename from Lib/test/bad_getattr3.py rename to Lib/test/test_module/bad_getattr3.py diff --git a/Lib/test/good_getattr.py b/Lib/test/test_module/good_getattr.py similarity index 100% rename from Lib/test/good_getattr.py rename to Lib/test/test_module/good_getattr.py diff --git a/Makefile.pre.in b/Makefile.pre.in index 9be5c3b50eb9e..beccab45a235a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2212,6 +2212,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_importlib/resources/zipdata02 \ test/test_importlib/source \ test/test_json \ + test/test_module \ test/test_peg_generator \ test/test_sqlite3 \ test/test_tkinter \ From webhook-mailer at python.org Tue Aug 22 11:29:53 2023 From: webhook-mailer at python.org (gvanrossum) Date: Tue, 22 Aug 2023 15:29:53 -0000 Subject: [Python-checkins] gh-108253: Fix bug in func version cache (#108296) Message-ID: https://github.com/python/cpython/commit/b8f96b5eda5b376b05a9dbf046208388249e30a6 commit: b8f96b5eda5b376b05a9dbf046208388249e30a6 branch: main author: Guido van Rossum committer: gvanrossum date: 2023-08-22T08:29:49-07:00 summary: gh-108253: Fix bug in func version cache (#108296) When a function object changed its version, a stale pointer might remain in the cache. Zap these whenever `func_version` changes (even when set to 0). files: M Objects/funcobject.c diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 33191d23f1823..2820e47bdf2e6 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -131,7 +131,7 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr) op->func_annotations = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; - op->func_version = 0; + _PyFunction_SetVersion(op, 0); _PyObject_GC_TRACK(op); handle_func_event(PyFunction_EVENT_CREATE, op, NULL); return op; @@ -207,7 +207,7 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_annotations = NULL; op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; - op->func_version = 0; + _PyFunction_SetVersion(op, 0); _PyObject_GC_TRACK(op); handle_func_event(PyFunction_EVENT_CREATE, op, NULL); return (PyObject *)op; @@ -268,9 +268,17 @@ code objects have been created during the process's lifetime. void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (func->func_version != 0) { + PyFunctionObject **slot = + interp->func_state.func_version_cache + + (func->func_version % FUNC_VERSION_CACHE_SIZE); + if (*slot == func) { + *slot = NULL; + } + } func->func_version = version; if (version != 0) { - PyInterpreterState *interp = _PyInterpreterState_GET(); interp->func_state.func_version_cache[ version % FUNC_VERSION_CACHE_SIZE] = func; } @@ -370,7 +378,7 @@ PyFunction_SetDefaults(PyObject *op, PyObject *defaults) } handle_func_event(PyFunction_EVENT_MODIFY_DEFAULTS, (PyFunctionObject *) op, defaults); - ((PyFunctionObject *)op)->func_version = 0; + _PyFunction_SetVersion((PyFunctionObject *)op, 0); Py_XSETREF(((PyFunctionObject *)op)->func_defaults, defaults); return 0; } @@ -379,7 +387,7 @@ void PyFunction_SetVectorcall(PyFunctionObject *func, vectorcallfunc vectorcall) { assert(func != NULL); - func->func_version = 0; + _PyFunction_SetVersion(func, 0); func->vectorcall = vectorcall; } @@ -412,7 +420,7 @@ PyFunction_SetKwDefaults(PyObject *op, PyObject *defaults) } handle_func_event(PyFunction_EVENT_MODIFY_KWDEFAULTS, (PyFunctionObject *) op, defaults); - ((PyFunctionObject *)op)->func_version = 0; + _PyFunction_SetVersion((PyFunctionObject *)op, 0); Py_XSETREF(((PyFunctionObject *)op)->func_kwdefaults, defaults); return 0; } @@ -445,7 +453,7 @@ PyFunction_SetClosure(PyObject *op, PyObject *closure) Py_TYPE(closure)->tp_name); return -1; } - ((PyFunctionObject *)op)->func_version = 0; + _PyFunction_SetVersion((PyFunctionObject *)op, 0); Py_XSETREF(((PyFunctionObject *)op)->func_closure, closure); return 0; } @@ -507,7 +515,7 @@ PyFunction_SetAnnotations(PyObject *op, PyObject *annotations) "non-dict annotations"); return -1; } - ((PyFunctionObject *)op)->func_version = 0; + _PyFunction_SetVersion((PyFunctionObject *)op, 0); Py_XSETREF(((PyFunctionObject *)op)->func_annotations, annotations); return 0; } @@ -566,7 +574,7 @@ func_set_code(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored)) return -1; } handle_func_event(PyFunction_EVENT_MODIFY_CODE, op, value); - op->func_version = 0; + _PyFunction_SetVersion(op, 0); Py_XSETREF(op->func_code, Py_NewRef(value)); return 0; } @@ -646,7 +654,7 @@ func_set_defaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored } handle_func_event(PyFunction_EVENT_MODIFY_DEFAULTS, op, value); - op->func_version = 0; + _PyFunction_SetVersion(op, 0); Py_XSETREF(op->func_defaults, Py_XNewRef(value)); return 0; } @@ -687,7 +695,7 @@ func_set_kwdefaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignor } handle_func_event(PyFunction_EVENT_MODIFY_KWDEFAULTS, op, value); - op->func_version = 0; + _PyFunction_SetVersion(op, 0); Py_XSETREF(op->func_kwdefaults, Py_XNewRef(value)); return 0; } @@ -717,7 +725,7 @@ func_set_annotations(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(igno "__annotations__ must be set to a dict object"); return -1; } - op->func_version = 0; + _PyFunction_SetVersion(op, 0); Py_XSETREF(op->func_annotations, Py_XNewRef(value)); return 0; } @@ -881,7 +889,7 @@ func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals, static int func_clear(PyFunctionObject *op) { - op->func_version = 0; + _PyFunction_SetVersion(op, 0); Py_CLEAR(op->func_globals); Py_CLEAR(op->func_builtins); Py_CLEAR(op->func_module); @@ -917,15 +925,7 @@ func_dealloc(PyFunctionObject *op) if (op->func_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) op); } - if (op->func_version != 0) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyFunctionObject **slot = - interp->func_state.func_version_cache - + (op->func_version % FUNC_VERSION_CACHE_SIZE); - if (*slot == op) { - *slot = NULL; - } - } + _PyFunction_SetVersion(op, 0); (void)func_clear(op); // These aren't cleared by func_clear(). Py_DECREF(op->func_code); From webhook-mailer at python.org Tue Aug 22 11:42:04 2023 From: webhook-mailer at python.org (vstinner) Date: Tue, 22 Aug 2023 15:42:04 -0000 Subject: [Python-checkins] [3.11] gh-106016: Add Lib/test/test_module/ directory (GH-108293) (#108304) Message-ID: https://github.com/python/cpython/commit/5be32d88182f956f423a56c6250ce4020e8cd53c commit: 5be32d88182f956f423a56c6250ce4020e8cd53c branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: vstinner date: 2023-08-22T15:42:00Z summary: [3.11] gh-106016: Add Lib/test/test_module/ directory (GH-108293) (#108304) gh-106016: Add Lib/test/test_module/ directory (GH-108293) * Move Python scripts related to test_module to this new directory: good_getattr.py and bad_getattrX.py scripts. * Move Lib/test/test_module.py to Lib/test/test_module/__init__.py. (cherry picked from commit adfc118fdab66882599e01a84c22bd897055f3f1) Co-authored-by: Victor Stinner files: A Lib/test/test_module/__init__.py A Lib/test/test_module/bad_getattr.py A Lib/test/test_module/bad_getattr2.py A Lib/test/test_module/bad_getattr3.py A Lib/test/test_module/good_getattr.py D Lib/test/bad_getattr.py D Lib/test/bad_getattr2.py D Lib/test/bad_getattr3.py D Lib/test/good_getattr.py D Lib/test/test_module.py M Makefile.pre.in diff --git a/Lib/test/test_module.py b/Lib/test/test_module/__init__.py similarity index 92% rename from Lib/test/test_module.py rename to Lib/test/test_module/__init__.py index f72177dda3702..87d1a8e3d6c97 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module/__init__.py @@ -126,8 +126,8 @@ def test_weakref(self): self.assertIs(wr(), None) def test_module_getattr(self): - import test.good_getattr as gga - from test.good_getattr import test + import test.test_module.good_getattr as gga + from test.test_module.good_getattr import test self.assertEqual(test, "There is test") self.assertEqual(gga.x, 1) self.assertEqual(gga.y, 2) @@ -135,46 +135,46 @@ def test_module_getattr(self): "Deprecated, use whatever instead"): gga.yolo self.assertEqual(gga.whatever, "There is whatever") - del sys.modules['test.good_getattr'] + del sys.modules['test.test_module.good_getattr'] def test_module_getattr_errors(self): - import test.bad_getattr as bga - from test import bad_getattr2 + import test.test_module.bad_getattr as bga + from test.test_module import bad_getattr2 self.assertEqual(bga.x, 1) self.assertEqual(bad_getattr2.x, 1) with self.assertRaises(TypeError): bga.nope with self.assertRaises(TypeError): bad_getattr2.nope - del sys.modules['test.bad_getattr'] - if 'test.bad_getattr2' in sys.modules: - del sys.modules['test.bad_getattr2'] + del sys.modules['test.test_module.bad_getattr'] + if 'test.test_module.bad_getattr2' in sys.modules: + del sys.modules['test.test_module.bad_getattr2'] def test_module_dir(self): - import test.good_getattr as gga + import test.test_module.good_getattr as gga self.assertEqual(dir(gga), ['a', 'b', 'c']) - del sys.modules['test.good_getattr'] + del sys.modules['test.test_module.good_getattr'] def test_module_dir_errors(self): - import test.bad_getattr as bga - from test import bad_getattr2 + import test.test_module.bad_getattr as bga + from test.test_module import bad_getattr2 with self.assertRaises(TypeError): dir(bga) with self.assertRaises(TypeError): dir(bad_getattr2) - del sys.modules['test.bad_getattr'] - if 'test.bad_getattr2' in sys.modules: - del sys.modules['test.bad_getattr2'] + del sys.modules['test.test_module.bad_getattr'] + if 'test.test_module.bad_getattr2' in sys.modules: + del sys.modules['test.test_module.bad_getattr2'] def test_module_getattr_tricky(self): - from test import bad_getattr3 + from test.test_module import bad_getattr3 # these lookups should not crash with self.assertRaises(AttributeError): bad_getattr3.one with self.assertRaises(AttributeError): bad_getattr3.delgetattr - if 'test.bad_getattr3' in sys.modules: - del sys.modules['test.bad_getattr3'] + if 'test.test_module.bad_getattr3' in sys.modules: + del sys.modules['test.test_module.bad_getattr3'] def test_module_repr_minimal(self): # reprs when modules have no __file__, __name__, or __loader__ diff --git a/Lib/test/bad_getattr.py b/Lib/test/test_module/bad_getattr.py similarity index 100% rename from Lib/test/bad_getattr.py rename to Lib/test/test_module/bad_getattr.py diff --git a/Lib/test/bad_getattr2.py b/Lib/test/test_module/bad_getattr2.py similarity index 100% rename from Lib/test/bad_getattr2.py rename to Lib/test/test_module/bad_getattr2.py diff --git a/Lib/test/bad_getattr3.py b/Lib/test/test_module/bad_getattr3.py similarity index 100% rename from Lib/test/bad_getattr3.py rename to Lib/test/test_module/bad_getattr3.py diff --git a/Lib/test/good_getattr.py b/Lib/test/test_module/good_getattr.py similarity index 100% rename from Lib/test/good_getattr.py rename to Lib/test/test_module/good_getattr.py diff --git a/Makefile.pre.in b/Makefile.pre.in index 2a407a7df697e..6405d06afe14e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2012,6 +2012,7 @@ TESTSUBDIRS= ctypes/test \ test/test_importlib/zipdata01 \ test/test_importlib/zipdata02 \ test/test_json \ + test/test_module \ test/test_peg_generator \ test/test_sqlite3 \ test/test_tomllib \ From webhook-mailer at python.org Tue Aug 22 13:02:34 2023 From: webhook-mailer at python.org (pablogsal) Date: Tue, 22 Aug 2023 17:02:34 -0000 Subject: [Python-checkins] [3.10] gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) (#108210) Message-ID: https://github.com/python/cpython/commit/7d445511f8b6eb32cbb295e87302e28ffec67454 commit: 7d445511f8b6eb32cbb295e87302e28ffec67454 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: pablogsal date: 2023-08-22T18:02:30+01:00 summary: [3.10] gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) (#108210) files: A Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst M Doc/library/tarfile.rst M Lib/tarfile.py M Lib/test/test_tarfile.py diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index bcec9ca3506f2..92e5c9dff52a3 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -732,6 +732,11 @@ A ``TarInfo`` object has the following public data attributes: Name of the target file name, which is only present in :class:`TarInfo` objects of type :const:`LNKTYPE` and :const:`SYMTYPE`. + For symbolic links (``SYMTYPE``), the *linkname* is relative to the directory + that contains the link. + For hard links (``LNKTYPE``), the *linkname* is relative to the root of + the archive. + .. attribute:: TarInfo.uid :type: int diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 40599f27bce9e..495349f08f9e7 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -741,7 +741,7 @@ def __init__(self, tarinfo): class AbsoluteLinkError(FilterError): def __init__(self, tarinfo): self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a symlink to an absolute path') + super().__init__(f'{tarinfo.name!r} is a link to an absolute path') class LinkOutsideDestinationError(FilterError): def __init__(self, tarinfo, path): @@ -801,7 +801,14 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) - target_path = os.path.realpath(os.path.join(dest_path, member.linkname)) + if member.issym(): + target_path = os.path.join(dest_path, + os.path.dirname(name), + member.linkname) + else: + target_path = os.path.join(dest_path, + member.linkname) + target_path = os.path.realpath(target_path) if os.path.commonpath([target_path, dest_path]) != dest_path: raise LinkOutsideDestinationError(member, target_path) return new_attrs diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 4bc32b71d15da..cfc13bccb2040 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3209,10 +3209,12 @@ def __exit__(self, *exc): self.bio = None def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, - mode=None, **kwargs): + mode=None, size=None, **kwargs): """Add a member to the test archive. Call within `with`.""" name = str(name) tarinfo = tarfile.TarInfo(name).replace(**kwargs) + if size is not None: + tarinfo.size = size if mode: tarinfo.mode = _filemode_to_int(mode) if symlink_to is not None: @@ -3276,7 +3278,8 @@ def check_context(self, tar, filter): raise self.raised_exception self.assertEqual(self.expected_paths, set()) - def expect_file(self, name, type=None, symlink_to=None, mode=None): + def expect_file(self, name, type=None, symlink_to=None, mode=None, + size=None): """Check a single file. See check_context.""" if self.raised_exception: raise self.raised_exception @@ -3310,6 +3313,8 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None): self.assertTrue(path.is_fifo()) else: raise NotImplementedError(type) + if size is not None: + self.assertEqual(path.stat().st_size, size) for parent in path.parents: self.expected_paths.discard(parent) @@ -3355,8 +3360,15 @@ def test_parent_symlink(self): # Test interplaying symlinks # Inspired by 'dirsymlink2a' in jwilk/traversal-archives with ArchiveMaker() as arc: + + # `current` links to `.` which is both: + # - the destination directory + # - `current` itself arc.add('current', symlink_to='.') + + # effectively points to ./../ arc.add('parent', symlink_to='current/..') + arc.add('parent/evil') if os_helper.can_symlink(): @@ -3397,9 +3409,46 @@ def test_parent_symlink(self): def test_parent_symlink2(self): # Test interplaying symlinks # Inspired by 'dirsymlink2b' in jwilk/traversal-archives + + # Posix and Windows have different pathname resolution: + # either symlink or a '..' component resolve first. + # Let's see which we are on. + if os_helper.can_symlink(): + testpath = os.path.join(TEMPDIR, 'resolution_test') + os.mkdir(testpath) + + # testpath/current links to `.` which is all of: + # - `testpath` + # - `testpath/current` + # - `testpath/current/current` + # - etc. + os.symlink('.', os.path.join(testpath, 'current')) + + # we'll test where `testpath/current/../file` ends up + with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): + pass + + if os.path.exists(os.path.join(testpath, 'file')): + # Windows collapses 'current\..' to '.' first, leaving + # 'testpath\file' + dotdot_resolves_early = True + elif os.path.exists(os.path.join(testpath, '..', 'file')): + # Posix resolves 'current' to '.' first, leaving + # 'testpath/../file' + dotdot_resolves_early = False + else: + raise AssertionError('Could not determine link resolution') + with ArchiveMaker() as arc: + + # `current` links to `.` which is both the destination directory + # and `current` itself arc.add('current', symlink_to='.') + + # `current/parent` is also available as `./parent`, + # and effectively points to `./../` arc.add('current/parent', symlink_to='..') + arc.add('parent/evil') with self.check_context(arc.open(), 'fully_trusted'): @@ -3413,6 +3462,7 @@ def test_parent_symlink2(self): with self.check_context(arc.open(), 'tar'): if os_helper.can_symlink(): + # Fail when extracting a file outside destination self.expect_exception( tarfile.OutsideDestinationError, "'parent/evil' would be extracted to " @@ -3423,10 +3473,24 @@ def test_parent_symlink2(self): self.expect_file('parent/evil') with self.check_context(arc.open(), 'data'): - self.expect_exception( - tarfile.LinkOutsideDestinationError, - """'current/parent' would link to ['"].*['"], """ - + "which is outside the destination") + if os_helper.can_symlink(): + if dotdot_resolves_early: + # Fail when extracting a file outside destination + self.expect_exception( + tarfile.OutsideDestinationError, + "'parent/evil' would be extracted to " + + """['"].*evil['"], which is outside """ + + "the destination") + else: + # Fail as soon as we have a symlink outside the destination + self.expect_exception( + tarfile.LinkOutsideDestinationError, + "'current/parent' would link to " + + """['"].*outerdir['"], which is outside """ + + "the destination") + else: + self.expect_file('current/') + self.expect_file('parent/evil') def test_absolute_symlink(self): # Test symlink to an absolute path @@ -3455,11 +3519,29 @@ def test_absolute_symlink(self): with self.check_context(arc.open(), 'data'): self.expect_exception( tarfile.AbsoluteLinkError, - "'parent' is a symlink to an absolute path") + "'parent' is a link to an absolute path") + + def test_absolute_hardlink(self): + # Test hardlink to an absolute path + # Inspired by 'dirsymlink' in https://github.com/jwilk/traversal-archives + with ArchiveMaker() as arc: + arc.add('parent', hardlink_to=self.outerdir / 'foo') + + with self.check_context(arc.open(), 'fully_trusted'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'tar'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.AbsoluteLinkError, + "'parent' is a link to an absolute path") def test_sly_relative0(self): # Inspired by 'relative0' in jwilk/traversal-archives with ArchiveMaker() as arc: + # points to `../../tmp/moo` arc.add('../moo', symlink_to='..//tmp/moo') try: @@ -3509,6 +3591,54 @@ def test_sly_relative2(self): + """['"].*moo['"], which is outside the """ + "destination") + def test_deep_symlink(self): + # Test that symlinks and hardlinks inside a directory + # point to the correct file (`target` of size 3). + # If links aren't supported we get a copy of the file. + with ArchiveMaker() as arc: + arc.add('targetdir/target', size=3) + # a hardlink's linkname is relative to the archive + arc.add('linkdir/hardlink', hardlink_to=os.path.join( + 'targetdir', 'target')) + # a symlink's linkname is relative to the link's directory + arc.add('linkdir/symlink', symlink_to=os.path.join( + '..', 'targetdir', 'target')) + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + if os_helper.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='../targetdir/target') + else: + self.expect_file('linkdir/symlink', size=3) + + def test_chains(self): + # Test chaining of symlinks/hardlinks. + # Symlinks are created before the files they point to. + with ArchiveMaker() as arc: + arc.add('linkdir/symlink', symlink_to='hardlink') + arc.add('symlink2', symlink_to=os.path.join( + 'linkdir', 'hardlink2')) + arc.add('targetdir/target', size=3) + arc.add('linkdir/hardlink', hardlink_to='targetdir/target') + arc.add('linkdir/hardlink2', hardlink_to='linkdir/symlink') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + self.expect_file('linkdir/hardlink2', size=3) + if os_helper.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='hardlink') + self.expect_file('symlink2', size=3, + symlink_to='linkdir/hardlink2') + else: + self.expect_file('linkdir/symlink', size=3) + self.expect_file('symlink2', size=3) + def test_modes(self): # Test how file modes are extracted # (Note that the modes are ignored on platforms without working chmod) diff --git a/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst new file mode 100644 index 0000000000000..32c1fb93f4ab2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst @@ -0,0 +1,3 @@ +:func:`tarfile.data_filter` now takes the location of symlinks into account +when determining their target, so it will no longer reject some valid +tarballs with ``LinkOutsideDestinationError``. From webhook-mailer at python.org Tue Aug 22 13:12:55 2023 From: webhook-mailer at python.org (vstinner) Date: Tue, 22 Aug 2023 17:12:55 -0000 Subject: [Python-checkins] gh-108303: Remove unused Lib/test/sgml_input.html (#108305) Message-ID: https://github.com/python/cpython/commit/d2879f2095abd5c8186c7f69c964a341c2053572 commit: d2879f2095abd5c8186c7f69c964a341c2053572 branch: main author: Victor Stinner committer: vstinner date: 2023-08-22T19:12:51+02:00 summary: gh-108303: Remove unused Lib/test/sgml_input.html (#108305) In Python 2.7, the file was used by Lib/test/test_sgmllib.py to test Lib/sgmllib.py. The sgmllib module and its tests have been removed in Python 3.0. files: D Lib/test/sgml_input.html diff --git a/Lib/test/sgml_input.html b/Lib/test/sgml_input.html deleted file mode 100644 index f4d2e6cc805e6..0000000000000 --- a/Lib/test/sgml_input.html +++ /dev/null @@ -1,212 +0,0 @@ - - - - - - - - -
- - - - - -
-
- - - - - -
- - -
-
-
- 		- - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - - -
  MetalCristalDeuterioEnerg?a  
160.6363.40639.230-80/3.965
-
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Flotas (max. 9)
Num.Misi?nCantidadComienzoSalidaObjetivoLlegadaOrden
1 - Espionaje - (F) - 3[2:250:6]Wed Aug 9 18:00:02[2:242:5]Wed Aug 9 18:01:02 -
- - -
-
2 - Espionaje - (V) - 3[2:250:6]Wed Aug 9 17:59:55[2:242:1]Wed Aug 9 18:01:55 -
- - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Nueva misi?n: elegir naves
NavesDisponibles--
Nave peque?a de carga10m?x
Nave grande de carga19m?x
Crucero6m?x
Reciclador1m?x
Sonda de espionaje139m?x
Ninguna naveTodas las naves
- -

-
- - From webhook-mailer at python.org Tue Aug 22 13:53:19 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 17:53:19 -0000 Subject: [Python-checkins] gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw (#108315) Message-ID: https://github.com/python/cpython/commit/0cb0c238d520a8718e313b52cffc356a5a7561bf commit: 0cb0c238d520a8718e313b52cffc356a5a7561bf branch: main author: ?ukasz Langa committer: ambv date: 2023-08-22T19:53:15+02:00 summary: gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw (#108315) Instances of `ssl.SSLSocket` were vulnerable to a bypass of the TLS handshake and included protections (like certificate verification) and treating sent unencrypted data as if it were post-handshake TLS encrypted data. The vulnerability is caused when a socket is connected, data is sent by the malicious peer and stored in a buffer, and then the malicious peer closes the socket within a small timing window before the other peers? TLS handshake can begin. After this sequence of events the closed socket will not immediately attempt a TLS handshake due to not being connected but will also allow the buffered data to be read as if a successful TLS handshake had occurred. Co-authored-by: Gregory P. Smith [Google LLC] files: A Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst M Lib/ssl.py M Lib/test/test_ssl.py diff --git a/Lib/ssl.py b/Lib/ssl.py index 1d5873726441e..ff363c75e7dfd 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -975,7 +975,7 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, ) self = cls.__new__(cls, **kwargs) super(SSLSocket, self).__init__(**kwargs) - self.settimeout(sock.gettimeout()) + sock_timeout = sock.gettimeout() sock.detach() self._context = context @@ -994,9 +994,38 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, if e.errno != errno.ENOTCONN: raise connected = False + blocking = self.getblocking() + self.setblocking(False) + try: + # We are not connected so this is not supposed to block, but + # testing revealed otherwise on macOS and Windows so we do + # the non-blocking dance regardless. Our raise when any data + # is found means consuming the data is harmless. + notconn_pre_handshake_data = self.recv(1) + except OSError as e: + # EINVAL occurs for recv(1) on non-connected on unix sockets. + if e.errno not in (errno.ENOTCONN, errno.EINVAL): + raise + notconn_pre_handshake_data = b'' + self.setblocking(blocking) + if notconn_pre_handshake_data: + # This prevents pending data sent to the socket before it was + # closed from escaping to the caller who could otherwise + # presume it came through a successful TLS connection. + reason = "Closed before TLS handshake with data in recv buffer." + notconn_pre_handshake_data_error = SSLError(e.errno, reason) + # Add the SSLError attributes that _ssl.c always adds. + notconn_pre_handshake_data_error.reason = reason + notconn_pre_handshake_data_error.library = None + try: + self.close() + except OSError: + pass + raise notconn_pre_handshake_data_error else: connected = True + self.settimeout(sock_timeout) # Must come after setblocking() calls. self._connected = connected if connected: # create the SSL object diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 6117ca3fdba1b..ad5377ec059c0 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -10,11 +10,14 @@ from test.support import threading_helper from test.support import warnings_helper from test.support import asyncore +import re import socket import select +import struct import time import enum import gc +import http.client import os import errno import pprint @@ -4659,6 +4662,214 @@ def sni_cb(sock, servername, ctx): s.connect((HOST, server.port)) +def set_socket_so_linger_on_with_zero_timeout(sock): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) + + +class TestPreHandshakeClose(unittest.TestCase): + """Verify behavior of close sockets with received data before to the handshake. + """ + + class SingleConnectionTestServerThread(threading.Thread): + + def __init__(self, *, name, call_after_accept): + self.call_after_accept = call_after_accept + self.received_data = b'' # set by .run() + self.wrap_error = None # set by .run() + self.listener = None # set by .start() + self.port = None # set by .start() + super().__init__(name=name) + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + try: + if self.listener: + self.listener.close() + except OSError: + pass + self.join() + self.wrap_error = None # avoid dangling references + + def start(self): + self.ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED + self.ssl_ctx.load_verify_locations(cafile=ONLYCERT) + self.ssl_ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + self.listener = socket.socket() + self.port = socket_helper.bind_port(self.listener) + self.listener.settimeout(2.0) + self.listener.listen(1) + super().start() + + def run(self): + conn, address = self.listener.accept() + self.listener.close() + with conn: + if self.call_after_accept(conn): + return + try: + tls_socket = self.ssl_ctx.wrap_socket(conn, server_side=True) + except OSError as err: # ssl.SSLError inherits from OSError + self.wrap_error = err + else: + try: + self.received_data = tls_socket.recv(400) + except OSError: + pass # closed, protocol error, etc. + + def non_linux_skip_if_other_okay_error(self, err): + if sys.platform == "linux": + return # Expect the full test setup to always work on Linux. + if (isinstance(err, ConnectionResetError) or + (isinstance(err, OSError) and err.errno == errno.EINVAL) or + re.search('wrong.version.number', getattr(err, "reason", ""), re.I)): + # On Windows the TCP RST leads to a ConnectionResetError + # (ECONNRESET) which Linux doesn't appear to surface to userspace. + # If wrap_socket() winds up on the "if connected:" path and doing + # the actual wrapping... we get an SSLError from OpenSSL. Typically + # WRONG_VERSION_NUMBER. While appropriate, neither is the scenario + # we're specifically trying to test. The way this test is written + # is known to work on Linux. We'll skip it anywhere else that it + # does not present as doing so. + self.skipTest(f"Could not recreate conditions on {sys.platform}:" + f" {err=}") + # If maintaining this conditional winds up being a problem. + # just turn this into an unconditional skip anything but Linux. + # The important thing is that our CI has the logic covered. + + def test_preauth_data_to_tls_server(self): + server_accept_called = threading.Event() + ready_for_server_wrap_socket = threading.Event() + + def call_after_accept(unused): + server_accept_called.set() + if not ready_for_server_wrap_socket.wait(2.0): + raise RuntimeError("wrap_socket event never set, test may fail.") + return False # Tell the server thread to continue. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_server") + self.enterContext(server) # starts it & unittest.TestCase stops it. + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(client) + client.setblocking(False) + + server_accept_called.wait() + client.send(b"DELETE /data HTTP/1.0\r\n\r\n") + client.close() # RST + + ready_for_server_wrap_socket.set() + server.join() + wrap_error = server.wrap_error + self.assertEqual(b"", server.received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + + def test_preauth_data_to_tls_client(self): + client_can_continue_with_wrap_socket = threading.Event() + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 307 Temporary Redirect\r\n" + b"Location: https://example.com/someone-elses-server\r\n" + b"\r\n") + conn_to_client.close() # RST + client_can_continue_with_wrap_socket.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_client") + self.enterContext(server) # starts it & unittest.TestCase stops it. + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + if not client_can_continue_with_wrap_socket.wait(2.0): + self.fail("test server took too long.") + ssl_ctx = ssl.create_default_context() + try: + tls_client = ssl_ctx.wrap_socket( + client, server_hostname="localhost") + except OSError as err: # SSLError inherits from OSError + wrap_error = err + received_data = b"" + else: + wrap_error = None + received_data = tls_client.recv(400) + tls_client.close() + + server.join() + self.assertEqual(b"", received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + + def test_https_client_non_tls_response_ignored(self): + + server_responding = threading.Event() + + class SynchronizedHTTPSConnection(http.client.HTTPSConnection): + def connect(self): + http.client.HTTPConnection.connect(self) + # Wait for our fault injection server to have done its thing. + if not server_responding.wait(1.0) and support.verbose: + sys.stdout.write("server_responding event never set.") + self.sock = self._context.wrap_socket( + self.sock, server_hostname=self.host) + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 402 Payment Required\r\n" + b"\r\n") + conn_to_client.close() # RST + server_responding.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="non_tls_http_RST_responder") + self.enterContext(server) # starts it & unittest.TestCase stops it. + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + connection = SynchronizedHTTPSConnection( + f"localhost", + port=server.port, + context=ssl.create_default_context(), + timeout=2.0, + ) + # There are lots of reasons this raises as desired, long before this + # test was added. Sending the request requires a successful TLS wrapped + # socket; that fails if the connection is broken. It may seem pointless + # to test this. It serves as an illustration of something that we never + # want to happen... properly not happening. + with self.assertRaises(OSError) as err_ctx: + connection.request("HEAD", "/test", headers={"Host": "localhost"}) + response = connection.getresponse() + + class TestEnumerations(unittest.TestCase): def test_tlsversion(self): diff --git a/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst new file mode 100644 index 0000000000000..403c77a9d480e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst @@ -0,0 +1,7 @@ +Fixed an issue where instances of :class:`ssl.SSLSocket` were vulnerable to +a bypass of the TLS handshake and included protections (like certificate +verification) and treating sent unencrypted data as if it were +post-handshake TLS encrypted data. Security issue reported as +`CVE-2023-40217 +`_ by +Aapo Oksman. Patch by Gregory P. Smith. From webhook-mailer at python.org Tue Aug 22 13:53:23 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 17:53:23 -0000 Subject: [Python-checkins] [3.11] gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw (#108317) Message-ID: https://github.com/python/cpython/commit/75a875e0df0530b75b1470d797942f90f4a718d3 commit: 75a875e0df0530b75b1470d797942f90f4a718d3 branch: 3.11 author: ?ukasz Langa committer: ambv date: 2023-08-22T19:53:19+02:00 summary: [3.11] gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw (#108317) gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw Instances of `ssl.SSLSocket` were vulnerable to a bypass of the TLS handshake and included protections (like certificate verification) and treating sent unencrypted data as if it were post-handshake TLS encrypted data. The vulnerability is caused when a socket is connected, data is sent by the malicious peer and stored in a buffer, and then the malicious peer closes the socket within a small timing window before the other peers? TLS handshake can begin. After this sequence of events the closed socket will not immediately attempt a TLS handshake due to not being connected but will also allow the buffered data to be read as if a successful TLS handshake had occurred. Co-authored-by: Gregory P. Smith [Google LLC] files: A Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst M Lib/ssl.py M Lib/test/test_ssl.py diff --git a/Lib/ssl.py b/Lib/ssl.py index ebac1d60d52de..ced87d406995c 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1037,7 +1037,7 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, ) self = cls.__new__(cls, **kwargs) super(SSLSocket, self).__init__(**kwargs) - self.settimeout(sock.gettimeout()) + sock_timeout = sock.gettimeout() sock.detach() self._context = context @@ -1056,9 +1056,38 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, if e.errno != errno.ENOTCONN: raise connected = False + blocking = self.getblocking() + self.setblocking(False) + try: + # We are not connected so this is not supposed to block, but + # testing revealed otherwise on macOS and Windows so we do + # the non-blocking dance regardless. Our raise when any data + # is found means consuming the data is harmless. + notconn_pre_handshake_data = self.recv(1) + except OSError as e: + # EINVAL occurs for recv(1) on non-connected on unix sockets. + if e.errno not in (errno.ENOTCONN, errno.EINVAL): + raise + notconn_pre_handshake_data = b'' + self.setblocking(blocking) + if notconn_pre_handshake_data: + # This prevents pending data sent to the socket before it was + # closed from escaping to the caller who could otherwise + # presume it came through a successful TLS connection. + reason = "Closed before TLS handshake with data in recv buffer." + notconn_pre_handshake_data_error = SSLError(e.errno, reason) + # Add the SSLError attributes that _ssl.c always adds. + notconn_pre_handshake_data_error.reason = reason + notconn_pre_handshake_data_error.library = None + try: + self.close() + except OSError: + pass + raise notconn_pre_handshake_data_error else: connected = True + self.settimeout(sock_timeout) # Must come after setblocking() calls. self._connected = connected if connected: # create the SSL object diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 85a561f6b7491..4a2508988c0f7 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -9,11 +9,14 @@ from test.support import socket_helper from test.support import threading_helper from test.support import warnings_helper +import re import socket import select +import struct import time import enum import gc +import http.client import os import errno import pprint @@ -4896,6 +4899,214 @@ def sni_cb(sock, servername, ctx): s.connect((HOST, server.port)) +def set_socket_so_linger_on_with_zero_timeout(sock): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) + + +class TestPreHandshakeClose(unittest.TestCase): + """Verify behavior of close sockets with received data before to the handshake. + """ + + class SingleConnectionTestServerThread(threading.Thread): + + def __init__(self, *, name, call_after_accept): + self.call_after_accept = call_after_accept + self.received_data = b'' # set by .run() + self.wrap_error = None # set by .run() + self.listener = None # set by .start() + self.port = None # set by .start() + super().__init__(name=name) + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + try: + if self.listener: + self.listener.close() + except OSError: + pass + self.join() + self.wrap_error = None # avoid dangling references + + def start(self): + self.ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED + self.ssl_ctx.load_verify_locations(cafile=ONLYCERT) + self.ssl_ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + self.listener = socket.socket() + self.port = socket_helper.bind_port(self.listener) + self.listener.settimeout(2.0) + self.listener.listen(1) + super().start() + + def run(self): + conn, address = self.listener.accept() + self.listener.close() + with conn: + if self.call_after_accept(conn): + return + try: + tls_socket = self.ssl_ctx.wrap_socket(conn, server_side=True) + except OSError as err: # ssl.SSLError inherits from OSError + self.wrap_error = err + else: + try: + self.received_data = tls_socket.recv(400) + except OSError: + pass # closed, protocol error, etc. + + def non_linux_skip_if_other_okay_error(self, err): + if sys.platform == "linux": + return # Expect the full test setup to always work on Linux. + if (isinstance(err, ConnectionResetError) or + (isinstance(err, OSError) and err.errno == errno.EINVAL) or + re.search('wrong.version.number', getattr(err, "reason", ""), re.I)): + # On Windows the TCP RST leads to a ConnectionResetError + # (ECONNRESET) which Linux doesn't appear to surface to userspace. + # If wrap_socket() winds up on the "if connected:" path and doing + # the actual wrapping... we get an SSLError from OpenSSL. Typically + # WRONG_VERSION_NUMBER. While appropriate, neither is the scenario + # we're specifically trying to test. The way this test is written + # is known to work on Linux. We'll skip it anywhere else that it + # does not present as doing so. + self.skipTest(f"Could not recreate conditions on {sys.platform}:" + f" {err=}") + # If maintaining this conditional winds up being a problem. + # just turn this into an unconditional skip anything but Linux. + # The important thing is that our CI has the logic covered. + + def test_preauth_data_to_tls_server(self): + server_accept_called = threading.Event() + ready_for_server_wrap_socket = threading.Event() + + def call_after_accept(unused): + server_accept_called.set() + if not ready_for_server_wrap_socket.wait(2.0): + raise RuntimeError("wrap_socket event never set, test may fail.") + return False # Tell the server thread to continue. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_server") + self.enterContext(server) # starts it & unittest.TestCase stops it. + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(client) + client.setblocking(False) + + server_accept_called.wait() + client.send(b"DELETE /data HTTP/1.0\r\n\r\n") + client.close() # RST + + ready_for_server_wrap_socket.set() + server.join() + wrap_error = server.wrap_error + self.assertEqual(b"", server.received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + + def test_preauth_data_to_tls_client(self): + client_can_continue_with_wrap_socket = threading.Event() + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 307 Temporary Redirect\r\n" + b"Location: https://example.com/someone-elses-server\r\n" + b"\r\n") + conn_to_client.close() # RST + client_can_continue_with_wrap_socket.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_client") + self.enterContext(server) # starts it & unittest.TestCase stops it. + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + if not client_can_continue_with_wrap_socket.wait(2.0): + self.fail("test server took too long.") + ssl_ctx = ssl.create_default_context() + try: + tls_client = ssl_ctx.wrap_socket( + client, server_hostname="localhost") + except OSError as err: # SSLError inherits from OSError + wrap_error = err + received_data = b"" + else: + wrap_error = None + received_data = tls_client.recv(400) + tls_client.close() + + server.join() + self.assertEqual(b"", received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + + def test_https_client_non_tls_response_ignored(self): + + server_responding = threading.Event() + + class SynchronizedHTTPSConnection(http.client.HTTPSConnection): + def connect(self): + http.client.HTTPConnection.connect(self) + # Wait for our fault injection server to have done its thing. + if not server_responding.wait(1.0) and support.verbose: + sys.stdout.write("server_responding event never set.") + self.sock = self._context.wrap_socket( + self.sock, server_hostname=self.host) + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 402 Payment Required\r\n" + b"\r\n") + conn_to_client.close() # RST + server_responding.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="non_tls_http_RST_responder") + self.enterContext(server) # starts it & unittest.TestCase stops it. + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + connection = SynchronizedHTTPSConnection( + f"localhost", + port=server.port, + context=ssl.create_default_context(), + timeout=2.0, + ) + # There are lots of reasons this raises as desired, long before this + # test was added. Sending the request requires a successful TLS wrapped + # socket; that fails if the connection is broken. It may seem pointless + # to test this. It serves as an illustration of something that we never + # want to happen... properly not happening. + with self.assertRaises(OSError) as err_ctx: + connection.request("HEAD", "/test", headers={"Host": "localhost"}) + response = connection.getresponse() + + class TestEnumerations(unittest.TestCase): def test_tlsversion(self): diff --git a/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst new file mode 100644 index 0000000000000..403c77a9d480e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst @@ -0,0 +1,7 @@ +Fixed an issue where instances of :class:`ssl.SSLSocket` were vulnerable to +a bypass of the TLS handshake and included protections (like certificate +verification) and treating sent unencrypted data as if it were +post-handshake TLS encrypted data. Security issue reported as +`CVE-2023-40217 +`_ by +Aapo Oksman. Patch by Gregory P. Smith. From webhook-mailer at python.org Tue Aug 22 13:53:27 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 17:53:27 -0000 Subject: [Python-checkins] [3.10] gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw (#108318) Message-ID: https://github.com/python/cpython/commit/37d7180cb647f0bed0c1caab0037f3bc82e2af96 commit: 37d7180cb647f0bed0c1caab0037f3bc82e2af96 branch: 3.10 author: ?ukasz Langa committer: ambv date: 2023-08-22T19:53:23+02:00 summary: [3.10] gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw (#108318) gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw Instances of `ssl.SSLSocket` were vulnerable to a bypass of the TLS handshake and included protections (like certificate verification) and treating sent unencrypted data as if it were post-handshake TLS encrypted data. The vulnerability is caused when a socket is connected, data is sent by the malicious peer and stored in a buffer, and then the malicious peer closes the socket within a small timing window before the other peers? TLS handshake can begin. After this sequence of events the closed socket will not immediately attempt a TLS handshake due to not being connected but will also allow the buffered data to be read as if a successful TLS handshake had occurred. Co-authored-by: Gregory P. Smith [Google LLC] files: A Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst M Lib/ssl.py M Lib/test/test_ssl.py diff --git a/Lib/ssl.py b/Lib/ssl.py index b09d684ed0fc6..fadbee217afd5 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1033,7 +1033,7 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, ) self = cls.__new__(cls, **kwargs) super(SSLSocket, self).__init__(**kwargs) - self.settimeout(sock.gettimeout()) + sock_timeout = sock.gettimeout() sock.detach() self._context = context @@ -1052,9 +1052,38 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, if e.errno != errno.ENOTCONN: raise connected = False + blocking = self.getblocking() + self.setblocking(False) + try: + # We are not connected so this is not supposed to block, but + # testing revealed otherwise on macOS and Windows so we do + # the non-blocking dance regardless. Our raise when any data + # is found means consuming the data is harmless. + notconn_pre_handshake_data = self.recv(1) + except OSError as e: + # EINVAL occurs for recv(1) on non-connected on unix sockets. + if e.errno not in (errno.ENOTCONN, errno.EINVAL): + raise + notconn_pre_handshake_data = b'' + self.setblocking(blocking) + if notconn_pre_handshake_data: + # This prevents pending data sent to the socket before it was + # closed from escaping to the caller who could otherwise + # presume it came through a successful TLS connection. + reason = "Closed before TLS handshake with data in recv buffer." + notconn_pre_handshake_data_error = SSLError(e.errno, reason) + # Add the SSLError attributes that _ssl.c always adds. + notconn_pre_handshake_data_error.reason = reason + notconn_pre_handshake_data_error.library = None + try: + self.close() + except OSError: + pass + raise notconn_pre_handshake_data_error else: connected = True + self.settimeout(sock_timeout) # Must come after setblocking() calls. self._connected = connected if connected: # create the SSL object diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index c28a574f8e052..a6be80d14fdff 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -9,11 +9,14 @@ from test.support import socket_helper from test.support import threading_helper from test.support import warnings_helper +import re import socket import select +import struct import time import datetime import gc +import http.client import os import errno import pprint @@ -4904,6 +4907,218 @@ def sni_cb(sock, servername, ctx): s.connect((HOST, server.port)) +def set_socket_so_linger_on_with_zero_timeout(sock): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) + + +class TestPreHandshakeClose(unittest.TestCase): + """Verify behavior of close sockets with received data before to the handshake. + """ + + class SingleConnectionTestServerThread(threading.Thread): + + def __init__(self, *, name, call_after_accept): + self.call_after_accept = call_after_accept + self.received_data = b'' # set by .run() + self.wrap_error = None # set by .run() + self.listener = None # set by .start() + self.port = None # set by .start() + super().__init__(name=name) + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + try: + if self.listener: + self.listener.close() + except OSError: + pass + self.join() + self.wrap_error = None # avoid dangling references + + def start(self): + self.ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED + self.ssl_ctx.load_verify_locations(cafile=ONLYCERT) + self.ssl_ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + self.listener = socket.socket() + self.port = socket_helper.bind_port(self.listener) + self.listener.settimeout(2.0) + self.listener.listen(1) + super().start() + + def run(self): + conn, address = self.listener.accept() + self.listener.close() + with conn: + if self.call_after_accept(conn): + return + try: + tls_socket = self.ssl_ctx.wrap_socket(conn, server_side=True) + except OSError as err: # ssl.SSLError inherits from OSError + self.wrap_error = err + else: + try: + self.received_data = tls_socket.recv(400) + except OSError: + pass # closed, protocol error, etc. + + def non_linux_skip_if_other_okay_error(self, err): + if sys.platform == "linux": + return # Expect the full test setup to always work on Linux. + if (isinstance(err, ConnectionResetError) or + (isinstance(err, OSError) and err.errno == errno.EINVAL) or + re.search('wrong.version.number', getattr(err, "reason", ""), re.I)): + # On Windows the TCP RST leads to a ConnectionResetError + # (ECONNRESET) which Linux doesn't appear to surface to userspace. + # If wrap_socket() winds up on the "if connected:" path and doing + # the actual wrapping... we get an SSLError from OpenSSL. Typically + # WRONG_VERSION_NUMBER. While appropriate, neither is the scenario + # we're specifically trying to test. The way this test is written + # is known to work on Linux. We'll skip it anywhere else that it + # does not present as doing so. + self.skipTest(f"Could not recreate conditions on {sys.platform}:" + f" {err=}") + # If maintaining this conditional winds up being a problem. + # just turn this into an unconditional skip anything but Linux. + # The important thing is that our CI has the logic covered. + + def test_preauth_data_to_tls_server(self): + server_accept_called = threading.Event() + ready_for_server_wrap_socket = threading.Event() + + def call_after_accept(unused): + server_accept_called.set() + if not ready_for_server_wrap_socket.wait(2.0): + raise RuntimeError("wrap_socket event never set, test may fail.") + return False # Tell the server thread to continue. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_server") + server.__enter__() # starts it + self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it. + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(client) + client.setblocking(False) + + server_accept_called.wait() + client.send(b"DELETE /data HTTP/1.0\r\n\r\n") + client.close() # RST + + ready_for_server_wrap_socket.set() + server.join() + wrap_error = server.wrap_error + self.assertEqual(b"", server.received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + + def test_preauth_data_to_tls_client(self): + client_can_continue_with_wrap_socket = threading.Event() + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 307 Temporary Redirect\r\n" + b"Location: https://example.com/someone-elses-server\r\n" + b"\r\n") + conn_to_client.close() # RST + client_can_continue_with_wrap_socket.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_client") + server.__enter__() # starts it + self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it. + + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + if not client_can_continue_with_wrap_socket.wait(2.0): + self.fail("test server took too long.") + ssl_ctx = ssl.create_default_context() + try: + tls_client = ssl_ctx.wrap_socket( + client, server_hostname="localhost") + except OSError as err: # SSLError inherits from OSError + wrap_error = err + received_data = b"" + else: + wrap_error = None + received_data = tls_client.recv(400) + tls_client.close() + + server.join() + self.assertEqual(b"", received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + + def test_https_client_non_tls_response_ignored(self): + + server_responding = threading.Event() + + class SynchronizedHTTPSConnection(http.client.HTTPSConnection): + def connect(self): + http.client.HTTPConnection.connect(self) + # Wait for our fault injection server to have done its thing. + if not server_responding.wait(1.0) and support.verbose: + sys.stdout.write("server_responding event never set.") + self.sock = self._context.wrap_socket( + self.sock, server_hostname=self.host) + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 402 Payment Required\r\n" + b"\r\n") + conn_to_client.close() # RST + server_responding.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="non_tls_http_RST_responder") + server.__enter__() # starts it + self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it. + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + connection = SynchronizedHTTPSConnection( + f"localhost", + port=server.port, + context=ssl.create_default_context(), + timeout=2.0, + ) + # There are lots of reasons this raises as desired, long before this + # test was added. Sending the request requires a successful TLS wrapped + # socket; that fails if the connection is broken. It may seem pointless + # to test this. It serves as an illustration of something that we never + # want to happen... properly not happening. + with self.assertRaises(OSError) as err_ctx: + connection.request("HEAD", "/test", headers={"Host": "localhost"}) + response = connection.getresponse() + + def setUpModule(): if support.verbose: plats = { diff --git a/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst new file mode 100644 index 0000000000000..403c77a9d480e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst @@ -0,0 +1,7 @@ +Fixed an issue where instances of :class:`ssl.SSLSocket` were vulnerable to +a bypass of the TLS handshake and included protections (like certificate +verification) and treating sent unencrypted data as if it were +post-handshake TLS encrypted data. Security issue reported as +`CVE-2023-40217 +`_ by +Aapo Oksman. Patch by Gregory P. Smith. From webhook-mailer at python.org Tue Aug 22 13:57:05 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 17:57:05 -0000 Subject: [Python-checkins] [3.8] gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw (#108321) Message-ID: https://github.com/python/cpython/commit/b4bcc06a9cfe13d96d5270809d963f8ba278f89b commit: b4bcc06a9cfe13d96d5270809d963f8ba278f89b branch: 3.8 author: ?ukasz Langa committer: ambv date: 2023-08-22T19:57:01+02:00 summary: [3.8] gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw (#108321) gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw Instances of `ssl.SSLSocket` were vulnerable to a bypass of the TLS handshake and included protections (like certificate verification) and treating sent unencrypted data as if it were post-handshake TLS encrypted data. The vulnerability is caused when a socket is connected, data is sent by the malicious peer and stored in a buffer, and then the malicious peer closes the socket within a small timing window before the other peers? TLS handshake can begin. After this sequence of events the closed socket will not immediately attempt a TLS handshake due to not being connected but will also allow the buffered data to be read as if a successful TLS handshake had occurred. Co-authored-by: Gregory P. Smith [Google LLC] files: A Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst M Lib/ssl.py M Lib/test/test_ssl.py diff --git a/Lib/ssl.py b/Lib/ssl.py index 0726caee49aa5..bd9ba346dae8c 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1002,7 +1002,7 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, ) self = cls.__new__(cls, **kwargs) super(SSLSocket, self).__init__(**kwargs) - self.settimeout(sock.gettimeout()) + sock_timeout = sock.gettimeout() sock.detach() self._context = context @@ -1021,9 +1021,38 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, if e.errno != errno.ENOTCONN: raise connected = False + blocking = self.getblocking() + self.setblocking(False) + try: + # We are not connected so this is not supposed to block, but + # testing revealed otherwise on macOS and Windows so we do + # the non-blocking dance regardless. Our raise when any data + # is found means consuming the data is harmless. + notconn_pre_handshake_data = self.recv(1) + except OSError as e: + # EINVAL occurs for recv(1) on non-connected on unix sockets. + if e.errno not in (errno.ENOTCONN, errno.EINVAL): + raise + notconn_pre_handshake_data = b'' + self.setblocking(blocking) + if notconn_pre_handshake_data: + # This prevents pending data sent to the socket before it was + # closed from escaping to the caller who could otherwise + # presume it came through a successful TLS connection. + reason = "Closed before TLS handshake with data in recv buffer." + notconn_pre_handshake_data_error = SSLError(e.errno, reason) + # Add the SSLError attributes that _ssl.c always adds. + notconn_pre_handshake_data_error.reason = reason + notconn_pre_handshake_data_error.library = None + try: + self.close() + except OSError: + pass + raise notconn_pre_handshake_data_error else: connected = True + self.settimeout(sock_timeout) # Must come after setblocking() calls. self._connected = connected if connected: # create the SSL object diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 71cfdcdf8a1c9..8de491566b839 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4,11 +4,14 @@ import unittest import unittest.mock from test import support +import re import socket import select +import struct import time import datetime import gc +import http.client import os import errno import pprint @@ -4815,6 +4818,218 @@ def sni_cb(sock, servername, ctx): s.connect((HOST, server.port)) +def set_socket_so_linger_on_with_zero_timeout(sock): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) + + +class TestPreHandshakeClose(unittest.TestCase): + """Verify behavior of close sockets with received data before to the handshake. + """ + + class SingleConnectionTestServerThread(threading.Thread): + + def __init__(self, *, name, call_after_accept): + self.call_after_accept = call_after_accept + self.received_data = b'' # set by .run() + self.wrap_error = None # set by .run() + self.listener = None # set by .start() + self.port = None # set by .start() + super().__init__(name=name) + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + try: + if self.listener: + self.listener.close() + except OSError: + pass + self.join() + self.wrap_error = None # avoid dangling references + + def start(self): + self.ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED + self.ssl_ctx.load_verify_locations(cafile=ONLYCERT) + self.ssl_ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + self.listener = socket.socket() + self.port = support.bind_port(self.listener) + self.listener.settimeout(2.0) + self.listener.listen(1) + super().start() + + def run(self): + conn, address = self.listener.accept() + self.listener.close() + with conn: + if self.call_after_accept(conn): + return + try: + tls_socket = self.ssl_ctx.wrap_socket(conn, server_side=True) + except OSError as err: # ssl.SSLError inherits from OSError + self.wrap_error = err + else: + try: + self.received_data = tls_socket.recv(400) + except OSError: + pass # closed, protocol error, etc. + + def non_linux_skip_if_other_okay_error(self, err): + if sys.platform == "linux": + return # Expect the full test setup to always work on Linux. + if (isinstance(err, ConnectionResetError) or + (isinstance(err, OSError) and err.errno == errno.EINVAL) or + re.search('wrong.version.number', getattr(err, "reason", ""), re.I)): + # On Windows the TCP RST leads to a ConnectionResetError + # (ECONNRESET) which Linux doesn't appear to surface to userspace. + # If wrap_socket() winds up on the "if connected:" path and doing + # the actual wrapping... we get an SSLError from OpenSSL. Typically + # WRONG_VERSION_NUMBER. While appropriate, neither is the scenario + # we're specifically trying to test. The way this test is written + # is known to work on Linux. We'll skip it anywhere else that it + # does not present as doing so. + self.skipTest(f"Could not recreate conditions on {sys.platform}:" + f" {err=}") + # If maintaining this conditional winds up being a problem. + # just turn this into an unconditional skip anything but Linux. + # The important thing is that our CI has the logic covered. + + def test_preauth_data_to_tls_server(self): + server_accept_called = threading.Event() + ready_for_server_wrap_socket = threading.Event() + + def call_after_accept(unused): + server_accept_called.set() + if not ready_for_server_wrap_socket.wait(2.0): + raise RuntimeError("wrap_socket event never set, test may fail.") + return False # Tell the server thread to continue. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_server") + server.__enter__() # starts it + self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it. + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(client) + client.setblocking(False) + + server_accept_called.wait() + client.send(b"DELETE /data HTTP/1.0\r\n\r\n") + client.close() # RST + + ready_for_server_wrap_socket.set() + server.join() + wrap_error = server.wrap_error + self.assertEqual(b"", server.received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + + def test_preauth_data_to_tls_client(self): + client_can_continue_with_wrap_socket = threading.Event() + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 307 Temporary Redirect\r\n" + b"Location: https://example.com/someone-elses-server\r\n" + b"\r\n") + conn_to_client.close() # RST + client_can_continue_with_wrap_socket.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_client") + server.__enter__() # starts it + self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it. + + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + if not client_can_continue_with_wrap_socket.wait(2.0): + self.fail("test server took too long.") + ssl_ctx = ssl.create_default_context() + try: + tls_client = ssl_ctx.wrap_socket( + client, server_hostname="localhost") + except OSError as err: # SSLError inherits from OSError + wrap_error = err + received_data = b"" + else: + wrap_error = None + received_data = tls_client.recv(400) + tls_client.close() + + server.join() + self.assertEqual(b"", received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + + def test_https_client_non_tls_response_ignored(self): + + server_responding = threading.Event() + + class SynchronizedHTTPSConnection(http.client.HTTPSConnection): + def connect(self): + http.client.HTTPConnection.connect(self) + # Wait for our fault injection server to have done its thing. + if not server_responding.wait(1.0) and support.verbose: + sys.stdout.write("server_responding event never set.") + self.sock = self._context.wrap_socket( + self.sock, server_hostname=self.host) + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 402 Payment Required\r\n" + b"\r\n") + conn_to_client.close() # RST + server_responding.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="non_tls_http_RST_responder") + server.__enter__() # starts it + self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it. + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + connection = SynchronizedHTTPSConnection( + f"localhost", + port=server.port, + context=ssl.create_default_context(), + timeout=2.0, + ) + # There are lots of reasons this raises as desired, long before this + # test was added. Sending the request requires a successful TLS wrapped + # socket; that fails if the connection is broken. It may seem pointless + # to test this. It serves as an illustration of something that we never + # want to happen... properly not happening. + with self.assertRaises(OSError) as err_ctx: + connection.request("HEAD", "/test", headers={"Host": "localhost"}) + response = connection.getresponse() + + def test_main(verbose=False): if support.verbose: plats = { diff --git a/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst new file mode 100644 index 0000000000000..403c77a9d480e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst @@ -0,0 +1,7 @@ +Fixed an issue where instances of :class:`ssl.SSLSocket` were vulnerable to +a bypass of the TLS handshake and included protections (like certificate +verification) and treating sent unencrypted data as if it were +post-handshake TLS encrypted data. Security issue reported as +`CVE-2023-40217 +`_ by +Aapo Oksman. Patch by Gregory P. Smith. From webhook-mailer at python.org Tue Aug 22 13:57:13 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 17:57:13 -0000 Subject: [Python-checkins] [3.9] gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw (#108320) Message-ID: https://github.com/python/cpython/commit/264b1dacc67346efa0933d1e63f622676e0ed96b commit: 264b1dacc67346efa0933d1e63f622676e0ed96b branch: 3.9 author: ?ukasz Langa committer: ambv date: 2023-08-22T19:57:10+02:00 summary: [3.9] gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw (#108320) gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw Instances of `ssl.SSLSocket` were vulnerable to a bypass of the TLS handshake and included protections (like certificate verification) and treating sent unencrypted data as if it were post-handshake TLS encrypted data. The vulnerability is caused when a socket is connected, data is sent by the malicious peer and stored in a buffer, and then the malicious peer closes the socket within a small timing window before the other peers? TLS handshake can begin. After this sequence of events the closed socket will not immediately attempt a TLS handshake due to not being connected but will also allow the buffered data to be read as if a successful TLS handshake had occurred. Co-authored-by: Gregory P. Smith [Google LLC] files: A Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst M Lib/ssl.py M Lib/test/test_ssl.py diff --git a/Lib/ssl.py b/Lib/ssl.py index 0e3606b835c6a..ef92e76a7c540 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1003,7 +1003,7 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, ) self = cls.__new__(cls, **kwargs) super(SSLSocket, self).__init__(**kwargs) - self.settimeout(sock.gettimeout()) + sock_timeout = sock.gettimeout() sock.detach() self._context = context @@ -1022,9 +1022,38 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, if e.errno != errno.ENOTCONN: raise connected = False + blocking = self.getblocking() + self.setblocking(False) + try: + # We are not connected so this is not supposed to block, but + # testing revealed otherwise on macOS and Windows so we do + # the non-blocking dance regardless. Our raise when any data + # is found means consuming the data is harmless. + notconn_pre_handshake_data = self.recv(1) + except OSError as e: + # EINVAL occurs for recv(1) on non-connected on unix sockets. + if e.errno not in (errno.ENOTCONN, errno.EINVAL): + raise + notconn_pre_handshake_data = b'' + self.setblocking(blocking) + if notconn_pre_handshake_data: + # This prevents pending data sent to the socket before it was + # closed from escaping to the caller who could otherwise + # presume it came through a successful TLS connection. + reason = "Closed before TLS handshake with data in recv buffer." + notconn_pre_handshake_data_error = SSLError(e.errno, reason) + # Add the SSLError attributes that _ssl.c always adds. + notconn_pre_handshake_data_error.reason = reason + notconn_pre_handshake_data_error.library = None + try: + self.close() + except OSError: + pass + raise notconn_pre_handshake_data_error else: connected = True + self.settimeout(sock_timeout) # Must come after setblocking() calls. self._connected = connected if connected: # create the SSL object diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index e270f44cf3fba..aeca48145b061 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -5,11 +5,14 @@ import unittest.mock from test import support from test.support import socket_helper, warnings_helper +import re import socket import select +import struct import time import datetime import gc +import http.client import os import errno import pprint @@ -4842,6 +4845,218 @@ def sni_cb(sock, servername, ctx): s.connect((HOST, server.port)) +def set_socket_so_linger_on_with_zero_timeout(sock): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) + + +class TestPreHandshakeClose(unittest.TestCase): + """Verify behavior of close sockets with received data before to the handshake. + """ + + class SingleConnectionTestServerThread(threading.Thread): + + def __init__(self, *, name, call_after_accept): + self.call_after_accept = call_after_accept + self.received_data = b'' # set by .run() + self.wrap_error = None # set by .run() + self.listener = None # set by .start() + self.port = None # set by .start() + super().__init__(name=name) + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + try: + if self.listener: + self.listener.close() + except OSError: + pass + self.join() + self.wrap_error = None # avoid dangling references + + def start(self): + self.ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED + self.ssl_ctx.load_verify_locations(cafile=ONLYCERT) + self.ssl_ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + self.listener = socket.socket() + self.port = socket_helper.bind_port(self.listener) + self.listener.settimeout(2.0) + self.listener.listen(1) + super().start() + + def run(self): + conn, address = self.listener.accept() + self.listener.close() + with conn: + if self.call_after_accept(conn): + return + try: + tls_socket = self.ssl_ctx.wrap_socket(conn, server_side=True) + except OSError as err: # ssl.SSLError inherits from OSError + self.wrap_error = err + else: + try: + self.received_data = tls_socket.recv(400) + except OSError: + pass # closed, protocol error, etc. + + def non_linux_skip_if_other_okay_error(self, err): + if sys.platform == "linux": + return # Expect the full test setup to always work on Linux. + if (isinstance(err, ConnectionResetError) or + (isinstance(err, OSError) and err.errno == errno.EINVAL) or + re.search('wrong.version.number', getattr(err, "reason", ""), re.I)): + # On Windows the TCP RST leads to a ConnectionResetError + # (ECONNRESET) which Linux doesn't appear to surface to userspace. + # If wrap_socket() winds up on the "if connected:" path and doing + # the actual wrapping... we get an SSLError from OpenSSL. Typically + # WRONG_VERSION_NUMBER. While appropriate, neither is the scenario + # we're specifically trying to test. The way this test is written + # is known to work on Linux. We'll skip it anywhere else that it + # does not present as doing so. + self.skipTest(f"Could not recreate conditions on {sys.platform}:" + f" {err=}") + # If maintaining this conditional winds up being a problem. + # just turn this into an unconditional skip anything but Linux. + # The important thing is that our CI has the logic covered. + + def test_preauth_data_to_tls_server(self): + server_accept_called = threading.Event() + ready_for_server_wrap_socket = threading.Event() + + def call_after_accept(unused): + server_accept_called.set() + if not ready_for_server_wrap_socket.wait(2.0): + raise RuntimeError("wrap_socket event never set, test may fail.") + return False # Tell the server thread to continue. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_server") + server.__enter__() # starts it + self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it. + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(client) + client.setblocking(False) + + server_accept_called.wait() + client.send(b"DELETE /data HTTP/1.0\r\n\r\n") + client.close() # RST + + ready_for_server_wrap_socket.set() + server.join() + wrap_error = server.wrap_error + self.assertEqual(b"", server.received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + + def test_preauth_data_to_tls_client(self): + client_can_continue_with_wrap_socket = threading.Event() + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 307 Temporary Redirect\r\n" + b"Location: https://example.com/someone-elses-server\r\n" + b"\r\n") + conn_to_client.close() # RST + client_can_continue_with_wrap_socket.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_client") + server.__enter__() # starts it + self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it. + + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + if not client_can_continue_with_wrap_socket.wait(2.0): + self.fail("test server took too long.") + ssl_ctx = ssl.create_default_context() + try: + tls_client = ssl_ctx.wrap_socket( + client, server_hostname="localhost") + except OSError as err: # SSLError inherits from OSError + wrap_error = err + received_data = b"" + else: + wrap_error = None + received_data = tls_client.recv(400) + tls_client.close() + + server.join() + self.assertEqual(b"", received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + + def test_https_client_non_tls_response_ignored(self): + + server_responding = threading.Event() + + class SynchronizedHTTPSConnection(http.client.HTTPSConnection): + def connect(self): + http.client.HTTPConnection.connect(self) + # Wait for our fault injection server to have done its thing. + if not server_responding.wait(1.0) and support.verbose: + sys.stdout.write("server_responding event never set.") + self.sock = self._context.wrap_socket( + self.sock, server_hostname=self.host) + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 402 Payment Required\r\n" + b"\r\n") + conn_to_client.close() # RST + server_responding.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="non_tls_http_RST_responder") + server.__enter__() # starts it + self.addCleanup(server.__exit__) # ... & unittest.TestCase stops it. + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + connection = SynchronizedHTTPSConnection( + f"localhost", + port=server.port, + context=ssl.create_default_context(), + timeout=2.0, + ) + # There are lots of reasons this raises as desired, long before this + # test was added. Sending the request requires a successful TLS wrapped + # socket; that fails if the connection is broken. It may seem pointless + # to test this. It serves as an illustration of something that we never + # want to happen... properly not happening. + with self.assertRaises(OSError) as err_ctx: + connection.request("HEAD", "/test", headers={"Host": "localhost"}) + response = connection.getresponse() + + def setUpModule(): if support.verbose: plats = { diff --git a/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst new file mode 100644 index 0000000000000..403c77a9d480e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst @@ -0,0 +1,7 @@ +Fixed an issue where instances of :class:`ssl.SSLSocket` were vulnerable to +a bypass of the TLS handshake and included protections (like certificate +verification) and treating sent unencrypted data as if it were +post-handshake TLS encrypted data. Security issue reported as +`CVE-2023-40217 +`_ by +Aapo Oksman. Patch by Gregory P. Smith. From webhook-mailer at python.org Tue Aug 22 14:17:29 2023 From: webhook-mailer at python.org (vstinner) Date: Tue, 22 Aug 2023 18:17:29 -0000 Subject: [Python-checkins] gh-106320: Remove _PyDict_GetItemStringWithError() function (#108313) Message-ID: https://github.com/python/cpython/commit/615f6e946db6d7eae66b8de9a75a6c58a84516a6 commit: 615f6e946db6d7eae66b8de9a75a6c58a84516a6 branch: main author: Victor Stinner committer: vstinner date: 2023-08-22T18:17:25Z summary: gh-106320: Remove _PyDict_GetItemStringWithError() function (#108313) Remove private _PyDict_GetItemStringWithError() function of the public C API: the new PyDict_GetItemStringRef() can be used instead. * Move private _PyDict_GetItemStringWithError() to the internal C API. * _testcapi get_code_extra_index() uses PyDict_GetItemStringRef(). Avoid using private functions in _testcapi which tests the public C API. files: M Include/cpython/dictobject.h M Include/internal/pycore_dict.h M Modules/_testcapi/code.c M Objects/structseq.c M Python/codecs.c M Python/import.c M Python/initconfig.c M Python/pythonrun.c diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 2a42794fdf0c8..470f59436748f 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -36,7 +36,6 @@ PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); PyAPI_FUNC(PyObject *) _PyDict_GetItemIdWithError(PyObject *dp, _Py_Identifier *key); -PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *); PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index df7bc7e58f6c9..8e9c27e3b294b 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -13,6 +13,7 @@ extern "C" { // Unsafe flavor of PyDict_GetItemWithError(): no error checking extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key); +extern PyObject* _PyDict_GetItemStringWithError(PyObject *, const char *); extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); diff --git a/Modules/_testcapi/code.c b/Modules/_testcapi/code.c index cadaf5eb94692..691dd5fe04381 100644 --- a/Modules/_testcapi/code.c +++ b/Modules/_testcapi/code.c @@ -9,12 +9,12 @@ get_code_extra_index(PyInterpreterState* interp) { PyObject *interp_dict = PyInterpreterState_GetDict(interp); // borrowed assert(interp_dict); // real users would handle missing dict... somehow - PyObject *index_obj = _PyDict_GetItemStringWithError(interp_dict, key); // borrowed + PyObject *index_obj; + if (PyDict_GetItemStringRef(interp_dict, key, &index_obj) < 0) { + goto finally; + } Py_ssize_t index = 0; if (!index_obj) { - if (PyErr_Occurred()) { - goto finally; - } index = PyUnstable_Eval_RequestCodeExtraIndex(NULL); if (index < 0 || PyErr_Occurred()) { goto finally; @@ -31,6 +31,7 @@ get_code_extra_index(PyInterpreterState* interp) { } else { index = PyLong_AsSsize_t(index_obj); + Py_DECREF(index_obj); if (index == -1 && PyErr_Occurred()) { goto finally; } diff --git a/Objects/structseq.c b/Objects/structseq.c index 700f67c09c9e5..95c4c15710d16 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -8,6 +8,7 @@ */ #include "Python.h" +#include "pycore_dict.h" // _PyDict_GetItemStringWithError() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_object.h" // _PyObject_GC_TRACK() diff --git a/Python/codecs.c b/Python/codecs.c index 4e47ff93a3691..3c418512e3aa0 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -10,6 +10,7 @@ Copyright (c) Corporation for National Research Initiatives. #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_dict.h" // _PyDict_GetItemStringWithError() #include "pycore_interp.h" // PyInterpreterState.codec_search_path #include "pycore_pyerrors.h" // _PyErr_FormatNote() #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Python/import.c b/Python/import.c index 56b2dc1a4ada2..4abb2f6a06b48 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1,7 +1,7 @@ /* Module definition and import implementation */ #include "Python.h" - +#include "pycore_dict.h" // _PyDict_GetItemStringWithError() #include "pycore_hashtable.h" // _Py_hashtable_new_full() #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -15,6 +15,7 @@ #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_weakref.h" // _PyWeakref_GET_REF() + #include "marshal.h" // PyMarshal_ReadObjectFromString() #include "importdl.h" // _PyImport_DynLoadFiletab #include "pydtrace.h" // PyDTrace_IMPORT_FIND_LOAD_START_ENABLED() diff --git a/Python/initconfig.c b/Python/initconfig.c index 787e583262406..c017abeb90563 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_dict.h" // _PyDict_GetItemStringWithError() #include "pycore_fileutils.h" // _Py_HasFileSystemDefaultEncodeErrors #include "pycore_getopt.h" // _PyOS_GetOpt() #include "pycore_initconfig.h" // _PyStatus_OK() diff --git a/Python/pythonrun.c b/Python/pythonrun.c index b2e04cfa317c0..a3de7792cf5bc 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -15,6 +15,7 @@ #include "pycore_ast.h" // PyAST_mod2obj #include "pycore_ceval.h" // _Py_EnterRecursiveCall #include "pycore_compile.h" // _PyAST_Compile() +#include "pycore_dict.h" // _PyDict_GetItemStringWithError() #include "pycore_interp.h" // PyInterpreterState.importlib #include "pycore_object.h" // _PyDebug_PrintTotalRefs() #include "pycore_parser.h" // _PyParser_ASTFromString() From webhook-mailer at python.org Tue Aug 22 14:20:55 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:20:55 -0000 Subject: [Python-checkins] [3.8] gh-107565: Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, and 3.1.2. (#108124) Message-ID: https://github.com/python/cpython/commit/a5d1571cdde38885586b1ac386d5f57d1079faf2 commit: a5d1571cdde38885586b1ac386d5f57d1079faf2 branch: 3.8 author: Ned Deily committer: ambv date: 2023-08-22T20:20:51+02:00 summary: [3.8] gh-107565: Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, and 3.1.2. (#108124) (cherry picked from commit 441797d4ffb12acda257370b9e5e19ed8d6e8a71) files: A Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst M .github/workflows/build.yml M Tools/ssl/multissltests.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b7bc77181515b..b8c9483a90fb7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -158,7 +158,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 1.1.1v steps: - uses: actions/checkout at v2 - name: Install Dependencies @@ -199,7 +199,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.0.2u, 1.1.1u, 3.0.9, 3.1.1] + openssl_ver: [1.0.2u, 1.1.1v, 3.0.10, 3.1.2] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst new file mode 100644 index 0000000000000..c43ee680e8158 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst @@ -0,0 +1,2 @@ +Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, +and 3.1.2. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index d6cc1f6e3a0c3..8a27a6b78eaae 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -49,8 +49,8 @@ ] OPENSSL_RECENT_VERSIONS = [ - "1.1.1u", - "3.0.9", + "1.1.1v", + "3.0.10", ] LIBRESSL_OLD_VERSIONS = [ From webhook-mailer at python.org Tue Aug 22 14:24:44 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:24:44 -0000 Subject: [Python-checkins] [3.10] gh-99612: Fix PyUnicode_DecodeUTF8Stateful() for ASCII-only data (GH-99613) (GH-107224) (#107230) Message-ID: https://github.com/python/cpython/commit/1c937e58873a4d07d9c94f3f28cd4733f62dc8ab commit: 1c937e58873a4d07d9c94f3f28cd4733f62dc8ab branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2023-08-22T20:24:40+02:00 summary: [3.10] gh-99612: Fix PyUnicode_DecodeUTF8Stateful() for ASCII-only data (GH-99613) (GH-107224) (#107230) Previously *consumed was not set in this case. (cherry picked from commit b8b3e6afc0a48c3cbb7c36d2f73e332edcd6058c) (cherry picked from commit f08e52ccb027f6f703302b8c1a82db9fd3934270) Co-authored-by: Serhiy Storchaka files: A Lib/test/test_capi/test_codecs.py A Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst M Modules/_testcapimodule.c M Objects/unicodeobject.c diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py new file mode 100644 index 0000000000000..e46726192aa05 --- /dev/null +++ b/Lib/test/test_capi/test_codecs.py @@ -0,0 +1,54 @@ +import unittest +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') + + +class CAPITest(unittest.TestCase): + + def test_decodeutf8(self): + """Test PyUnicode_DecodeUTF8()""" + decodeutf8 = _testcapi.unicode_decodeutf8 + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-8') + self.assertEqual(decodeutf8(b), s) + self.assertEqual(decodeutf8(b, 'strict'), s) + + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f') + self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd') + self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb') + + self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo') + # TODO: Test PyUnicode_DecodeUTF8() with NULL as data and + # negative size. + + def test_decodeutf8stateful(self): + """Test PyUnicode_DecodeUTF8Stateful()""" + decodeutf8stateful = _testcapi.unicode_decodeutf8stateful + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-8') + self.assertEqual(decodeutf8stateful(b), (s, len(b))) + self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b))) + + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80') + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0') + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff') + self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1)) + self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1)) + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb') + self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4)) + + self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo') + # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as data and + # negative size. + # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of + # "consumed". + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst b/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst new file mode 100644 index 0000000000000..40e3c8db5403c --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst @@ -0,0 +1,2 @@ +Fix :c:func:`PyUnicode_DecodeUTF8Stateful` for ASCII-only data: +``*consumed`` was not set. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index bafd0c4e697a8..fad0934d47991 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2112,6 +2112,40 @@ unicode_asutf8andsize(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", result, utf8_len); } +/* Test PyUnicode_DecodeUTF8() */ +static PyObject * +unicode_decodeutf8(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUTF8(data, size, errors); +} + +/* Test PyUnicode_DecodeUTF8Stateful() */ +static PyObject * +unicode_decodeutf8stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = 123456789; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed); + if (!result) { + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + static PyObject * unicode_findchar(PyObject *self, PyObject *args) { @@ -5846,7 +5880,8 @@ static PyMethodDef TestMethods[] = { {"unicode_asucs4", unicode_asucs4, METH_VARARGS}, {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, - {"unicode_findchar", unicode_findchar, METH_VARARGS}, + {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, + {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, {"unicode_findchar", unicode_findchar, METH_VARARGS}, {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS}, #if USE_UNICODE_WCHAR_CACHE {"unicode_encodedecimal", unicode_encodedecimal, METH_VARARGS}, diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 156c29bf212ed..3a0065efe15ce 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5206,6 +5206,9 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, } s += ascii_decode(s, end, PyUnicode_1BYTE_DATA(u)); if (s == end) { + if (consumed) { + *consumed = size; + } return u; } From webhook-mailer at python.org Tue Aug 22 14:25:09 2023 From: webhook-mailer at python.org (vstinner) Date: Tue, 22 Aug 2023 18:25:09 -0000 Subject: [Python-checkins] gh-89745: Remove commented code in getpath.c (#108307) Message-ID: https://github.com/python/cpython/commit/c1c9cd61da1c5b6318da9f7055d45ec82a9949ba commit: c1c9cd61da1c5b6318da9f7055d45ec82a9949ba branch: main author: Victor Stinner committer: vstinner date: 2023-08-22T20:25:06+02:00 summary: gh-89745: Remove commented code in getpath.c (#108307) files: M Modules/getpath.c diff --git a/Modules/getpath.c b/Modules/getpath.c index 76e3c7e65249f..fb1656ada0149 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -922,23 +922,6 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) } Py_DECREF(r); -#if 0 - PyObject *it = PyObject_GetIter(configDict); - for (PyObject *k = PyIter_Next(it); k; k = PyIter_Next(it)) { - if (!strcmp("__builtins__", PyUnicode_AsUTF8(k))) { - Py_DECREF(k); - continue; - } - fprintf(stderr, "%s = ", PyUnicode_AsUTF8(k)); - PyObject *o = PyDict_GetItem(configDict, k); - o = PyObject_Repr(o); - fprintf(stderr, "%s\n", PyUnicode_AsUTF8(o)); - Py_DECREF(o); - Py_DECREF(k); - } - Py_DECREF(it); -#endif - if (_PyConfig_FromDict(config, configDict) < 0) { _PyErr_WriteUnraisableMsg("reading getpath results", NULL); Py_DECREF(dict); @@ -949,4 +932,3 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) return _PyStatus_OK(); } - From webhook-mailer at python.org Tue Aug 22 14:25:18 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:25:18 -0000 Subject: [Python-checkins] [3.9] gh-99612: Fix PyUnicode_DecodeUTF8Stateful() for ASCII-only data (GH-99613) (GH-107224) (#107231) Message-ID: https://github.com/python/cpython/commit/4a793281956db0e4a1ca5fdc5f3a0e91f331a75d commit: 4a793281956db0e4a1ca5fdc5f3a0e91f331a75d branch: 3.9 author: Serhiy Storchaka committer: ambv date: 2023-08-22T20:25:15+02:00 summary: [3.9] gh-99612: Fix PyUnicode_DecodeUTF8Stateful() for ASCII-only data (GH-99613) (GH-107224) (#107231) Previously *consumed was not set in this case. (cherry picked from commit f08e52ccb027f6f703302b8c1a82db9fd3934270). (cherry picked from commit b8b3e6afc0a48c3cbb7c36d2f73e332edcd6058c) files: A Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst M Lib/test/test_unicode.py M Modules/_testcapimodule.c M Objects/unicodeobject.c diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 787f841573876..0dd18a637f9cc 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2870,6 +2870,53 @@ def test_asutf8andsize(self): self.assertEqual(unicode_asutf8andsize(nonbmp), (b'\xf4\x8f\xbf\xbf', 4)) self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, 'a\ud800b\udfffc') + @support.cpython_only + def test_decodeutf8(self): + """Test PyUnicode_DecodeUTF8()""" + import _testcapi + decodeutf8 = _testcapi.unicode_decodeutf8 + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-8') + self.assertEqual(decodeutf8(b), s) + self.assertEqual(decodeutf8(b, 'strict'), s) + + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f') + self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd') + self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb') + + self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo') + # TODO: Test PyUnicode_DecodeUTF8() with NULL as data and + # negative size. + + @support.cpython_only + def test_decodeutf8stateful(self): + """Test PyUnicode_DecodeUTF8Stateful()""" + import _testcapi + decodeutf8stateful = _testcapi.unicode_decodeutf8stateful + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-8') + self.assertEqual(decodeutf8stateful(b), (s, len(b))) + self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b))) + + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80') + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0') + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff') + self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1)) + self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1)) + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb') + self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4)) + + self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo') + # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as data and + # negative size. + # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of + # "consumed". + # Test PyUnicode_FindChar() @support.cpython_only def test_findchar(self): diff --git a/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst b/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst new file mode 100644 index 0000000000000..40e3c8db5403c --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst @@ -0,0 +1,2 @@ +Fix :c:func:`PyUnicode_DecodeUTF8Stateful` for ASCII-only data: +``*consumed`` was not set. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 361ce7b55a26d..8380ad9a34cec 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1987,6 +1987,40 @@ unicode_asutf8andsize(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", result, utf8_len); } +/* Test PyUnicode_DecodeUTF8() */ +static PyObject * +unicode_decodeutf8(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUTF8(data, size, errors); +} + +/* Test PyUnicode_DecodeUTF8Stateful() */ +static PyObject * +unicode_decodeutf8stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = 123456789; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed); + if (!result) { + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + static PyObject * unicode_findchar(PyObject *self, PyObject *args) { @@ -5502,7 +5536,8 @@ static PyMethodDef TestMethods[] = { {"unicode_asucs4", unicode_asucs4, METH_VARARGS}, {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, - {"unicode_findchar", unicode_findchar, METH_VARARGS}, + {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, + {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, {"unicode_findchar", unicode_findchar, METH_VARARGS}, {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS}, {"unicode_encodedecimal", unicode_encodedecimal, METH_VARARGS}, {"unicode_transformdecimaltoascii", unicode_transformdecimaltoascii, METH_VARARGS}, diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index e5e9ba20301c0..bd08b198781ae 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5052,6 +5052,9 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, } s += ascii_decode(s, end, PyUnicode_1BYTE_DATA(u)); if (s == end) { + if (consumed) { + *consumed = size; + } return u; } From webhook-mailer at python.org Tue Aug 22 14:28:14 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:28:14 -0000 Subject: [Python-checkins] [3.9] gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) (#108274) Message-ID: https://github.com/python/cpython/commit/42deeab5b2efc2930d4eb73416e1dde9cf790dd2 commit: 42deeab5b2efc2930d4eb73416e1dde9cf790dd2 branch: 3.9 author: Petr Viktorin committer: ambv date: 2023-08-22T20:28:10+02:00 summary: [3.9] gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) (#108274) (cherry picked from commit acbd3f9c5c5f23e95267714e41236140d84fe962) Co-authored-by: Petr Viktorin Co-authored-by: Victor Stinner Co-authored-by: Lum?r 'Frenzy' Balhar files: A Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst M Doc/library/tarfile.rst M Lib/tarfile.py M Lib/test/test_tarfile.py diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index f1d9bd34536b1..165088529a856 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -732,6 +732,11 @@ A ``TarInfo`` object has the following public data attributes: Name of the target file name, which is only present in :class:`TarInfo` objects of type :const:`LNKTYPE` and :const:`SYMTYPE`. + For symbolic links (``SYMTYPE``), the *linkname* is relative to the directory + that contains the link. + For hard links (``LNKTYPE``), the *linkname* is relative to the root of + the archive. + .. attribute:: TarInfo.uid :type: int diff --git a/Lib/tarfile.py b/Lib/tarfile.py index b6ad7dbe2a4dd..7a6158c2eb989 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -740,7 +740,7 @@ def __init__(self, tarinfo): class AbsoluteLinkError(FilterError): def __init__(self, tarinfo): self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a symlink to an absolute path') + super().__init__(f'{tarinfo.name!r} is a link to an absolute path') class LinkOutsideDestinationError(FilterError): def __init__(self, tarinfo, path): @@ -800,7 +800,14 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) - target_path = os.path.realpath(os.path.join(dest_path, member.linkname)) + if member.issym(): + target_path = os.path.join(dest_path, + os.path.dirname(name), + member.linkname) + else: + target_path = os.path.join(dest_path, + member.linkname) + target_path = os.path.realpath(target_path) if os.path.commonpath([target_path, dest_path]) != dest_path: raise LinkOutsideDestinationError(member, target_path) return new_attrs diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index a66f7efd2d6ce..3df64c7803227 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3169,10 +3169,12 @@ def __exit__(self, *exc): self.bio = None def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, - mode=None, **kwargs): + mode=None, size=None, **kwargs): """Add a member to the test archive. Call within `with`.""" name = str(name) tarinfo = tarfile.TarInfo(name).replace(**kwargs) + if size is not None: + tarinfo.size = size if mode: tarinfo.mode = _filemode_to_int(mode) if symlink_to is not None: @@ -3236,7 +3238,8 @@ def check_context(self, tar, filter): raise self.raised_exception self.assertEqual(self.expected_paths, set()) - def expect_file(self, name, type=None, symlink_to=None, mode=None): + def expect_file(self, name, type=None, symlink_to=None, mode=None, + size=None): """Check a single file. See check_context.""" if self.raised_exception: raise self.raised_exception @@ -3270,6 +3273,8 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None): self.assertTrue(path.is_fifo()) else: raise NotImplementedError(type) + if size is not None: + self.assertEqual(path.stat().st_size, size) for parent in path.parents: self.expected_paths.discard(parent) @@ -3315,8 +3320,15 @@ def test_parent_symlink(self): # Test interplaying symlinks # Inspired by 'dirsymlink2a' in jwilk/traversal-archives with ArchiveMaker() as arc: + + # `current` links to `.` which is both: + # - the destination directory + # - `current` itself arc.add('current', symlink_to='.') + + # effectively points to ./../ arc.add('parent', symlink_to='current/..') + arc.add('parent/evil') if support.can_symlink(): @@ -3357,9 +3369,46 @@ def test_parent_symlink(self): def test_parent_symlink2(self): # Test interplaying symlinks # Inspired by 'dirsymlink2b' in jwilk/traversal-archives + + # Posix and Windows have different pathname resolution: + # either symlink or a '..' component resolve first. + # Let's see which we are on. + if support.can_symlink(): + testpath = os.path.join(TEMPDIR, 'resolution_test') + os.mkdir(testpath) + + # testpath/current links to `.` which is all of: + # - `testpath` + # - `testpath/current` + # - `testpath/current/current` + # - etc. + os.symlink('.', os.path.join(testpath, 'current')) + + # we'll test where `testpath/current/../file` ends up + with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): + pass + + if os.path.exists(os.path.join(testpath, 'file')): + # Windows collapses 'current\..' to '.' first, leaving + # 'testpath\file' + dotdot_resolves_early = True + elif os.path.exists(os.path.join(testpath, '..', 'file')): + # Posix resolves 'current' to '.' first, leaving + # 'testpath/../file' + dotdot_resolves_early = False + else: + raise AssertionError('Could not determine link resolution') + with ArchiveMaker() as arc: + + # `current` links to `.` which is both the destination directory + # and `current` itself arc.add('current', symlink_to='.') + + # `current/parent` is also available as `./parent`, + # and effectively points to `./../` arc.add('current/parent', symlink_to='..') + arc.add('parent/evil') with self.check_context(arc.open(), 'fully_trusted'): @@ -3373,6 +3422,7 @@ def test_parent_symlink2(self): with self.check_context(arc.open(), 'tar'): if support.can_symlink(): + # Fail when extracting a file outside destination self.expect_exception( tarfile.OutsideDestinationError, "'parent/evil' would be extracted to " @@ -3383,10 +3433,24 @@ def test_parent_symlink2(self): self.expect_file('parent/evil') with self.check_context(arc.open(), 'data'): - self.expect_exception( - tarfile.LinkOutsideDestinationError, - """'current/parent' would link to ['"].*['"], """ - + "which is outside the destination") + if support.can_symlink(): + if dotdot_resolves_early: + # Fail when extracting a file outside destination + self.expect_exception( + tarfile.OutsideDestinationError, + "'parent/evil' would be extracted to " + + """['"].*evil['"], which is outside """ + + "the destination") + else: + # Fail as soon as we have a symlink outside the destination + self.expect_exception( + tarfile.LinkOutsideDestinationError, + "'current/parent' would link to " + + """['"].*outerdir['"], which is outside """ + + "the destination") + else: + self.expect_file('current/') + self.expect_file('parent/evil') def test_absolute_symlink(self): # Test symlink to an absolute path @@ -3415,11 +3479,29 @@ def test_absolute_symlink(self): with self.check_context(arc.open(), 'data'): self.expect_exception( tarfile.AbsoluteLinkError, - "'parent' is a symlink to an absolute path") + "'parent' is a link to an absolute path") + + def test_absolute_hardlink(self): + # Test hardlink to an absolute path + # Inspired by 'dirsymlink' in https://github.com/jwilk/traversal-archives + with ArchiveMaker() as arc: + arc.add('parent', hardlink_to=self.outerdir / 'foo') + + with self.check_context(arc.open(), 'fully_trusted'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'tar'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.AbsoluteLinkError, + "'parent' is a link to an absolute path") def test_sly_relative0(self): # Inspired by 'relative0' in jwilk/traversal-archives with ArchiveMaker() as arc: + # points to `../../tmp/moo` arc.add('../moo', symlink_to='..//tmp/moo') try: @@ -3469,6 +3551,54 @@ def test_sly_relative2(self): + """['"].*moo['"], which is outside the """ + "destination") + def test_deep_symlink(self): + # Test that symlinks and hardlinks inside a directory + # point to the correct file (`target` of size 3). + # If links aren't supported we get a copy of the file. + with ArchiveMaker() as arc: + arc.add('targetdir/target', size=3) + # a hardlink's linkname is relative to the archive + arc.add('linkdir/hardlink', hardlink_to=os.path.join( + 'targetdir', 'target')) + # a symlink's linkname is relative to the link's directory + arc.add('linkdir/symlink', symlink_to=os.path.join( + '..', 'targetdir', 'target')) + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + if support.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='../targetdir/target') + else: + self.expect_file('linkdir/symlink', size=3) + + def test_chains(self): + # Test chaining of symlinks/hardlinks. + # Symlinks are created before the files they point to. + with ArchiveMaker() as arc: + arc.add('linkdir/symlink', symlink_to='hardlink') + arc.add('symlink2', symlink_to=os.path.join( + 'linkdir', 'hardlink2')) + arc.add('targetdir/target', size=3) + arc.add('linkdir/hardlink', hardlink_to='targetdir/target') + arc.add('linkdir/hardlink2', hardlink_to='linkdir/symlink') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + self.expect_file('linkdir/hardlink2', size=3) + if support.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='hardlink') + self.expect_file('symlink2', size=3, + symlink_to='linkdir/hardlink2') + else: + self.expect_file('linkdir/symlink', size=3) + self.expect_file('symlink2', size=3) + def test_modes(self): # Test how file modes are extracted # (Note that the modes are ignored on platforms without working chmod) diff --git a/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst new file mode 100644 index 0000000000000..32c1fb93f4ab2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst @@ -0,0 +1,3 @@ +:func:`tarfile.data_filter` now takes the location of symlinks into account +when determining their target, so it will no longer reject some valid +tarballs with ``LinkOutsideDestinationError``. From webhook-mailer at python.org Tue Aug 22 14:28:37 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:28:37 -0000 Subject: [Python-checkins] [3.8] gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) (#108279) Message-ID: https://github.com/python/cpython/commit/b4368b5d581b9661d846feb6d9e69ce4f2580e99 commit: b4368b5d581b9661d846feb6d9e69ce4f2580e99 branch: 3.8 author: Petr Viktorin committer: ambv date: 2023-08-22T20:28:34+02:00 summary: [3.8] gh-107845: Fix symlink handling for tarfile.data_filter (GH-107846) (#108279) (cherry picked from commit acbd3f9c5c5f23e95267714e41236140d84fe962) Co-authored-by: Petr Viktorin Co-authored-by: Victor Stinner Co-authored-by: Lum?r 'Frenzy' Balhar files: A Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst M Doc/library/tarfile.rst M Lib/tarfile.py M Lib/test/test_tarfile.py diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 85d07ffd75fad..b97d6769ebd42 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -725,6 +725,11 @@ A ``TarInfo`` object has the following public data attributes: Name of the target file name, which is only present in :class:`TarInfo` objects of type :const:`LNKTYPE` and :const:`SYMTYPE`. + For symbolic links (``SYMTYPE``), the *linkname* is relative to the directory + that contains the link. + For hard links (``LNKTYPE``), the *linkname* is relative to the root of + the archive. + .. attribute:: TarInfo.uid :type: int diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 5291622ab8e97..aa06f54d2c3c5 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -740,7 +740,7 @@ def __init__(self, tarinfo): class AbsoluteLinkError(FilterError): def __init__(self, tarinfo): self.tarinfo = tarinfo - super().__init__(f'{tarinfo.name!r} is a symlink to an absolute path') + super().__init__(f'{tarinfo.name!r} is a link to an absolute path') class LinkOutsideDestinationError(FilterError): def __init__(self, tarinfo, path): @@ -800,7 +800,14 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) - target_path = os.path.realpath(os.path.join(dest_path, member.linkname)) + if member.issym(): + target_path = os.path.join(dest_path, + os.path.dirname(name), + member.linkname) + else: + target_path = os.path.join(dest_path, + member.linkname) + target_path = os.path.realpath(target_path) if os.path.commonpath([target_path, dest_path]) != dest_path: raise LinkOutsideDestinationError(member, target_path) return new_attrs diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 03be10b1fee25..8658014cab4ce 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -2984,10 +2984,12 @@ def __exit__(self, *exc): self.bio = None def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, - mode=None, **kwargs): + mode=None, size=None, **kwargs): """Add a member to the test archive. Call within `with`.""" name = str(name) tarinfo = tarfile.TarInfo(name).replace(**kwargs) + if size is not None: + tarinfo.size = size if mode: tarinfo.mode = _filemode_to_int(mode) if symlink_to is not None: @@ -3051,7 +3053,8 @@ def check_context(self, tar, filter): raise self.raised_exception self.assertEqual(self.expected_paths, set()) - def expect_file(self, name, type=None, symlink_to=None, mode=None): + def expect_file(self, name, type=None, symlink_to=None, mode=None, + size=None): """Check a single file. See check_context.""" if self.raised_exception: raise self.raised_exception @@ -3085,6 +3088,8 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None): self.assertTrue(path.is_fifo()) else: raise NotImplementedError(type) + if size is not None: + self.assertEqual(path.stat().st_size, size) for parent in path.parents: self.expected_paths.discard(parent) @@ -3130,8 +3135,15 @@ def test_parent_symlink(self): # Test interplaying symlinks # Inspired by 'dirsymlink2a' in jwilk/traversal-archives with ArchiveMaker() as arc: + + # `current` links to `.` which is both: + # - the destination directory + # - `current` itself arc.add('current', symlink_to='.') + + # effectively points to ./../ arc.add('parent', symlink_to='current/..') + arc.add('parent/evil') if support.can_symlink(): @@ -3172,9 +3184,46 @@ def test_parent_symlink(self): def test_parent_symlink2(self): # Test interplaying symlinks # Inspired by 'dirsymlink2b' in jwilk/traversal-archives + + # Posix and Windows have different pathname resolution: + # either symlink or a '..' component resolve first. + # Let's see which we are on. + if support.can_symlink(): + testpath = os.path.join(TEMPDIR, 'resolution_test') + os.mkdir(testpath) + + # testpath/current links to `.` which is all of: + # - `testpath` + # - `testpath/current` + # - `testpath/current/current` + # - etc. + os.symlink('.', os.path.join(testpath, 'current')) + + # we'll test where `testpath/current/../file` ends up + with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): + pass + + if os.path.exists(os.path.join(testpath, 'file')): + # Windows collapses 'current\..' to '.' first, leaving + # 'testpath\file' + dotdot_resolves_early = True + elif os.path.exists(os.path.join(testpath, '..', 'file')): + # Posix resolves 'current' to '.' first, leaving + # 'testpath/../file' + dotdot_resolves_early = False + else: + raise AssertionError('Could not determine link resolution') + with ArchiveMaker() as arc: + + # `current` links to `.` which is both the destination directory + # and `current` itself arc.add('current', symlink_to='.') + + # `current/parent` is also available as `./parent`, + # and effectively points to `./../` arc.add('current/parent', symlink_to='..') + arc.add('parent/evil') with self.check_context(arc.open(), 'fully_trusted'): @@ -3188,6 +3237,7 @@ def test_parent_symlink2(self): with self.check_context(arc.open(), 'tar'): if support.can_symlink(): + # Fail when extracting a file outside destination self.expect_exception( tarfile.OutsideDestinationError, "'parent/evil' would be extracted to " @@ -3198,10 +3248,24 @@ def test_parent_symlink2(self): self.expect_file('parent/evil') with self.check_context(arc.open(), 'data'): - self.expect_exception( - tarfile.LinkOutsideDestinationError, - """'current/parent' would link to ['"].*['"], """ - + "which is outside the destination") + if support.can_symlink(): + if dotdot_resolves_early: + # Fail when extracting a file outside destination + self.expect_exception( + tarfile.OutsideDestinationError, + "'parent/evil' would be extracted to " + + """['"].*evil['"], which is outside """ + + "the destination") + else: + # Fail as soon as we have a symlink outside the destination + self.expect_exception( + tarfile.LinkOutsideDestinationError, + "'current/parent' would link to " + + """['"].*outerdir['"], which is outside """ + + "the destination") + else: + self.expect_file('current/') + self.expect_file('parent/evil') def test_absolute_symlink(self): # Test symlink to an absolute path @@ -3230,11 +3294,29 @@ def test_absolute_symlink(self): with self.check_context(arc.open(), 'data'): self.expect_exception( tarfile.AbsoluteLinkError, - "'parent' is a symlink to an absolute path") + "'parent' is a link to an absolute path") + + def test_absolute_hardlink(self): + # Test hardlink to an absolute path + # Inspired by 'dirsymlink' in https://github.com/jwilk/traversal-archives + with ArchiveMaker() as arc: + arc.add('parent', hardlink_to=self.outerdir / 'foo') + + with self.check_context(arc.open(), 'fully_trusted'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'tar'): + self.expect_exception(KeyError, ".*foo. not found") + + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.AbsoluteLinkError, + "'parent' is a link to an absolute path") def test_sly_relative0(self): # Inspired by 'relative0' in jwilk/traversal-archives with ArchiveMaker() as arc: + # points to `../../tmp/moo` arc.add('../moo', symlink_to='..//tmp/moo') try: @@ -3284,6 +3366,54 @@ def test_sly_relative2(self): + """['"].*moo['"], which is outside the """ + "destination") + def test_deep_symlink(self): + # Test that symlinks and hardlinks inside a directory + # point to the correct file (`target` of size 3). + # If links aren't supported we get a copy of the file. + with ArchiveMaker() as arc: + arc.add('targetdir/target', size=3) + # a hardlink's linkname is relative to the archive + arc.add('linkdir/hardlink', hardlink_to=os.path.join( + 'targetdir', 'target')) + # a symlink's linkname is relative to the link's directory + arc.add('linkdir/symlink', symlink_to=os.path.join( + '..', 'targetdir', 'target')) + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + if support.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='../targetdir/target') + else: + self.expect_file('linkdir/symlink', size=3) + + def test_chains(self): + # Test chaining of symlinks/hardlinks. + # Symlinks are created before the files they point to. + with ArchiveMaker() as arc: + arc.add('linkdir/symlink', symlink_to='hardlink') + arc.add('symlink2', symlink_to=os.path.join( + 'linkdir', 'hardlink2')) + arc.add('targetdir/target', size=3) + arc.add('linkdir/hardlink', hardlink_to='targetdir/target') + arc.add('linkdir/hardlink2', hardlink_to='linkdir/symlink') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + self.expect_file('targetdir/target', size=3) + self.expect_file('linkdir/hardlink', size=3) + self.expect_file('linkdir/hardlink2', size=3) + if support.can_symlink(): + self.expect_file('linkdir/symlink', size=3, + symlink_to='hardlink') + self.expect_file('symlink2', size=3, + symlink_to='linkdir/hardlink2') + else: + self.expect_file('linkdir/symlink', size=3) + self.expect_file('symlink2', size=3) + def test_modes(self): # Test how file modes are extracted # (Note that the modes are ignored on platforms without working chmod) diff --git a/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst new file mode 100644 index 0000000000000..32c1fb93f4ab2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-10-17-36-22.gh-issue-107845.dABiMJ.rst @@ -0,0 +1,3 @@ +:func:`tarfile.data_filter` now takes the location of symlinks into account +when determining their target, so it will no longer reject some valid +tarballs with ``LinkOutsideDestinationError``. From webhook-mailer at python.org Tue Aug 22 14:29:01 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:29:01 -0000 Subject: [Python-checkins] [3.9] gh-107565: Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, and 3.1.2. (#108123) Message-ID: https://github.com/python/cpython/commit/d31ae21e5d0e7ec7cd2416176e36b3c135469b0f commit: d31ae21e5d0e7ec7cd2416176e36b3c135469b0f branch: 3.9 author: Ned Deily committer: ambv date: 2023-08-22T20:28:57+02:00 summary: [3.9] gh-107565: Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, and 3.1.2. (#108123) [3.9] gh-107565: Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, and 3.1.2. (cherry picked from commit 441797d4ffb12acda257370b9e5e19ed8d6e8a71) files: A Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst M .github/workflows/build.yml M Tools/ssl/multissltests.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 37fd2ee863f72..9a2d24a88300f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -178,7 +178,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 1.1.1v PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout at v3 @@ -220,7 +220,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.0.2u, 1.1.0l, 1.1.1u, 3.0.9, 3.1.1] + openssl_ver: [1.0.2u, 1.1.0l, 1.1.1v, 3.0.10, 3.1.2] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst b/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst new file mode 100644 index 0000000000000..c43ee680e8158 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2023-08-12-13-18-15.gh-issue-107565.Tv22Ne.rst @@ -0,0 +1,2 @@ +Update multissltests and GitHub CI workflows to use OpenSSL 1.1.1v, 3.0.10, +and 3.1.2. diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index b704aed930b7e..921f0f5cd5f41 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -49,8 +49,8 @@ ] OPENSSL_RECENT_VERSIONS = [ - "1.1.1u", - "3.0.9", + "1.1.1v", + "3.0.10", ] LIBRESSL_OLD_VERSIONS = [ From webhook-mailer at python.org Tue Aug 22 14:30:23 2023 From: webhook-mailer at python.org (vstinner) Date: Tue, 22 Aug 2023 18:30:23 -0000 Subject: [Python-checkins] gh-108303: Add Lib/test/test_cppext/ sub-directory (#108325) Message-ID: https://github.com/python/cpython/commit/21dda09600848ac280481f7c64f8d9516dc69bb2 commit: 21dda09600848ac280481f7c64f8d9516dc69bb2 branch: main author: Victor Stinner committer: vstinner date: 2023-08-22T18:30:18Z summary: gh-108303: Add Lib/test/test_cppext/ sub-directory (#108325) * Move test_cppext to its own directory * Rename setup_testcppext.py to setup.py * Rename _testcppext.cpp to extension.cpp * The source (extension.cpp) is now also copied by the test. files: A Lib/test/test_cppext/__init__.py A Lib/test/test_cppext/extension.cpp A Lib/test/test_cppext/setup.py D Lib/test/_testcppext.cpp D Lib/test/setup_testcppext.py D Lib/test/test_cppext.py M Makefile.pre.in diff --git a/Lib/test/test_cppext.py b/Lib/test/test_cppext/__init__.py similarity index 91% rename from Lib/test/test_cppext.py rename to Lib/test/test_cppext/__init__.py index 7a143f4464013..8adff91c38efc 100644 --- a/Lib/test/test_cppext.py +++ b/Lib/test/test_cppext/__init__.py @@ -10,9 +10,8 @@ MS_WINDOWS = (sys.platform == 'win32') - - -SETUP_TESTCPPEXT = support.findfile('setup_testcppext.py') +SOURCE = os.path.join(os.path.dirname(__file__), 'extension.cpp') +SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') @support.requires_subprocess() @@ -41,7 +40,8 @@ def check_build(self, std_cpp03, extension_name): def _check_build(self, std_cpp03, extension_name, python_exe): pkg_dir = 'pkg' os.mkdir(pkg_dir) - shutil.copy(SETUP_TESTCPPEXT, os.path.join(pkg_dir, "setup.py")) + shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) + shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE))) def run_cmd(operation, cmd): env = os.environ.copy() diff --git a/Lib/test/_testcppext.cpp b/Lib/test/test_cppext/extension.cpp similarity index 99% rename from Lib/test/_testcppext.cpp rename to Lib/test/test_cppext/extension.cpp index 82b471312dd2b..90669b10cb2c6 100644 --- a/Lib/test/_testcppext.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -1,5 +1,7 @@ // gh-91321: Very basic C++ test extension to check that the Python C API is // compatible with C++ and does not emit C++ compiler warnings. +// +// The code is only built, not executed. // Always enable assertions #undef NDEBUG diff --git a/Lib/test/setup_testcppext.py b/Lib/test/test_cppext/setup.py similarity index 93% rename from Lib/test/setup_testcppext.py rename to Lib/test/test_cppext/setup.py index 22fe750085fd7..6867094a42043 100644 --- a/Lib/test/setup_testcppext.py +++ b/Lib/test/test_cppext/setup.py @@ -2,7 +2,6 @@ # compatible with C++ and does not emit C++ compiler warnings. import os import sys -from test import support from setuptools import setup, Extension @@ -10,7 +9,7 @@ MS_WINDOWS = (sys.platform == 'win32') -SOURCE = support.findfile('_testcppext.cpp') +SOURCE = 'extension.cpp' if not MS_WINDOWS: # C++ compiler flags for GCC and clang CPPFLAGS = [ diff --git a/Makefile.pre.in b/Makefile.pre.in index beccab45a235a..d66764e616540 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2156,6 +2156,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/support/_hypothesis_stubs \ test/test_asyncio \ test/test_capi \ + test/test_cppext \ test/test_ctypes \ test/test_email \ test/test_email/data \ From webhook-mailer at python.org Tue Aug 22 14:32:20 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:32:20 -0000 Subject: [Python-checkins] [3.11] Resolve reference warnings in faq/gui.rst (GH-108147) (#108194) Message-ID: https://github.com/python/cpython/commit/3a39e151799d4882bb73ef216705c3e71b67ca00 commit: 3a39e151799d4882bb73ef216705c3e71b67ca00 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2023-08-22T20:32:16+02:00 summary: [3.11] Resolve reference warnings in faq/gui.rst (GH-108147) (#108194) (cherry picked from commit 8f3d09bf5d16b508fece5420a22abe6f0c1f00b7) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> Co-authored-by: Hugo van Kemenade files: M Doc/faq/gui.rst M Doc/tools/.nitignore diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index 023ffdf0db510..77c0a27a10c9a 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -43,7 +43,7 @@ applications, the applications will not be truly stand-alone, as the application will still need the Tcl and Tk libraries. One solution is to ship the application with the Tcl and Tk libraries, and point -to them at run-time using the :envvar:`TCL_LIBRARY` and :envvar:`TK_LIBRARY` +to them at run-time using the :envvar:`!TCL_LIBRARY` and :envvar:`!TK_LIBRARY` environment variables. To get truly stand-alone applications, the Tcl scripts that form the library @@ -62,7 +62,7 @@ Can I have Tk events handled while waiting for I/O? On platforms other than Windows, yes, and you don't even need threads! But you'll have to restructure your I/O -code a bit. Tk has the equivalent of Xt's :c:func:`XtAddInput()` call, which allows you +code a bit. Tk has the equivalent of Xt's :c:func:`!XtAddInput` call, which allows you to register a callback function which will be called from the Tk mainloop when I/O is possible on a file descriptor. See :ref:`tkinter-file-handlers`. @@ -70,8 +70,9 @@ I/O is possible on a file descriptor. See :ref:`tkinter-file-handlers`. I can't get key bindings to work in Tkinter: why? ------------------------------------------------- -An often-heard complaint is that event handlers bound to events with the -:meth:`bind` method don't get handled even when the appropriate key is pressed. +An often-heard complaint is that event handlers :ref:`bound ` +to events with the :meth:`!bind` method +don't get handled even when the appropriate key is pressed. The most common cause is that the widget to which the binding applies doesn't have "keyboard focus". Check out the Tk documentation for the focus command. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 8c44199cb861d..c8f3080cb6edd 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -24,7 +24,6 @@ Doc/c-api/typeobj.rst Doc/c-api/unicode.rst Doc/extending/extending.rst Doc/extending/newtypes.rst -Doc/faq/gui.rst Doc/glossary.rst Doc/howto/descriptor.rst Doc/howto/enum.rst From webhook-mailer at python.org Tue Aug 22 14:32:52 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:32:52 -0000 Subject: [Python-checkins] [3.11] gh-107298: Fix C API datetime documentation (GH-108034) (#108233) Message-ID: https://github.com/python/cpython/commit/80c7ae00f10a374e1d79d1c2719056951fdd3421 commit: 80c7ae00f10a374e1d79d1c2719056951fdd3421 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2023-08-22T20:32:48+02:00 summary: [3.11] gh-107298: Fix C API datetime documentation (GH-108034) (#108233) (cherry picked from commit d63972e289e05b0d82e59f32f107312a8b3de7b5) Co-authored-by: Serhiy Storchaka files: M Doc/c-api/datetime.rst M Doc/tools/.nitignore diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index 72fc07afbf1f4..97522da773477 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -8,11 +8,54 @@ DateTime Objects Various date and time objects are supplied by the :mod:`datetime` module. Before using any of these functions, the header file :file:`datetime.h` must be included in your source (note that this is not included by :file:`Python.h`), -and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of +and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of the module initialisation function. The macro puts a pointer to a C structure -into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following +into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following macros. +.. c:type:: PyDateTime_Date + + This subtype of :c:type:`PyObject` represents a Python date object. + +.. c:type:: PyDateTime_DateTime + + This subtype of :c:type:`PyObject` represents a Python datetime object. + +.. c:type:: PyDateTime_Time + + This subtype of :c:type:`PyObject` represents a Python time object. + +.. c:type:: PyDateTime_Delta + + This subtype of :c:type:`PyObject` represents the difference between two datetime values. + +.. c:var:: PyTypeObject PyDateTime_DateType + + This instance of :c:type:`PyTypeObject` represents the Python date type; + it is the same object as :class:`datetime.date` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_DateTimeType + + This instance of :c:type:`PyTypeObject` represents the Python datetime type; + it is the same object as :class:`datetime.datetime` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_TimeType + + This instance of :c:type:`PyTypeObject` represents the Python time type; + it is the same object as :class:`datetime.time` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_DeltaType + + This instance of :c:type:`PyTypeObject` represents Python type for + the difference between two datetime values; + it is the same object as :class:`datetime.timedelta` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_TZInfoType + + This instance of :c:type:`PyTypeObject` represents the Python time zone info type; + it is the same object as :class:`datetime.tzinfo` in the Python layer. + + Macro for access to the UTC singleton: .. c:var:: PyObject* PyDateTime_TimeZone_UTC @@ -28,7 +71,7 @@ Type-check macros: .. c:function:: int PyDate_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateType` or a subtype of - :c:data:`PyDateTime_DateType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DateType`. *ob* must not be ``NULL``. This function always succeeds. @@ -41,7 +84,7 @@ Type-check macros: .. c:function:: int PyDateTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType` or a subtype of - :c:data:`PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always succeeds. @@ -54,7 +97,7 @@ Type-check macros: .. c:function:: int PyTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TimeType` or a subtype of - :c:data:`PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always succeeds. @@ -67,7 +110,7 @@ Type-check macros: .. c:function:: int PyDelta_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DeltaType` or a subtype of - :c:data:`PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always succeeds. @@ -80,7 +123,7 @@ Type-check macros: .. c:function:: int PyTZInfo_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType` or a subtype of - :c:data:`PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always succeeds. @@ -133,7 +176,7 @@ Macros to create objects: :class:`datetime.timedelta` objects. -.. c:function:: PyObject* PyTimeZone_FromOffset(PyDateTime_DeltaType* offset) +.. c:function:: PyObject* PyTimeZone_FromOffset(PyObject *offset) Return a :class:`datetime.timezone` object with an unnamed fixed offset represented by the *offset* argument. @@ -141,7 +184,7 @@ Macros to create objects: .. versionadded:: 3.7 -.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyDateTime_DeltaType* offset, PyUnicode* name) +.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyObject *offset, PyObject *name) Return a :class:`datetime.timezone` object with a fixed offset represented by the *offset* argument and with tzname *name*. @@ -150,8 +193,8 @@ Macros to create objects: Macros to extract fields from date objects. The argument must be an instance of -:c:data:`PyDateTime_Date`, including subclasses (such as -:c:data:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is +:c:type:`PyDateTime_Date`, including subclasses (such as +:c:type:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_GET_YEAR(PyDateTime_Date *o) @@ -170,7 +213,7 @@ not checked: Macros to extract fields from datetime objects. The argument must be an -instance of :c:data:`PyDateTime_DateTime`, including subclasses. The argument +instance of :c:type:`PyDateTime_DateTime`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DATE_GET_HOUR(PyDateTime_DateTime *o) @@ -208,7 +251,7 @@ must not be ``NULL``, and the type is not checked: Macros to extract fields from time objects. The argument must be an instance of -:c:data:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``, +:c:type:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_TIME_GET_HOUR(PyDateTime_Time *o) @@ -246,7 +289,7 @@ and the type is not checked: Macros to extract fields from time delta objects. The argument must be an -instance of :c:data:`PyDateTime_Delta`, including subclasses. The argument must +instance of :c:type:`PyDateTime_Delta`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DELTA_GET_DAYS(PyDateTime_Delta *o) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index c8f3080cb6edd..4021f97e34f7f 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -2,7 +2,6 @@ # as tested on the CI via check-warnings.py in reusable-docs.yml. # Keep lines sorted lexicographically to help avoid merge conflicts. -Doc/c-api/datetime.rst Doc/c-api/descriptor.rst Doc/c-api/exceptions.rst Doc/c-api/file.rst From webhook-mailer at python.org Tue Aug 22 14:33:11 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:33:11 -0000 Subject: [Python-checkins] [3.11] gh-107298: Fix numerous ref errors and typos in the C API docs (GH-108258) (#108290) Message-ID: https://github.com/python/cpython/commit/3ca9264aba054672e8a33c2e0d9cf32cb4e17a86 commit: 3ca9264aba054672e8a33c2e0d9cf32cb4e17a86 branch: 3.11 author: Serhiy Storchaka committer: ambv date: 2023-08-22T20:33:07+02:00 summary: [3.11] gh-107298: Fix numerous ref errors and typos in the C API docs (GH-108258) (#108290) (cherry picked from commit d7202e4879bf4e7e00a69500ddcb3143864139b4) files: M Doc/c-api/exceptions.rst M Doc/c-api/init_config.rst M Doc/c-api/module.rst M Doc/c-api/unicode.rst M Doc/extending/newtypes.rst M Doc/tools/.nitignore M Doc/whatsnew/2.2.rst M Doc/whatsnew/2.3.rst M Doc/whatsnew/2.4.rst M Doc/whatsnew/2.5.rst M Doc/whatsnew/2.6.rst M Doc/whatsnew/2.7.rst M Doc/whatsnew/3.0.rst M Doc/whatsnew/3.1.rst M Doc/whatsnew/3.10.rst M Doc/whatsnew/3.11.rst M Doc/whatsnew/3.2.rst M Doc/whatsnew/3.3.rst M Doc/whatsnew/3.5.rst M Doc/whatsnew/3.8.rst M Doc/whatsnew/3.9.rst M Misc/NEWS.d/3.11.0a1.rst M Misc/NEWS.d/3.8.0a1.rst M Misc/NEWS.d/3.8.0a4.rst M Misc/NEWS.d/3.8.0b1.rst M Misc/NEWS.d/3.9.0a1.rst M Misc/NEWS.d/3.9.0a5.rst diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index ae45aaef238fe..82050845237bb 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -211,17 +211,21 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the - filename is given as a C string. *filename* is decoded from the filesystem - encoding (:func:`os.fsdecode`). + Similar to :c:func:`PyErr_SetFromWindowsErr`, with the additional behavior + that if *filename* is not ``NULL``, it is decoded from the filesystem + encoding (:func:`os.fsdecode`) and passed to the constructor of + :exc:`OSError` as a third parameter to be used to define the + :attr:`!filename` attribute of the exception instance. .. availability:: Windows. .. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an - additional parameter specifying the exception type to be raised. + Similar to :c:func:`PyErr_SetExcFromWindowsErr`, with the additional behavior + that if *filename* is not ``NULL``, it is passed to the constructor of + :exc:`OSError` as a third parameter to be used to define the + :attr:`!filename` attribute of the exception instance. .. availability:: Windows. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index c6798d68b0e8a..ad1cd3f4f0668 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -82,6 +82,8 @@ PyWideStringList If *length* is non-zero, *items* must be non-``NULL`` and all strings must be non-``NULL``. + .. c:namespace:: NULL + Methods: .. c:function:: PyStatus PyWideStringList_Append(PyWideStringList *list, const wchar_t *item) @@ -101,6 +103,8 @@ PyWideStringList Python must be preinitialized to call this function. + .. c:namespace:: PyWideStringList + Structure fields: .. c:member:: Py_ssize_t length diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 6ef4eea6a07f7..71a079a8e67f4 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -338,6 +338,7 @@ The available slot types are: The *value* pointer of this slot must point to a function of the signature: .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) + :noindex: The function receives a :py:class:`~importlib.machinery.ModuleSpec` instance, as defined in :PEP:`451`, and the module definition. @@ -372,6 +373,7 @@ The available slot types are: The signature of the function is: .. c:function:: int exec_module(PyObject* module) + :noindex: If multiple ``Py_mod_exec`` slots are specified, they are processed in the order they appear in the *m_slots* array. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 3d6b4c49fa9c8..146511b9e3658 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1391,7 +1391,7 @@ the user settings on the machine running the codec. Encode the Unicode object using the specified code page and return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. Use - :c:macro:`CP_ACP` code page to get the MBCS encoder. + :c:macro:`!CP_ACP` code page to get the MBCS encoder. .. versionadded:: 3.3 diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index c9d2301d1470c..8ea76aec8290d 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -323,7 +323,7 @@ have an associated doc string simply by providing the text in the table. An application can use the introspection API to retrieve the descriptor from the class object, and get the doc string using its :attr:`__doc__` attribute. -As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.name` value +As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.ml_name` value of ``NULL`` is required. .. XXX Descriptors need to be explained in more detail somewhere, but not here. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 4021f97e34f7f..42f1cc052d68a 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -20,7 +20,6 @@ Doc/c-api/structures.rst Doc/c-api/sys.rst Doc/c-api/type.rst Doc/c-api/typeobj.rst -Doc/c-api/unicode.rst Doc/extending/extending.rst Doc/extending/newtypes.rst Doc/glossary.rst diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 7de48a4026303..d9ead57413cbb 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -1078,17 +1078,17 @@ code, none of the changes described here will affect you very much. To upgrade an extension module to the new API, perform the following steps: -* Rename :c:func:`Py_TPFLAGS_GC` to :c:func:`PyTPFLAGS_HAVE_GC`. +* Rename :c:macro:`!Py_TPFLAGS_GC` to :c:macro:`Py_TPFLAGS_HAVE_GC`. * Use :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar` to allocate objects, and :c:func:`PyObject_GC_Del` to deallocate them. -* Rename :c:func:`PyObject_GC_Init` to :c:func:`PyObject_GC_Track` and - :c:func:`PyObject_GC_Fini` to :c:func:`PyObject_GC_UnTrack`. +* Rename :c:func:`!PyObject_GC_Init` to :c:func:`PyObject_GC_Track` and + :c:func:`!PyObject_GC_Fini` to :c:func:`PyObject_GC_UnTrack`. -* Remove :c:func:`PyGC_HEAD_SIZE` from object size calculations. +* Remove :c:macro:`!PyGC_HEAD_SIZE` from object size calculations. -* Remove calls to :c:func:`PyObject_AS_GC` and :c:func:`PyObject_FROM_GC`. +* Remove calls to :c:func:`!PyObject_AS_GC` and :c:func:`!PyObject_FROM_GC`. * A new ``et`` format sequence was added to :c:func:`PyArg_ParseTuple`; ``et`` takes both a parameter and an encoding name, and converts the parameter to the @@ -1219,7 +1219,7 @@ Some of the more notable changes are: operator, but these features were rarely used and therefore buggy. The :meth:`tolist` method and the :attr:`start`, :attr:`stop`, and :attr:`step` attributes are also being deprecated. At the C level, the fourth argument to - the :c:func:`PyRange_New` function, ``repeat``, has also been deprecated. + the :c:func:`!PyRange_New` function, ``repeat``, has also been deprecated. * There were a bunch of patches to the dictionary implementation, mostly to fix potential core dumps if a dictionary contains objects that sneakily changed diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index 4390f245d3ab4..72aef3d269f8d 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1897,7 +1897,7 @@ Changes to Python's build process and to the C API include: but will also mean that you can't get help for Python's built-ins. (Contributed by Gustavo Niemeyer.) -* The :c:func:`PyArg_NoArgs` macro is now deprecated, and code that uses it +* The :c:func:`!PyArg_NoArgs` macro is now deprecated, and code that uses it should be changed. For Python 2.2 and later, the method definition table can specify the :c:macro:`METH_NOARGS` flag, signalling that there are no arguments, and the argument checking can then be removed. If compatibility with pre-2.2 diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 4272928fb6789..ceda03afd5f4f 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -1468,7 +1468,7 @@ Some of the changes to Python's build process and to the C API are: *X* is a NaN. (Contributed by Tim Peters.) * C code can avoid unnecessary locking by using the new - :c:func:`PyEval_ThreadsInitialized` function to tell if any thread operations + :c:func:`!PyEval_ThreadsInitialized` function to tell if any thread operations have been performed. If this function returns false, no lock operations are needed. (Contributed by Nick Coghlan.) diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index a15eefc2953f4..f58b3ede27fac 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -2119,9 +2119,9 @@ Changes to Python's build process and to the C API include: the various AST nodes in :file:`Parser/Python.asdl`. A Python script reads this file and generates a set of C structure definitions in :file:`Include/Python-ast.h`. The :c:func:`PyParser_ASTFromString` and - :c:func:`PyParser_ASTFromFile`, defined in :file:`Include/pythonrun.h`, take + :c:func:`!PyParser_ASTFromFile`, defined in :file:`Include/pythonrun.h`, take Python source as input and return the root of an AST representing the contents. - This AST can then be turned into a code object by :c:func:`PyAST_Compile`. For + This AST can then be turned into a code object by :c:func:`!PyAST_Compile`. For more information, read the source code, and then ask questions on python-dev. The AST code was developed under Jeremy Hylton's management, and implemented by @@ -2172,7 +2172,7 @@ Changes to Python's build process and to the C API include: ``Py_LOCAL(type)`` declares the function as returning a value of the specified *type* and uses a fast-calling qualifier. ``Py_LOCAL_INLINE(type)`` does the same thing and also requests the - function be inlined. If :c:func:`PY_LOCAL_AGGRESSIVE` is defined before + function be inlined. If macro :c:macro:`!PY_LOCAL_AGGRESSIVE` is defined before :file:`python.h` is included, a set of more aggressive optimizations are enabled for the module; you should benchmark the results to find out if these optimizations actually make the code faster. (Contributed by Fredrik Lundh at @@ -2181,7 +2181,7 @@ Changes to Python's build process and to the C API include: * ``PyErr_NewException(name, base, dict)`` can now accept a tuple of base classes as its *base* argument. (Contributed by Georg Brandl.) -* The :c:func:`PyErr_Warn` function for issuing warnings is now deprecated in +* The :c:func:`!PyErr_Warn` function for issuing warnings is now deprecated in favour of ``PyErr_WarnEx(category, message, stacklevel)`` which lets you specify the number of stack frames separating this function and the caller. A *stacklevel* of 1 is the function calling :c:func:`PyErr_WarnEx`, 2 is the @@ -2191,7 +2191,7 @@ Changes to Python's build process and to the C API include: compiled with a C++ compiler without errors. (Implemented by Anthony Baxter, Martin von L?wis, Skip Montanaro.) -* The :c:func:`PyRange_New` function was removed. It was never documented, never +* The :c:func:`!PyRange_New` function was removed. It was never documented, never used in the core code, and had dangerously lax error checking. In the unlikely case that your extensions were using it, you can replace it by something like the following:: diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 563fa461d2a86..b975aea7d6894 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -977,7 +977,7 @@ can be used to include Unicode characters:: print len(s) # 12 Unicode characters At the C level, Python 3.0 will rename the existing 8-bit -string type, called :c:type:`PyStringObject` in Python 2.x, +string type, called :c:type:`!PyStringObject` in Python 2.x, to :c:type:`PyBytesObject`. Python 2.6 uses ``#define`` to support using the names :c:func:`PyBytesObject`, :c:func:`PyBytes_Check`, :c:func:`PyBytes_FromStringAndSize`, @@ -3012,11 +3012,11 @@ Changes to Python's build process and to the C API include: bug occurred if one thread closed a file object while another thread was reading from or writing to the object. In 2.6 file objects have a reference count, manipulated by the - :c:func:`PyFile_IncUseCount` and :c:func:`PyFile_DecUseCount` + :c:func:`!PyFile_IncUseCount` and :c:func:`!PyFile_DecUseCount` functions. File objects can't be closed unless the reference count - is zero. :c:func:`PyFile_IncUseCount` should be called while the GIL + is zero. :c:func:`!PyFile_IncUseCount` should be called while the GIL is still held, before carrying out an I/O operation using the - ``FILE *`` pointer, and :c:func:`PyFile_DecUseCount` should be called + ``FILE *`` pointer, and :c:func:`!PyFile_DecUseCount` should be called immediately after the GIL is re-acquired. (Contributed by Antoine Pitrou and Gregory P. Smith.) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index bc62a2155f1b6..787fe8468b50d 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2151,7 +2151,7 @@ Changes to Python's build process and to the C API include: * New function: stemming from the rewrite of string-to-float conversion, a new :c:func:`PyOS_string_to_double` function was added. The old - :c:func:`PyOS_ascii_strtod` and :c:func:`PyOS_ascii_atof` functions + :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof` functions are now deprecated. * New function: :c:func:`PySys_SetArgvEx` sets the value of @@ -2194,13 +2194,13 @@ Changes to Python's build process and to the C API include: .. XXX these macros don't seem to be described in the c-api docs. -* Removed function: :c:macro:`PyEval_CallObject` is now only available +* Removed function: :c:func:`!PyEval_CallObject` is now only available as a macro. A function version was being kept around to preserve ABI linking compatibility, but that was in 1997; it can certainly be deleted by now. (Removed by Antoine Pitrou; :issue:`8276`.) -* New format codes: the :c:func:`PyFormat_FromString`, - :c:func:`PyFormat_FromStringV`, and :c:func:`PyErr_Format` functions now +* New format codes: the :c:func:`!PyString_FromFormat`, + :c:func:`!PyString_FromFormatV`, and :c:func:`PyErr_Format` functions now accept ``%lld`` and ``%llu`` format codes for displaying C's :c:expr:`long long` types. (Contributed by Mark Dickinson; :issue:`7228`.) @@ -2539,7 +2539,7 @@ For C extensions: instead of triggering a :exc:`DeprecationWarning` (:issue:`5080`). * Use the new :c:func:`PyOS_string_to_double` function instead of the old - :c:func:`PyOS_ascii_strtod` and :c:func:`PyOS_ascii_atof` functions, + :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof` functions, which are now deprecated. For applications that embed Python: diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 65fcc071a5cf5..4089f4cb4866b 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -865,8 +865,8 @@ to the C API. * No more C API support for restricted execution. -* :c:func:`PyNumber_Coerce`, :c:func:`PyNumber_CoerceEx`, - :c:func:`PyMember_Get`, and :c:func:`PyMember_Set` C APIs are removed. +* :c:func:`!PyNumber_Coerce`, :c:func:`!PyNumber_CoerceEx`, + :c:func:`!PyMember_Get`, and :c:func:`!PyMember_Set` C APIs are removed. * New C API :c:func:`PyImport_ImportModuleNoBlock`, works like :c:func:`PyImport_ImportModule` but won't block on the import lock diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index ceef622f9f8da..cc6619c53cd62 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -501,12 +501,12 @@ Changes to Python's build process and to the C API include: (Contributed by Mark Dickinson and Lisandro Dalcrin; :issue:`5175`.) -* Deprecated :c:func:`PyNumber_Int`. Use :c:func:`PyNumber_Long` instead. +* Deprecated :c:func:`!PyNumber_Int`. Use :c:func:`PyNumber_Long` instead. (Contributed by Mark Dickinson; :issue:`4910`.) * Added a new :c:func:`PyOS_string_to_double` function to replace the - deprecated functions :c:func:`PyOS_ascii_strtod` and :c:func:`PyOS_ascii_atof`. + deprecated functions :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof`. (Contributed by Mark Dickinson; :issue:`5914`.) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 50b85174ffdef..9213c9c3e45e1 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1818,8 +1818,8 @@ Removed into their code. (Contributed by Dong-hee Na and Terry J. Reedy in :issue:`42299`.) -* Removed the :c:func:`PyModule_GetWarningsModule` function that was useless - now due to the _warnings module was converted to a builtin module in 2.6. +* Removed the :c:func:`!PyModule_GetWarningsModule` function that was useless + now due to the :mod:`!_warnings` module was converted to a builtin module in 2.6. (Contributed by Hai Shi in :issue:`42599`.) * Remove deprecated aliases to :ref:`collections-abstract-base-classes` from diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 9054452fe12d5..5de165a566547 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2216,7 +2216,7 @@ New Features * :c:func:`PyBuffer_SizeFromFormat` * :c:func:`PyBuffer_ToContiguous` * :c:func:`PyBuffer_FromContiguous` - * :c:func:`PyBuffer_CopyData` + * :c:func:`PyObject_CopyData` * :c:func:`PyBuffer_IsContiguous` * :c:func:`PyBuffer_FillContiguousStrides` * :c:func:`PyBuffer_FillInfo` @@ -2562,18 +2562,18 @@ Deprecated * Deprecate the following functions to configure the Python initialization: - * :c:func:`PySys_AddWarnOptionUnicode` - * :c:func:`PySys_AddWarnOption` - * :c:func:`PySys_AddXOption` - * :c:func:`PySys_HasWarnOptions` - * :c:func:`PySys_SetArgvEx` - * :c:func:`PySys_SetArgv` - * :c:func:`PySys_SetPath` - * :c:func:`Py_SetPath` - * :c:func:`Py_SetProgramName` - * :c:func:`Py_SetPythonHome` - * :c:func:`Py_SetStandardStreamEncoding` - * :c:func:`_Py_SetProgramFullPath` + * :c:func:`!PySys_AddWarnOptionUnicode` + * :c:func:`!PySys_AddWarnOption` + * :c:func:`!PySys_AddXOption` + * :c:func:`!PySys_HasWarnOptions` + * :c:func:`!PySys_SetArgvEx` + * :c:func:`!PySys_SetArgv` + * :c:func:`!PySys_SetPath` + * :c:func:`!Py_SetPath` + * :c:func:`!Py_SetProgramName` + * :c:func:`!Py_SetPythonHome` + * :c:func:`!Py_SetStandardStreamEncoding` + * :c:func:`!_Py_SetProgramFullPath` Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization Configuration ` instead (:pep:`587`). diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index de355c19ac0e4..f372b6be05fca 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -2567,7 +2567,7 @@ Changes to Python's build process and to the C API include: to set :data:`sys.argv` without also modifying :data:`sys.path` (:issue:`5753`). -* :c:macro:`PyEval_CallObject` is now only available in macro form. The +* :c:func:`!PyEval_CallObject` is now only available in macro form. The function declaration, which was kept for backwards compatibility reasons, is now removed -- the macro was introduced in 1997 (:issue:`8276`). @@ -2729,15 +2729,15 @@ require changes to your code: (Contributed by Antoine Pitrou, :issue:`10272`.) -* The misleading functions :c:func:`PyEval_AcquireLock()` and - :c:func:`PyEval_ReleaseLock()` have been officially deprecated. The - thread-state aware APIs (such as :c:func:`PyEval_SaveThread()` - and :c:func:`PyEval_RestoreThread()`) should be used instead. +* The misleading functions :c:func:`!PyEval_AcquireLock` and + :c:func:`!PyEval_ReleaseLock` have been officially deprecated. The + thread-state aware APIs (such as :c:func:`PyEval_SaveThread` + and :c:func:`PyEval_RestoreThread`) should be used instead. * Due to security risks, :func:`asyncore.handle_accept` has been deprecated, and a new function, :func:`asyncore.handle_accepted`, was added to replace it. (Contributed by Giampaolo Rodola in :issue:`6706`.) -* Due to the new :term:`GIL` implementation, :c:func:`PyEval_InitThreads()` - cannot be called before :c:func:`Py_Initialize()` anymore. +* Due to the new :term:`GIL` implementation, :c:func:`!PyEval_InitThreads` + cannot be called before :c:func:`Py_Initialize` anymore. diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 91ffc99325a01..e0aa82dcdb8df 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -2303,7 +2303,7 @@ Functions and macros manipulating Py_UNICODE* strings: Encoders: -* :c:func:`!PyUnicode_Encode`: use :c:func:`PyUnicode_AsEncodedObject` +* :c:func:`!PyUnicode_Encode`: use :c:func:`!PyUnicode_AsEncodedObject` * :c:func:`!PyUnicode_EncodeUTF7` * :c:func:`!PyUnicode_EncodeUTF8`: use :c:func:`PyUnicode_AsUTF8` or :c:func:`PyUnicode_AsUTF8String` @@ -2461,7 +2461,7 @@ Porting C code -------------- * In the course of changes to the buffer API the undocumented - :c:member:`~Py_buffer.smalltable` member of the + :c:member:`!smalltable` member of the :c:type:`Py_buffer` structure has been removed and the layout of the :c:type:`PyMemoryViewObject` has changed. diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 9bd97e4726ca2..6d911096c1739 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -2512,7 +2512,7 @@ Changes in the Python API Changes in the C API -------------------- -* The undocumented :c:member:`~PyMemoryViewObject.format` member of the +* The undocumented :c:member:`!format` member of the (non-public) :c:type:`PyMemoryViewObject` structure has been removed. All extensions relying on the relevant parts in ``memoryobject.h`` must be rebuilt. @@ -2520,7 +2520,7 @@ Changes in the C API * The :c:type:`PyMemAllocator` structure was renamed to :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. -* Removed non-documented macro :c:macro:`PyObject_REPR` which leaked references. +* Removed non-documented macro :c:macro:`!PyObject_REPR()` which leaked references. Use format character ``%R`` in :c:func:`PyUnicode_FromFormat`-like functions to format the :func:`repr` of the object. (Contributed by Serhiy Storchaka in :issue:`22453`.) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index b3a6f114dd444..1d2f0a6ff3ca5 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -1571,12 +1571,12 @@ Build and C API Changes * :c:func:`Py_INCREF`, :c:func:`Py_DECREF` * :c:func:`Py_XINCREF`, :c:func:`Py_XDECREF` * :c:func:`PyObject_INIT`, :c:func:`PyObject_INIT_VAR` - * Private functions: :c:func:`_PyObject_GC_TRACK`, - :c:func:`_PyObject_GC_UNTRACK`, :c:func:`_Py_Dealloc` + * Private functions: :c:func:`!_PyObject_GC_TRACK`, + :c:func:`!_PyObject_GC_UNTRACK`, :c:func:`!_Py_Dealloc` (Contributed by Victor Stinner in :issue:`35059`.) -* The :c:func:`PyByteArray_Init` and :c:func:`PyByteArray_Fini` functions have +* The :c:func:`!PyByteArray_Init` and :c:func:`!PyByteArray_Fini` functions have been removed. They did nothing since Python 2.7.4 and Python 3.2.0, were excluded from the limited API (stable ABI), and were not documented. (Contributed by Victor Stinner in :issue:`35713`.) @@ -1625,7 +1625,7 @@ Build and C API Changes parameter for indicating the number of positional-only arguments. (Contributed by Pablo Galindo in :issue:`37221`.) -* :c:func:`Py_SetPath` now sets :data:`sys.executable` to the program full +* :c:func:`!Py_SetPath` now sets :data:`sys.executable` to the program full path (:c:func:`Py_GetProgramFullPath`) rather than to the program name (:c:func:`Py_GetProgramName`). (Contributed by Victor Stinner in :issue:`38234`.) @@ -1842,11 +1842,11 @@ Changes in Python behavior always use ``sys.platform.startswith('aix')``. (Contributed by M. Felt in :issue:`36588`.) -* :c:func:`PyEval_AcquireLock` and :c:func:`PyEval_AcquireThread` now +* :c:func:`!PyEval_AcquireLock` and :c:func:`!PyEval_AcquireThread` now terminate the current thread if called while the interpreter is finalizing, making them consistent with :c:func:`PyEval_RestoreThread`, :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`. If this - behavior is not desired, guard the call by checking :c:func:`_Py_IsFinalizing` + behavior is not desired, guard the call by checking :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing`. (Contributed by Joannah Nanjekye in :issue:`36475`.) @@ -2018,7 +2018,7 @@ Changes in the C API *cf_flags*. (Contributed by Guido van Rossum in :issue:`35766`.) -* The :c:func:`PyEval_ReInitThreads` function has been removed from the C API. +* The :c:func:`!PyEval_ReInitThreads` function has been removed from the C API. It should not be called explicitly: use :c:func:`PyOS_AfterFork_Child` instead. (Contributed by Victor Stinner in :issue:`36728`.) @@ -2118,7 +2118,7 @@ Changes in the C API (Contributed by Antoine Pitrou in :issue:`32388`.) -* The functions :c:func:`PyNode_AddChild` and :c:func:`PyParser_AddToken` now accept +* The functions :c:func:`!PyNode_AddChild` and :c:func:`!PyParser_AddToken` now accept two additional ``int`` arguments *end_lineno* and *end_col_offset*. * The :file:`libpython38.a` file to allow MinGW tools to link directly against diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 664c8d8a545db..6829970900930 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -870,9 +870,9 @@ Deprecated users can leverage the Abstract Syntax Tree (AST) generation and compilation stage, using the :mod:`ast` module. -* The Public C API functions :c:func:`PyParser_SimpleParseStringFlags`, - :c:func:`PyParser_SimpleParseStringFlagsFilename`, - :c:func:`PyParser_SimpleParseFileFlags` and :c:func:`PyNode_Compile` +* The Public C API functions :c:func:`!PyParser_SimpleParseStringFlags`, + :c:func:`!PyParser_SimpleParseStringFlagsFilename`, + :c:func:`!PyParser_SimpleParseFileFlags` and :c:func:`!PyNode_Compile` are deprecated and will be removed in Python 3.10 together with the old parser. * Using :data:`NotImplemented` in a boolean context has been deprecated, @@ -923,10 +923,10 @@ Deprecated (Contributed by Batuhan Taskaya in :issue:`39639` and :issue:`39969` and Serhiy Storchaka in :issue:`39988`.) -* The :c:func:`PyEval_InitThreads` and :c:func:`PyEval_ThreadsInitialized` +* The :c:func:`!PyEval_InitThreads` and :c:func:`!PyEval_ThreadsInitialized` functions are now deprecated and will be removed in Python 3.11. Calling - :c:func:`PyEval_InitThreads` now does nothing. The :term:`GIL` is initialized - by :c:func:`Py_Initialize()` since Python 3.7. + :c:func:`!PyEval_InitThreads` now does nothing. The :term:`GIL` is initialized + by :c:func:`Py_Initialize` since Python 3.7. (Contributed by Victor Stinner in :issue:`39877`.) * Passing ``None`` as the first argument to the :func:`shlex.split` function diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 17cc1fa7cb7ac..1fea35c42f26c 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -5036,15 +5036,15 @@ Limited API. Deprecate the following functions to configure the Python initialization: -* :c:func:`PySys_AddWarnOptionUnicode` -* :c:func:`PySys_AddWarnOption` -* :c:func:`PySys_AddXOption` -* :c:func:`PySys_HasWarnOptions` -* :c:func:`Py_SetPath` -* :c:func:`Py_SetProgramName` -* :c:func:`Py_SetPythonHome` -* :c:func:`Py_SetStandardStreamEncoding` -* :c:func:`_Py_SetProgramFullPath` +* :c:func:`!PySys_AddWarnOptionUnicode` +* :c:func:`!PySys_AddWarnOption` +* :c:func:`!PySys_AddXOption` +* :c:func:`!PySys_HasWarnOptions` +* :c:func:`!Py_SetPath` +* :c:func:`!Py_SetProgramName` +* :c:func:`!Py_SetPythonHome` +* :c:func:`!Py_SetStandardStreamEncoding` +* :c:func:`!_Py_SetProgramFullPath` Use the new :c:type:`PyConfig` API of the :ref:`Python Initialization Configuration ` instead (:pep:`587`). diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 5f1a0142ae778..85a5447ceb5f2 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -8765,7 +8765,7 @@ for relative path to files in current directory. .. nonce: fmehdG .. section: C API -The :c:func:`PyByteArray_Init` and :c:func:`PyByteArray_Fini` functions have +The :c:func:`!PyByteArray_Init` and :c:func:`!PyByteArray_Fini` functions have been removed. They did nothing since Python 2.7.4 and Python 3.2.0, were excluded from the limited API (stable ABI), and were not documented. @@ -8836,7 +8836,7 @@ Py_LIMITED_API. Patch by Arthur Neufeld. .. nonce: gFd85N .. section: C API -The :c:func:`_PyObject_GC_TRACK` and :c:func:`_PyObject_GC_UNTRACK` macros +The :c:func:`!_PyObject_GC_TRACK` and :c:func:`!_PyObject_GC_UNTRACK` macros have been removed from the public C API. .. diff --git a/Misc/NEWS.d/3.8.0a4.rst b/Misc/NEWS.d/3.8.0a4.rst index 17bbac6238913..00079c8491fe6 100644 --- a/Misc/NEWS.d/3.8.0a4.rst +++ b/Misc/NEWS.d/3.8.0a4.rst @@ -124,7 +124,7 @@ Galindo. .. nonce: CjRps3 .. section: Core and Builtins -:c:func:`PyEval_AcquireLock` and :c:func:`PyEval_AcquireThread` now +:c:func:`!PyEval_AcquireLock` and :c:func:`!PyEval_AcquireThread` now terminate the current thread if called while the interpreter is finalizing, making them consistent with :c:func:`PyEval_RestoreThread`, :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`. diff --git a/Misc/NEWS.d/3.8.0b1.rst b/Misc/NEWS.d/3.8.0b1.rst index 5285770de1318..f8f180bc731fd 100644 --- a/Misc/NEWS.d/3.8.0b1.rst +++ b/Misc/NEWS.d/3.8.0b1.rst @@ -2047,6 +2047,6 @@ unbound methods. These are objects supporting the optimization given by the .. nonce: FR-dMP .. section: C API -The :c:func:`PyEval_ReInitThreads` function has been removed from the C API. +The :c:func:`!PyEval_ReInitThreads` function has been removed from the C API. It should not be called explicitly: use :c:func:`PyOS_AfterFork_Child` instead. diff --git a/Misc/NEWS.d/3.9.0a1.rst b/Misc/NEWS.d/3.9.0a1.rst index 64410a1d34dfe..41ab33fac3322 100644 --- a/Misc/NEWS.d/3.9.0a1.rst +++ b/Misc/NEWS.d/3.9.0a1.rst @@ -5535,7 +5535,7 @@ Tyler Kieft. .. nonce: d0bhEA .. section: C API -:c:func:`Py_SetPath` now sets :data:`sys.executable` to the program full +:c:func:`!Py_SetPath` now sets :data:`sys.executable` to the program full path (:c:func:`Py_GetProgramFullPath`) rather than to the program name (:c:func:`Py_GetProgramName`). @@ -5546,8 +5546,8 @@ path (:c:func:`Py_GetProgramFullPath`) rather than to the program name .. nonce: ZbquVK .. section: C API -Python ignored arguments passed to :c:func:`Py_SetPath`, -:c:func:`Py_SetPythonHome` and :c:func:`Py_SetProgramName`: fix Python +Python ignored arguments passed to :c:func:`!Py_SetPath`, +:c:func:`!Py_SetPythonHome` and :c:func:`!Py_SetProgramName`: fix Python initialization to use specified arguments. .. diff --git a/Misc/NEWS.d/3.9.0a5.rst b/Misc/NEWS.d/3.9.0a5.rst index a129e721a4cf9..8c6057aa3c8a4 100644 --- a/Misc/NEWS.d/3.9.0a5.rst +++ b/Misc/NEWS.d/3.9.0a5.rst @@ -1234,8 +1234,8 @@ method name in the SystemError "bad call flags" error message to ease debug. .. nonce: GOYtIm .. section: C API -Deprecated :c:func:`PyEval_InitThreads` and -:c:func:`PyEval_ThreadsInitialized`. Calling :c:func:`PyEval_InitThreads` +Deprecated :c:func:`!PyEval_InitThreads` and +:c:func:`!PyEval_ThreadsInitialized`. Calling :c:func:`!PyEval_InitThreads` now does nothing. .. From webhook-mailer at python.org Tue Aug 22 14:33:30 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:33:30 -0000 Subject: [Python-checkins] [3.11] gh-107700: [Enum] Document that `EnumType` was added in 3.11 (GH-108260) (#108300) Message-ID: https://github.com/python/cpython/commit/9d7f2705b51eb7a4c48799ea16786f4a69f57dc5 commit: 9d7f2705b51eb7a4c48799ea16786f4a69f57dc5 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2023-08-22T20:33:26+02:00 summary: [3.11] gh-107700: [Enum] Document that `EnumType` was added in 3.11 (GH-108260) (#108300) (cherry picked from commit e8ef0bdd8c613a722bf7965bf1da912882141a52) Co-authored-by: Nikita Sobolev files: M Doc/library/enum.rst diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 15b8e2918d754..85da33156b1e3 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -247,6 +247,10 @@ Data Types >>> list(reversed(Color)) [, , ] + .. versionadded:: 3.11 + + Before 3.11 ``enum`` used ``EnumMeta`` type, which is kept as an alias. + .. class:: Enum From webhook-mailer at python.org Tue Aug 22 14:35:00 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:35:00 -0000 Subject: [Python-checkins] [3.12] gh-107700: [Enum] Document that `EnumType` was added in 3.11 (GH-108260) (#108301) Message-ID: https://github.com/python/cpython/commit/1dbf11814f32c7d4c1fc66cb0a29aa3667962b0d commit: 1dbf11814f32c7d4c1fc66cb0a29aa3667962b0d branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2023-08-22T20:34:56+02:00 summary: [3.12] gh-107700: [Enum] Document that `EnumType` was added in 3.11 (GH-108260) (#108301) (cherry picked from commit e8ef0bdd8c613a722bf7965bf1da912882141a52) Co-authored-by: Nikita Sobolev files: M Doc/library/enum.rst diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index e9c4f0e2c5f59..7653865f0b9b3 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -241,6 +241,10 @@ Data Types >>> list(reversed(Color)) [, , ] + .. versionadded:: 3.11 + + Before 3.11 ``enum`` used ``EnumMeta`` type, which is kept as an alias. + .. class:: Enum From webhook-mailer at python.org Tue Aug 22 14:38:42 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:38:42 -0000 Subject: [Python-checkins] Docs: Add link to skip to datetime's format codes (#108027) Message-ID: https://github.com/python/cpython/commit/35cb1605d08a77f1c18bd476b26391acaaa35599 commit: 35cb1605d08a77f1c18bd476b26391acaaa35599 branch: main author: Hugo van Kemenade committer: ambv date: 2023-08-22T20:38:38+02:00 summary: Docs: Add link to skip to datetime's format codes (#108027) files: M Doc/library/datetime.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index bed19ad145a20..400c369a9bb73 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -19,6 +19,10 @@ The :mod:`datetime` module supplies classes for manipulating dates and times. While date and time arithmetic is supported, the focus of the implementation is on efficient attribute extraction for output formatting and manipulation. +.. tip:: + + Skip to :ref:`the format codes `. + .. seealso:: Module :mod:`calendar` @@ -2322,6 +2326,8 @@ versus :meth:`strptime`: +----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ + .. _format-codes: + :meth:`strftime` and :meth:`strptime` Format Codes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From webhook-mailer at python.org Tue Aug 22 14:40:50 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:40:50 -0000 Subject: [Python-checkins] Document 3.13, 3.14 and future removals (#108055) Message-ID: https://github.com/python/cpython/commit/39de79b345f925ce3bbb79b33534872fe0c90877 commit: 39de79b345f925ce3bbb79b33534872fe0c90877 branch: main author: Hugo van Kemenade committer: ambv date: 2023-08-22T20:40:46+02:00 summary: Document 3.13, 3.14 and future removals (#108055) files: M Doc/whatsnew/3.12.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index ff651bcd04728..b557e6cde6a20 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1112,6 +1112,10 @@ Modules (see :pep:`594`): * :mod:`!uu` * :mod:`!xdrlib` +Other modules: + +* :mod:`!lib2to3`, and the :program:`2to3` program (:gh:`84540`) + APIs: * :class:`!configparser.LegacyInterpolation` (:gh:`90765`) @@ -1146,6 +1150,10 @@ Pending Removal in Python 3.14 Use :class:`ast.Constant` instead. (Contributed by Serhiy Storchaka in :gh:`90953`.) +* :mod:`asyncio`: the *msg* parameter of both + :meth:`asyncio.Future.cancel` and + :meth:`asyncio.Task.cancel` (:gh:`90985`) + * :mod:`collections.abc`: Deprecated :class:`collections.abc.ByteString`. Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. @@ -1212,12 +1220,17 @@ Pending Removal in Python 3.14 May be removed in 3.14. (Contributed by Nikita Sobolev in :gh:`101866`.) +* Creating :c:data:`immutable types ` with mutable + bases using the C API (:gh:`95388`) + Pending Removal in Future Versions ---------------------------------- The following APIs were deprecated in earlier Python versions and will be removed, although there is currently no date scheduled for their removal. +* :mod:`array`'s ``'u'`` format code (:gh:`57281`) + * :class:`typing.Text` (:gh:`92332`) * Currently Python accepts numeric literals immediately followed by keywords, From webhook-mailer at python.org Tue Aug 22 14:41:59 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:41:59 -0000 Subject: [Python-checkins] gh-105857: Document that asyncio subprocess std{in, out, err} can be file handles (#107986) Message-ID: https://github.com/python/cpython/commit/13966da71b693b1fae1a8ef66e34e2f0a90ec6c0 commit: 13966da71b693b1fae1a8ef66e34e2f0a90ec6c0 branch: main author: Hadh?zy Tam?s <85063808+Hels15 at users.noreply.github.com> committer: ambv date: 2023-08-22T20:41:56+02:00 summary: gh-105857: Document that asyncio subprocess std{in,out,err} can be file handles (#107986) stdin/out can be filehandles -> add to docs. files: M Doc/library/asyncio-eventloop.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 8f2d8f336c82b..04af53b980ff9 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1442,6 +1442,7 @@ async/await code consider using the high-level * *stdin* can be any of these: * a file-like object + * an existing file descriptor (a positive integer), for example those created with :meth:`os.pipe()` * the :const:`subprocess.PIPE` constant (default) which will create a new pipe and connect it, * the value ``None`` which will make the subprocess inherit the file From webhook-mailer at python.org Tue Aug 22 14:42:54 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:42:54 -0000 Subject: [Python-checkins] GH-92584: Remove distutils from the newtypes tutorial includes (#108024) Message-ID: https://github.com/python/cpython/commit/e97b7bef4fbe71821d59d2f41f311e514fd29e39 commit: e97b7bef4fbe71821d59d2f41f311e514fd29e39 branch: main author: Adam Turner <9087854+AA-Turner at users.noreply.github.com> committer: ambv date: 2023-08-22T20:42:51+02:00 summary: GH-92584: Remove distutils from the newtypes tutorial includes (#108024) files: A Doc/includes/newtypes/custom.c A Doc/includes/newtypes/custom2.c A Doc/includes/newtypes/custom3.c A Doc/includes/newtypes/custom4.c A Doc/includes/newtypes/pyproject.toml A Doc/includes/newtypes/setup.py A Doc/includes/newtypes/sublist.c A Doc/includes/newtypes/test.py D Doc/includes/custom.c D Doc/includes/custom2.c D Doc/includes/custom3.c D Doc/includes/custom4.c D Doc/includes/setup.py D Doc/includes/sublist.c D Doc/includes/test.py M Doc/extending/newtypes_tutorial.rst diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index 209a4ab76d226..7eba9759119b3 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -45,7 +45,7 @@ extension module :mod:`!custom`: allows defining heap-allocated extension types using the :c:func:`PyType_FromSpec` function, which isn't covered in this tutorial. -.. literalinclude:: ../includes/custom.c +.. literalinclude:: ../includes/newtypes/custom.c Now that's quite a bit to take in at once, but hopefully bits will seem familiar from the previous chapter. This file defines three things: @@ -194,36 +194,32 @@ This adds the type to the module dictionary. This allows us to create >>> mycustom = custom.Custom() That's it! All that remains is to build it; put the above code in a file called -:file:`custom.c` and: +:file:`custom.c`, + +.. literalinclude:: ../includes/newtypes/pyproject.toml + +in a file called :file:`pyproject.toml`, and .. code-block:: python - from distutils.core import setup, Extension - setup(name="custom", version="1.0", - ext_modules=[Extension("custom", ["custom.c"])]) + from setuptools import Extension, setup + setup(ext_modules=[Extension("custom", ["custom.c"])]) in a file called :file:`setup.py`; then typing .. code-block:: shell-session - $ python setup.py build + $ python -m pip install . -at a shell should produce a file :file:`custom.so` in a subdirectory; move to -that directory and fire up Python --- you should be able to ``import custom`` and -play around with Custom objects. +in a shell should produce a file :file:`custom.so` in a subdirectory +and install it; now fire up Python --- you should be able to ``import custom`` +and play around with ``Custom`` objects. That wasn't so hard, was it? Of course, the current Custom type is pretty uninteresting. It has no data and doesn't do anything. It can't even be subclassed. -.. note:: - While this documentation showcases the standard :mod:`!distutils` module - for building C extensions, it is recommended in real-world use cases to - use the newer and better-maintained ``setuptools`` library. Documentation - on how to do this is out of scope for this document and can be found in - the `Python Packaging User's Guide `_. - Adding data and methods to the Basic example ============================================ @@ -232,7 +228,7 @@ Let's extend the basic example to add some data and methods. Let's also make the type usable as a base class. We'll create a new module, :mod:`!custom2` that adds these capabilities: -.. literalinclude:: ../includes/custom2.c +.. literalinclude:: ../includes/newtypes/custom2.c This version of the module has a number of changes. @@ -514,17 +510,21 @@ We rename :c:func:`!PyInit_custom` to :c:func:`!PyInit_custom2`, update the module name in the :c:type:`PyModuleDef` struct, and update the full class name in the :c:type:`PyTypeObject` struct. -Finally, we update our :file:`setup.py` file to build the new module: +Finally, we update our :file:`setup.py` file to include the new module, .. code-block:: python - from distutils.core import setup, Extension - setup(name="custom", version="1.0", - ext_modules=[ - Extension("custom", ["custom.c"]), - Extension("custom2", ["custom2.c"]), - ]) + from setuptools import Extension, setup + setup(ext_modules=[ + Extension("custom", ["custom.c"]), + Extension("custom2", ["custom2.c"]), + ]) + +and then we re-install so that we can ``import custom2``: + +.. code-block:: shell-session + $ python -m pip install . Providing finer control over data attributes ============================================ @@ -535,7 +535,7 @@ version of our module, the instance variables :attr:`!first` and :attr:`!last` could be set to non-string values or even deleted. We want to make sure that these attributes always contain strings. -.. literalinclude:: ../includes/custom3.c +.. literalinclude:: ../includes/newtypes/custom3.c To provide greater control, over the :attr:`!first` and :attr:`!last` attributes, @@ -682,7 +682,7 @@ To allow a :class:`!Custom` instance participating in a reference cycle to be properly detected and collected by the cyclic GC, our :class:`!Custom` type needs to fill two additional slots and to enable a flag that enables these slots: -.. literalinclude:: ../includes/custom4.c +.. literalinclude:: ../includes/newtypes/custom4.c First, the traversal method lets the cyclic GC know about subobjects that could @@ -806,7 +806,7 @@ increases an internal counter: >>> print(s.increment()) 2 -.. literalinclude:: ../includes/sublist.c +.. literalinclude:: ../includes/newtypes/sublist.c As you can see, the source code closely resembles the :class:`!Custom` examples in diff --git a/Doc/includes/custom.c b/Doc/includes/newtypes/custom.c similarity index 100% rename from Doc/includes/custom.c rename to Doc/includes/newtypes/custom.c diff --git a/Doc/includes/custom2.c b/Doc/includes/newtypes/custom2.c similarity index 100% rename from Doc/includes/custom2.c rename to Doc/includes/newtypes/custom2.c diff --git a/Doc/includes/custom3.c b/Doc/includes/newtypes/custom3.c similarity index 100% rename from Doc/includes/custom3.c rename to Doc/includes/newtypes/custom3.c diff --git a/Doc/includes/custom4.c b/Doc/includes/newtypes/custom4.c similarity index 100% rename from Doc/includes/custom4.c rename to Doc/includes/newtypes/custom4.c diff --git a/Doc/includes/newtypes/pyproject.toml b/Doc/includes/newtypes/pyproject.toml new file mode 100644 index 0000000000000..ea7937a317147 --- /dev/null +++ b/Doc/includes/newtypes/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "custom" +version = "1" diff --git a/Doc/includes/newtypes/setup.py b/Doc/includes/newtypes/setup.py new file mode 100644 index 0000000000000..67f83673bcc65 --- /dev/null +++ b/Doc/includes/newtypes/setup.py @@ -0,0 +1,8 @@ +from setuptools import Extension, setup +setup(ext_modules=[ + Extension("custom", ["custom.c"]), + Extension("custom2", ["custom2.c"]), + Extension("custom3", ["custom3.c"]), + Extension("custom4", ["custom4.c"]), + Extension("sublist", ["sublist.c"]), +]) diff --git a/Doc/includes/sublist.c b/Doc/includes/newtypes/sublist.c similarity index 100% rename from Doc/includes/sublist.c rename to Doc/includes/newtypes/sublist.c diff --git a/Doc/includes/test.py b/Doc/includes/newtypes/test.py similarity index 94% rename from Doc/includes/test.py rename to Doc/includes/newtypes/test.py index 09ebe3fec0bdb..55a5cf9f68b94 100644 --- a/Doc/includes/test.py +++ b/Doc/includes/newtypes/test.py @@ -187,13 +187,6 @@ >>> gc.enable() """ -import os -import sys -from distutils.util import get_platform -PLAT_SPEC = "%s-%d.%d" % (get_platform(), *sys.version_info[:2]) -src = os.path.join("build", "lib.%s" % PLAT_SPEC) -sys.path.append(src) - if __name__ == "__main__": import doctest, __main__ doctest.testmod(__main__) diff --git a/Doc/includes/setup.py b/Doc/includes/setup.py deleted file mode 100644 index a38a39de3e7c8..0000000000000 --- a/Doc/includes/setup.py +++ /dev/null @@ -1,9 +0,0 @@ -from distutils.core import setup, Extension -setup(name="noddy", version="1.0", - ext_modules=[ - Extension("noddy", ["noddy.c"]), - Extension("noddy2", ["noddy2.c"]), - Extension("noddy3", ["noddy3.c"]), - Extension("noddy4", ["noddy4.c"]), - Extension("shoddy", ["shoddy.c"]), - ]) From webhook-mailer at python.org Tue Aug 22 14:45:26 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 18:45:26 -0000 Subject: [Python-checkins] gh-107136: Remove Plausible for docs metrics (#107856) Message-ID: https://github.com/python/cpython/commit/fc23f34cc9701949e6832eb32f26ea89f6622b82 commit: fc23f34cc9701949e6832eb32f26ea89f6622b82 branch: main author: Hugo van Kemenade committer: ambv date: 2023-08-22T20:45:22+02:00 summary: gh-107136: Remove Plausible for docs metrics (#107856) files: M Doc/tools/templates/layout.html diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 80103158ea01e..9498b2ccc5af9 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,9 +26,6 @@ {% endblock %} {% block extrahead %} - {% if builder == "html" %} - - {% endif %} {% if builder != "htmlhelp" %} {% if pagename == 'whatsnew/changelog' and not embedded %} From webhook-mailer at python.org Tue Aug 22 15:03:24 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 19:03:24 -0000 Subject: [Python-checkins] [3.12] gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw (#108316) Message-ID: https://github.com/python/cpython/commit/256586ab8776e4526ca594b4866b9a3492e628f1 commit: 256586ab8776e4526ca594b4866b9a3492e628f1 branch: 3.12 author: ?ukasz Langa committer: ambv date: 2023-08-22T21:03:20+02:00 summary: [3.12] gh-108310: Fix CVE-2023-40217: Check for & avoid the ssl pre-close flaw (#108316) Instances of `ssl.SSLSocket` were vulnerable to a bypass of the TLS handshake and included protections (like certificate verification) and treating sent unencrypted data as if it were post-handshake TLS encrypted data. The vulnerability is caused when a socket is connected, data is sent by the malicious peer and stored in a buffer, and then the malicious peer closes the socket within a small timing window before the other peers? TLS handshake can begin. After this sequence of events the closed socket will not immediately attempt a TLS handshake due to not being connected but will also allow the buffered data to be read as if a successful TLS handshake had occurred. Co-authored-by: Gregory P. Smith [Google LLC] files: A Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst M Lib/ssl.py M Lib/test/test_ssl.py diff --git a/Lib/ssl.py b/Lib/ssl.py index 1d5873726441e..ff363c75e7dfd 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -975,7 +975,7 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, ) self = cls.__new__(cls, **kwargs) super(SSLSocket, self).__init__(**kwargs) - self.settimeout(sock.gettimeout()) + sock_timeout = sock.gettimeout() sock.detach() self._context = context @@ -994,9 +994,38 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, if e.errno != errno.ENOTCONN: raise connected = False + blocking = self.getblocking() + self.setblocking(False) + try: + # We are not connected so this is not supposed to block, but + # testing revealed otherwise on macOS and Windows so we do + # the non-blocking dance regardless. Our raise when any data + # is found means consuming the data is harmless. + notconn_pre_handshake_data = self.recv(1) + except OSError as e: + # EINVAL occurs for recv(1) on non-connected on unix sockets. + if e.errno not in (errno.ENOTCONN, errno.EINVAL): + raise + notconn_pre_handshake_data = b'' + self.setblocking(blocking) + if notconn_pre_handshake_data: + # This prevents pending data sent to the socket before it was + # closed from escaping to the caller who could otherwise + # presume it came through a successful TLS connection. + reason = "Closed before TLS handshake with data in recv buffer." + notconn_pre_handshake_data_error = SSLError(e.errno, reason) + # Add the SSLError attributes that _ssl.c always adds. + notconn_pre_handshake_data_error.reason = reason + notconn_pre_handshake_data_error.library = None + try: + self.close() + except OSError: + pass + raise notconn_pre_handshake_data_error else: connected = True + self.settimeout(sock_timeout) # Must come after setblocking() calls. self._connected = connected if connected: # create the SSL object diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 6117ca3fdba1b..ad5377ec059c0 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -10,11 +10,14 @@ from test.support import threading_helper from test.support import warnings_helper from test.support import asyncore +import re import socket import select +import struct import time import enum import gc +import http.client import os import errno import pprint @@ -4659,6 +4662,214 @@ def sni_cb(sock, servername, ctx): s.connect((HOST, server.port)) +def set_socket_so_linger_on_with_zero_timeout(sock): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) + + +class TestPreHandshakeClose(unittest.TestCase): + """Verify behavior of close sockets with received data before to the handshake. + """ + + class SingleConnectionTestServerThread(threading.Thread): + + def __init__(self, *, name, call_after_accept): + self.call_after_accept = call_after_accept + self.received_data = b'' # set by .run() + self.wrap_error = None # set by .run() + self.listener = None # set by .start() + self.port = None # set by .start() + super().__init__(name=name) + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + try: + if self.listener: + self.listener.close() + except OSError: + pass + self.join() + self.wrap_error = None # avoid dangling references + + def start(self): + self.ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED + self.ssl_ctx.load_verify_locations(cafile=ONLYCERT) + self.ssl_ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + self.listener = socket.socket() + self.port = socket_helper.bind_port(self.listener) + self.listener.settimeout(2.0) + self.listener.listen(1) + super().start() + + def run(self): + conn, address = self.listener.accept() + self.listener.close() + with conn: + if self.call_after_accept(conn): + return + try: + tls_socket = self.ssl_ctx.wrap_socket(conn, server_side=True) + except OSError as err: # ssl.SSLError inherits from OSError + self.wrap_error = err + else: + try: + self.received_data = tls_socket.recv(400) + except OSError: + pass # closed, protocol error, etc. + + def non_linux_skip_if_other_okay_error(self, err): + if sys.platform == "linux": + return # Expect the full test setup to always work on Linux. + if (isinstance(err, ConnectionResetError) or + (isinstance(err, OSError) and err.errno == errno.EINVAL) or + re.search('wrong.version.number', getattr(err, "reason", ""), re.I)): + # On Windows the TCP RST leads to a ConnectionResetError + # (ECONNRESET) which Linux doesn't appear to surface to userspace. + # If wrap_socket() winds up on the "if connected:" path and doing + # the actual wrapping... we get an SSLError from OpenSSL. Typically + # WRONG_VERSION_NUMBER. While appropriate, neither is the scenario + # we're specifically trying to test. The way this test is written + # is known to work on Linux. We'll skip it anywhere else that it + # does not present as doing so. + self.skipTest(f"Could not recreate conditions on {sys.platform}:" + f" {err=}") + # If maintaining this conditional winds up being a problem. + # just turn this into an unconditional skip anything but Linux. + # The important thing is that our CI has the logic covered. + + def test_preauth_data_to_tls_server(self): + server_accept_called = threading.Event() + ready_for_server_wrap_socket = threading.Event() + + def call_after_accept(unused): + server_accept_called.set() + if not ready_for_server_wrap_socket.wait(2.0): + raise RuntimeError("wrap_socket event never set, test may fail.") + return False # Tell the server thread to continue. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_server") + self.enterContext(server) # starts it & unittest.TestCase stops it. + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(client) + client.setblocking(False) + + server_accept_called.wait() + client.send(b"DELETE /data HTTP/1.0\r\n\r\n") + client.close() # RST + + ready_for_server_wrap_socket.set() + server.join() + wrap_error = server.wrap_error + self.assertEqual(b"", server.received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + + def test_preauth_data_to_tls_client(self): + client_can_continue_with_wrap_socket = threading.Event() + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 307 Temporary Redirect\r\n" + b"Location: https://example.com/someone-elses-server\r\n" + b"\r\n") + conn_to_client.close() # RST + client_can_continue_with_wrap_socket.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_client") + self.enterContext(server) # starts it & unittest.TestCase stops it. + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + if not client_can_continue_with_wrap_socket.wait(2.0): + self.fail("test server took too long.") + ssl_ctx = ssl.create_default_context() + try: + tls_client = ssl_ctx.wrap_socket( + client, server_hostname="localhost") + except OSError as err: # SSLError inherits from OSError + wrap_error = err + received_data = b"" + else: + wrap_error = None + received_data = tls_client.recv(400) + tls_client.close() + + server.join() + self.assertEqual(b"", received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + + def test_https_client_non_tls_response_ignored(self): + + server_responding = threading.Event() + + class SynchronizedHTTPSConnection(http.client.HTTPSConnection): + def connect(self): + http.client.HTTPConnection.connect(self) + # Wait for our fault injection server to have done its thing. + if not server_responding.wait(1.0) and support.verbose: + sys.stdout.write("server_responding event never set.") + self.sock = self._context.wrap_socket( + self.sock, server_hostname=self.host) + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 402 Payment Required\r\n" + b"\r\n") + conn_to_client.close() # RST + server_responding.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="non_tls_http_RST_responder") + self.enterContext(server) # starts it & unittest.TestCase stops it. + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + connection = SynchronizedHTTPSConnection( + f"localhost", + port=server.port, + context=ssl.create_default_context(), + timeout=2.0, + ) + # There are lots of reasons this raises as desired, long before this + # test was added. Sending the request requires a successful TLS wrapped + # socket; that fails if the connection is broken. It may seem pointless + # to test this. It serves as an illustration of something that we never + # want to happen... properly not happening. + with self.assertRaises(OSError) as err_ctx: + connection.request("HEAD", "/test", headers={"Host": "localhost"}) + response = connection.getresponse() + + class TestEnumerations(unittest.TestCase): def test_tlsversion(self): diff --git a/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst new file mode 100644 index 0000000000000..403c77a9d480e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-08-22-17-39-12.gh-issue-108310.fVM3sg.rst @@ -0,0 +1,7 @@ +Fixed an issue where instances of :class:`ssl.SSLSocket` were vulnerable to +a bypass of the TLS handshake and included protections (like certificate +verification) and treating sent unencrypted data as if it were +post-handshake TLS encrypted data. Security issue reported as +`CVE-2023-40217 +`_ by +Aapo Oksman. Patch by Gregory P. Smith. From webhook-mailer at python.org Tue Aug 22 15:23:59 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Tue, 22 Aug 2023 19:23:59 -0000 Subject: [Python-checkins] [3.11] Clarify how topics.py gets created. (GH-106121) (GH-106580) Message-ID: https://github.com/python/cpython/commit/fadf8a032b8b602ed2681aa912ceab01d095fd8f commit: fadf8a032b8b602ed2681aa912ceab01d095fd8f branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: serhiy-storchaka date: 2023-08-22T19:23:55Z summary: [3.11] Clarify how topics.py gets created. (GH-106121) (GH-106580) When changing docs, it was easy to find text in topics.py, and I wondered whether I was supposed to edit it. Thankfully, the top of the file says it's auto-generated, so I knew I didn't have to edit it. But I didn't know what started the auto-generation process. It's part of the release process, so I'll leave a note here for future editors. (cherry picked from commit dac1e364901d3668742e6eecc2ce63586330c11f) Co-authored-by: Ned Batchelder files: M Doc/tools/extensions/pyspecific.py diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index d13a9dfd4c8eb..31c42e9d9d3a6 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -562,6 +562,7 @@ def finish(self): try: f.write('# -*- coding: utf-8 -*-\n'.encode('utf-8')) f.write(('# Autogenerated by Sphinx on %s\n' % asctime()).encode('utf-8')) + f.write('# as part of the release process.\n'.encode('utf-8')) f.write(('topics = ' + pformat(self.topics) + '\n').encode('utf-8')) finally: f.close() From webhook-mailer at python.org Tue Aug 22 15:37:17 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 19:37:17 -0000 Subject: [Python-checkins] [3.12] gh-105857: Document that asyncio subprocess std{in, out, err} can be file handles (GH-107986) (#108332) Message-ID: https://github.com/python/cpython/commit/3cdaa6a7941696c30c774f9dc5b0bad72f8f07e2 commit: 3cdaa6a7941696c30c774f9dc5b0bad72f8f07e2 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2023-08-22T21:37:14+02:00 summary: [3.12] gh-105857: Document that asyncio subprocess std{in,out,err} can be file handles (GH-107986) (#108332) (cherry picked from commit 13966da71b693b1fae1a8ef66e34e2f0a90ec6c0) Co-authored-by: Hadh?zy Tam?s <85063808+Hels15 at users.noreply.github.com> files: M Doc/library/asyncio-eventloop.rst diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 8f2d8f336c82b..04af53b980ff9 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1442,6 +1442,7 @@ async/await code consider using the high-level * *stdin* can be any of these: * a file-like object + * an existing file descriptor (a positive integer), for example those created with :meth:`os.pipe()` * the :const:`subprocess.PIPE` constant (default) which will create a new pipe and connect it, * the value ``None`` which will make the subprocess inherit the file From webhook-mailer at python.org Tue Aug 22 15:38:17 2023 From: webhook-mailer at python.org (ambv) Date: Tue, 22 Aug 2023 19:38:17 -0000 Subject: [Python-checkins] [3.12] Document 3.13, 3.14 and future removals (GH-108055) (#108331) Message-ID: https://github.com/python/cpython/commit/fad80598dc0c5a6c1b9bc9802e80a0f9686628fe commit: fad80598dc0c5a6c1b9bc9802e80a0f9686628fe branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2023-08-22T21:38:13+02:00 summary: [3.12] Document 3.13, 3.14 and future removals (GH-108055) (#108331) (cherry picked from commit 39de79b345f925ce3bbb79b33534872fe0c90877) Co-authored-by: Hugo van Kemenade files: M Doc/whatsnew/3.12.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 2084f4ed0b9bc..dca4fd999621d 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1113,6 +1113,10 @@ Modules (see :pep:`594`): * :mod:`uu` * :mod:`xdrlib` +Other modules: + +* :mod:`!lib2to3`, and the :program:`2to3` program (:gh:`84540`) + APIs: * :class:`!configparser.LegacyInterpolation` (:gh:`90765`) @@ -1146,6 +1150,10 @@ Pending Removal in Python 3.14 Use :class:`ast.Constant` instead. (Contributed by Serhiy Storchaka in :gh:`90953`.) +* :mod:`asyncio`: the *msg* parameter of both + :meth:`asyncio.Future.cancel` and + :meth:`asyncio.Task.cancel` (:gh:`90985`) + * :mod:`collections.abc`: Deprecated :class:`collections.abc.ByteString`. Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. @@ -1212,12 +1220,17 @@ Pending Removal in Python 3.14 May be removed in 3.14. (Contributed by Nikita Sobolev in :gh:`101866`.) +* Creating :c:data:`immutable types ` with mutable + bases using the C API (:gh:`95388`) + Pending Removal in Future Versions ---------------------------------- The following APIs were deprecated in earlier Python versions and will be removed, although there is currently no date scheduled for their removal. +* :mod:`array`'s ``'u'`` format code (:gh:`57281`) + * :class:`typing.Text` (:gh:`92332`) * Currently Python accepts numeric literals immediately followed by keywords, From webhook-mailer at python.org Tue Aug 22 15:49:42 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Tue, 22 Aug 2023 19:49:42 -0000 Subject: [Python-checkins] Fix spurious diff if the cases generator is run on Windows (#108319) Message-ID: https://github.com/python/cpython/commit/a0bb4a39d1ca10e4a75f50a9fbe90cc9db28d29e commit: a0bb4a39d1ca10e4a75f50a9fbe90cc9db28d29e branch: main author: Alex Waygood committer: AlexWaygood date: 2023-08-22T20:49:39+01:00 summary: Fix spurious diff if the cases generator is run on Windows (#108319) files: M Tools/cases_generator/generate_cases.py diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 6ee6836305811..f605dcc5e4632 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -237,7 +237,7 @@ def from_source_files(self) -> str: except ValueError: # May happen on Windows if root and temp on different volumes pass - filenames.append(filename) + filenames.append(filename.replace(os.path.sep, posixpath.sep)) paths = f"\n{self.out.comment} ".join(filenames) return f"{self.out.comment} from:\n{self.out.comment} {paths}\n" From webhook-mailer at python.org Tue Aug 22 15:52:54 2023 From: webhook-mailer at python.org (hugovk) Date: Tue, 22 Aug 2023 19:52:54 -0000 Subject: [Python-checkins] [3.11] Docs: Add link to skip to datetime's format codes (GH-108027) (#108330) Message-ID: https://github.com/python/cpython/commit/12cad6155bb1291b4b6c111ef480bbc0e6a68941 commit: 12cad6155bb1291b4b6c111ef480bbc0e6a68941 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: hugovk date: 2023-08-22T13:52:51-06:00 summary: [3.11] Docs: Add link to skip to datetime's format codes (GH-108027) (#108330) Co-authored-by: Hugo van Kemenade files: M Doc/library/datetime.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 9711f944e90dc..d7b0401b7feef 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -19,6 +19,10 @@ The :mod:`datetime` module supplies classes for manipulating dates and times. While date and time arithmetic is supported, the focus of the implementation is on efficient attribute extraction for output formatting and manipulation. +.. tip:: + + Skip to :ref:`the format codes `. + .. seealso:: Module :mod:`calendar` @@ -2314,6 +2318,8 @@ versus :meth:`strptime`: +----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ + .. _format-codes: + :meth:`strftime` and :meth:`strptime` Format Codes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From webhook-mailer at python.org Tue Aug 22 16:06:12 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 22 Aug 2023 20:06:12 -0000 Subject: [Python-checkins] [3.12] gh-106016: Add Lib/test/test_module/ directory (#108293) (#108306) Message-ID: https://github.com/python/cpython/commit/ca47a2183a35d4b71cc062c926b7f96112fdf5ce commit: ca47a2183a35d4b71cc062c926b7f96112fdf5ce branch: 3.12 author: Victor Stinner committer: Yhg1s date: 2023-08-22T22:06:07+02:00 summary: [3.12] gh-106016: Add Lib/test/test_module/ directory (#108293) (#108306) gh-106016: Add Lib/test/test_module/ directory (#108293) * Move Python scripts related to test_module to this new directory: good_getattr.py and bad_getattrX.py scripts. * Move Lib/test/test_module.py to Lib/test/test_module/__init__.py. (cherry picked from commit adfc118fdab66882599e01a84c22bd897055f3f1) files: A Lib/test/test_module/__init__.py A Lib/test/test_module/bad_getattr.py A Lib/test/test_module/bad_getattr2.py A Lib/test/test_module/bad_getattr3.py A Lib/test/test_module/good_getattr.py D Lib/test/bad_getattr.py D Lib/test/bad_getattr2.py D Lib/test/bad_getattr3.py D Lib/test/good_getattr.py D Lib/test/test_module.py M Makefile.pre.in diff --git a/Lib/test/test_module.py b/Lib/test/test_module/__init__.py similarity index 92% rename from Lib/test/test_module.py rename to Lib/test/test_module/__init__.py index c7eb92290e1b6..cfc4d9ccf1cc8 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module/__init__.py @@ -126,8 +126,8 @@ def test_weakref(self): self.assertIs(wr(), None) def test_module_getattr(self): - import test.good_getattr as gga - from test.good_getattr import test + import test.test_module.good_getattr as gga + from test.test_module.good_getattr import test self.assertEqual(test, "There is test") self.assertEqual(gga.x, 1) self.assertEqual(gga.y, 2) @@ -135,46 +135,46 @@ def test_module_getattr(self): "Deprecated, use whatever instead"): gga.yolo self.assertEqual(gga.whatever, "There is whatever") - del sys.modules['test.good_getattr'] + del sys.modules['test.test_module.good_getattr'] def test_module_getattr_errors(self): - import test.bad_getattr as bga - from test import bad_getattr2 + import test.test_module.bad_getattr as bga + from test.test_module import bad_getattr2 self.assertEqual(bga.x, 1) self.assertEqual(bad_getattr2.x, 1) with self.assertRaises(TypeError): bga.nope with self.assertRaises(TypeError): bad_getattr2.nope - del sys.modules['test.bad_getattr'] - if 'test.bad_getattr2' in sys.modules: - del sys.modules['test.bad_getattr2'] + del sys.modules['test.test_module.bad_getattr'] + if 'test.test_module.bad_getattr2' in sys.modules: + del sys.modules['test.test_module.bad_getattr2'] def test_module_dir(self): - import test.good_getattr as gga + import test.test_module.good_getattr as gga self.assertEqual(dir(gga), ['a', 'b', 'c']) - del sys.modules['test.good_getattr'] + del sys.modules['test.test_module.good_getattr'] def test_module_dir_errors(self): - import test.bad_getattr as bga - from test import bad_getattr2 + import test.test_module.bad_getattr as bga + from test.test_module import bad_getattr2 with self.assertRaises(TypeError): dir(bga) with self.assertRaises(TypeError): dir(bad_getattr2) - del sys.modules['test.bad_getattr'] - if 'test.bad_getattr2' in sys.modules: - del sys.modules['test.bad_getattr2'] + del sys.modules['test.test_module.bad_getattr'] + if 'test.test_module.bad_getattr2' in sys.modules: + del sys.modules['test.test_module.bad_getattr2'] def test_module_getattr_tricky(self): - from test import bad_getattr3 + from test.test_module import bad_getattr3 # these lookups should not crash with self.assertRaises(AttributeError): bad_getattr3.one with self.assertRaises(AttributeError): bad_getattr3.delgetattr - if 'test.bad_getattr3' in sys.modules: - del sys.modules['test.bad_getattr3'] + if 'test.test_module.bad_getattr3' in sys.modules: + del sys.modules['test.test_module.bad_getattr3'] def test_module_repr_minimal(self): # reprs when modules have no __file__, __name__, or __loader__ diff --git a/Lib/test/bad_getattr.py b/Lib/test/test_module/bad_getattr.py similarity index 100% rename from Lib/test/bad_getattr.py rename to Lib/test/test_module/bad_getattr.py diff --git a/Lib/test/bad_getattr2.py b/Lib/test/test_module/bad_getattr2.py similarity index 100% rename from Lib/test/bad_getattr2.py rename to Lib/test/test_module/bad_getattr2.py diff --git a/Lib/test/bad_getattr3.py b/Lib/test/test_module/bad_getattr3.py similarity index 100% rename from Lib/test/bad_getattr3.py rename to Lib/test/test_module/bad_getattr3.py diff --git a/Lib/test/good_getattr.py b/Lib/test/test_module/good_getattr.py similarity index 100% rename from Lib/test/good_getattr.py rename to Lib/test/test_module/good_getattr.py diff --git a/Makefile.pre.in b/Makefile.pre.in index a74f4fd66d323..d9d56bb45ce71 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2192,6 +2192,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_lib2to3/data \ test/test_lib2to3/data/fixers \ test/test_lib2to3/data/fixers/myfixes \ + test/test_module \ test/test_peg_generator \ test/test_sqlite3 \ test/test_tkinter \ From webhook-mailer at python.org Tue Aug 22 16:06:37 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 22 Aug 2023 20:06:37 -0000 Subject: [Python-checkins] [3.12] gh-108303: Add Lib/test/test_cppext/ sub-directory (GH-108325) (#108328) Message-ID: https://github.com/python/cpython/commit/b539dd3073f9e37de25f73ad19d188cd56cc19b4 commit: b539dd3073f9e37de25f73ad19d188cd56cc19b4 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-22T22:06:33+02:00 summary: [3.12] gh-108303: Add Lib/test/test_cppext/ sub-directory (GH-108325) (#108328) gh-108303: Add Lib/test/test_cppext/ sub-directory (GH-108325) * Move test_cppext to its own directory * Rename setup_testcppext.py to setup.py * Rename _testcppext.cpp to extension.cpp * The source (extension.cpp) is now also copied by the test. (cherry picked from commit 21dda09600848ac280481f7c64f8d9516dc69bb2) Co-authored-by: Victor Stinner files: A Lib/test/test_cppext/__init__.py A Lib/test/test_cppext/extension.cpp A Lib/test/test_cppext/setup.py D Lib/test/_testcppext.cpp D Lib/test/setup_testcppext.py D Lib/test/test_cppext.py M Makefile.pre.in diff --git a/Lib/test/test_cppext.py b/Lib/test/test_cppext/__init__.py similarity index 91% rename from Lib/test/test_cppext.py rename to Lib/test/test_cppext/__init__.py index e2fedc9735079..f3d32a3c7612c 100644 --- a/Lib/test/test_cppext.py +++ b/Lib/test/test_cppext/__init__.py @@ -11,9 +11,8 @@ MS_WINDOWS = (sys.platform == 'win32') - - -SETUP_TESTCPPEXT = support.findfile('setup_testcppext.py') +SOURCE = os.path.join(os.path.dirname(__file__), 'extension.cpp') +SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') @support.requires_subprocess() @@ -42,7 +41,8 @@ def check_build(self, std_cpp03, extension_name): def _check_build(self, std_cpp03, extension_name, python_exe): pkg_dir = 'pkg' os.mkdir(pkg_dir) - shutil.copy(SETUP_TESTCPPEXT, os.path.join(pkg_dir, "setup.py")) + shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) + shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE))) def run_cmd(operation, cmd): env = os.environ.copy() diff --git a/Lib/test/_testcppext.cpp b/Lib/test/test_cppext/extension.cpp similarity index 99% rename from Lib/test/_testcppext.cpp rename to Lib/test/test_cppext/extension.cpp index 82b471312dd2b..90669b10cb2c6 100644 --- a/Lib/test/_testcppext.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -1,5 +1,7 @@ // gh-91321: Very basic C++ test extension to check that the Python C API is // compatible with C++ and does not emit C++ compiler warnings. +// +// The code is only built, not executed. // Always enable assertions #undef NDEBUG diff --git a/Lib/test/setup_testcppext.py b/Lib/test/test_cppext/setup.py similarity index 93% rename from Lib/test/setup_testcppext.py rename to Lib/test/test_cppext/setup.py index 22fe750085fd7..6867094a42043 100644 --- a/Lib/test/setup_testcppext.py +++ b/Lib/test/test_cppext/setup.py @@ -2,7 +2,6 @@ # compatible with C++ and does not emit C++ compiler warnings. import os import sys -from test import support from setuptools import setup, Extension @@ -10,7 +9,7 @@ MS_WINDOWS = (sys.platform == 'win32') -SOURCE = support.findfile('_testcppext.cpp') +SOURCE = 'extension.cpp' if not MS_WINDOWS: # C++ compiler flags for GCC and clang CPPFLAGS = [ diff --git a/Makefile.pre.in b/Makefile.pre.in index d9d56bb45ce71..ae6a12c2a2c43 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2132,6 +2132,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/support/_hypothesis_stubs \ test/test_asyncio \ test/test_capi \ + test/test_cppext \ test/test_ctypes \ test/test_email \ test/test_email/data \ From webhook-mailer at python.org Tue Aug 22 16:06:56 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 22 Aug 2023 20:06:56 -0000 Subject: [Python-checkins] [3.12] Docs: Add link to skip to datetime's format codes (GH-108027) (#108329) Message-ID: https://github.com/python/cpython/commit/92cbe739d42acde858e27d9c06524312dda7534e commit: 92cbe739d42acde858e27d9c06524312dda7534e branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-22T22:06:53+02:00 summary: [3.12] Docs: Add link to skip to datetime's format codes (GH-108027) (#108329) Docs: Add link to skip to datetime's format codes (GH-108027) (cherry picked from commit 35cb1605d08a77f1c18bd476b26391acaaa35599) Co-authored-by: Hugo van Kemenade files: M Doc/library/datetime.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index bed19ad145a20..400c369a9bb73 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -19,6 +19,10 @@ The :mod:`datetime` module supplies classes for manipulating dates and times. While date and time arithmetic is supported, the focus of the implementation is on efficient attribute extraction for output formatting and manipulation. +.. tip:: + + Skip to :ref:`the format codes `. + .. seealso:: Module :mod:`calendar` @@ -2322,6 +2326,8 @@ versus :meth:`strptime`: +----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ + .. _format-codes: + :meth:`strftime` and :meth:`strptime` Format Codes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From webhook-mailer at python.org Tue Aug 22 16:07:19 2023 From: webhook-mailer at python.org (Yhg1s) Date: Tue, 22 Aug 2023 20:07:19 -0000 Subject: [Python-checkins] [3.12] GH-92584: Remove distutils from the newtypes tutorial includes (GH-108024) (#108333) Message-ID: https://github.com/python/cpython/commit/200af4294e9a99bf0d4f535c43f75f304eb53497 commit: 200af4294e9a99bf0d4f535c43f75f304eb53497 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-22T22:07:15+02:00 summary: [3.12] GH-92584: Remove distutils from the newtypes tutorial includes (GH-108024) (#108333) GH-92584: Remove distutils from the newtypes tutorial includes (GH-108024) (cherry picked from commit e97b7bef4fbe71821d59d2f41f311e514fd29e39) Co-authored-by: Adam Turner <9087854+AA-Turner at users.noreply.github.com> files: A Doc/includes/newtypes/custom.c A Doc/includes/newtypes/custom2.c A Doc/includes/newtypes/custom3.c A Doc/includes/newtypes/custom4.c A Doc/includes/newtypes/pyproject.toml A Doc/includes/newtypes/setup.py A Doc/includes/newtypes/sublist.c A Doc/includes/newtypes/test.py D Doc/includes/custom.c D Doc/includes/custom2.c D Doc/includes/custom3.c D Doc/includes/custom4.c D Doc/includes/setup.py D Doc/includes/sublist.c D Doc/includes/test.py M Doc/extending/newtypes_tutorial.rst diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index f8684df5dd829..c2bc5f699a159 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -45,7 +45,7 @@ extension module :mod:`!custom`: allows defining heap-allocated extension types using the :c:func:`PyType_FromSpec` function, which isn't covered in this tutorial. -.. literalinclude:: ../includes/custom.c +.. literalinclude:: ../includes/newtypes/custom.c Now that's quite a bit to take in at once, but hopefully bits will seem familiar from the previous chapter. This file defines three things: @@ -196,36 +196,32 @@ This adds the type to the module dictionary. This allows us to create >>> mycustom = custom.Custom() That's it! All that remains is to build it; put the above code in a file called -:file:`custom.c` and: +:file:`custom.c`, + +.. literalinclude:: ../includes/newtypes/pyproject.toml + +in a file called :file:`pyproject.toml`, and .. code-block:: python - from distutils.core import setup, Extension - setup(name="custom", version="1.0", - ext_modules=[Extension("custom", ["custom.c"])]) + from setuptools import Extension, setup + setup(ext_modules=[Extension("custom", ["custom.c"])]) in a file called :file:`setup.py`; then typing .. code-block:: shell-session - $ python setup.py build + $ python -m pip install . -at a shell should produce a file :file:`custom.so` in a subdirectory; move to -that directory and fire up Python --- you should be able to ``import custom`` and -play around with Custom objects. +in a shell should produce a file :file:`custom.so` in a subdirectory +and install it; now fire up Python --- you should be able to ``import custom`` +and play around with ``Custom`` objects. That wasn't so hard, was it? Of course, the current Custom type is pretty uninteresting. It has no data and doesn't do anything. It can't even be subclassed. -.. note:: - While this documentation showcases the standard :mod:`!distutils` module - for building C extensions, it is recommended in real-world use cases to - use the newer and better-maintained ``setuptools`` library. Documentation - on how to do this is out of scope for this document and can be found in - the `Python Packaging User's Guide `_. - Adding data and methods to the Basic example ============================================ @@ -234,7 +230,7 @@ Let's extend the basic example to add some data and methods. Let's also make the type usable as a base class. We'll create a new module, :mod:`!custom2` that adds these capabilities: -.. literalinclude:: ../includes/custom2.c +.. literalinclude:: ../includes/newtypes/custom2.c This version of the module has a number of changes. @@ -516,17 +512,21 @@ We rename :c:func:`!PyInit_custom` to :c:func:`!PyInit_custom2`, update the module name in the :c:type:`PyModuleDef` struct, and update the full class name in the :c:type:`PyTypeObject` struct. -Finally, we update our :file:`setup.py` file to build the new module: +Finally, we update our :file:`setup.py` file to include the new module, .. code-block:: python - from distutils.core import setup, Extension - setup(name="custom", version="1.0", - ext_modules=[ - Extension("custom", ["custom.c"]), - Extension("custom2", ["custom2.c"]), - ]) + from setuptools import Extension, setup + setup(ext_modules=[ + Extension("custom", ["custom.c"]), + Extension("custom2", ["custom2.c"]), + ]) + +and then we re-install so that we can ``import custom2``: + +.. code-block:: shell-session + $ python -m pip install . Providing finer control over data attributes ============================================ @@ -537,7 +537,7 @@ version of our module, the instance variables :attr:`!first` and :attr:`!last` could be set to non-string values or even deleted. We want to make sure that these attributes always contain strings. -.. literalinclude:: ../includes/custom3.c +.. literalinclude:: ../includes/newtypes/custom3.c To provide greater control, over the :attr:`!first` and :attr:`!last` attributes, @@ -684,7 +684,7 @@ To allow a :class:`!Custom` instance participating in a reference cycle to be properly detected and collected by the cyclic GC, our :class:`!Custom` type needs to fill two additional slots and to enable a flag that enables these slots: -.. literalinclude:: ../includes/custom4.c +.. literalinclude:: ../includes/newtypes/custom4.c First, the traversal method lets the cyclic GC know about subobjects that could @@ -808,7 +808,7 @@ increases an internal counter: >>> print(s.increment()) 2 -.. literalinclude:: ../includes/sublist.c +.. literalinclude:: ../includes/newtypes/sublist.c As you can see, the source code closely resembles the :class:`!Custom` examples in diff --git a/Doc/includes/custom.c b/Doc/includes/newtypes/custom.c similarity index 100% rename from Doc/includes/custom.c rename to Doc/includes/newtypes/custom.c diff --git a/Doc/includes/custom2.c b/Doc/includes/newtypes/custom2.c similarity index 100% rename from Doc/includes/custom2.c rename to Doc/includes/newtypes/custom2.c diff --git a/Doc/includes/custom3.c b/Doc/includes/newtypes/custom3.c similarity index 100% rename from Doc/includes/custom3.c rename to Doc/includes/newtypes/custom3.c diff --git a/Doc/includes/custom4.c b/Doc/includes/newtypes/custom4.c similarity index 100% rename from Doc/includes/custom4.c rename to Doc/includes/newtypes/custom4.c diff --git a/Doc/includes/newtypes/pyproject.toml b/Doc/includes/newtypes/pyproject.toml new file mode 100644 index 0000000000000..ea7937a317147 --- /dev/null +++ b/Doc/includes/newtypes/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "custom" +version = "1" diff --git a/Doc/includes/newtypes/setup.py b/Doc/includes/newtypes/setup.py new file mode 100644 index 0000000000000..67f83673bcc65 --- /dev/null +++ b/Doc/includes/newtypes/setup.py @@ -0,0 +1,8 @@ +from setuptools import Extension, setup +setup(ext_modules=[ + Extension("custom", ["custom.c"]), + Extension("custom2", ["custom2.c"]), + Extension("custom3", ["custom3.c"]), + Extension("custom4", ["custom4.c"]), + Extension("sublist", ["sublist.c"]), +]) diff --git a/Doc/includes/sublist.c b/Doc/includes/newtypes/sublist.c similarity index 100% rename from Doc/includes/sublist.c rename to Doc/includes/newtypes/sublist.c diff --git a/Doc/includes/test.py b/Doc/includes/newtypes/test.py similarity index 94% rename from Doc/includes/test.py rename to Doc/includes/newtypes/test.py index 09ebe3fec0bdb..55a5cf9f68b94 100644 --- a/Doc/includes/test.py +++ b/Doc/includes/newtypes/test.py @@ -187,13 +187,6 @@ >>> gc.enable() """ -import os -import sys -from distutils.util import get_platform -PLAT_SPEC = "%s-%d.%d" % (get_platform(), *sys.version_info[:2]) -src = os.path.join("build", "lib.%s" % PLAT_SPEC) -sys.path.append(src) - if __name__ == "__main__": import doctest, __main__ doctest.testmod(__main__) diff --git a/Doc/includes/setup.py b/Doc/includes/setup.py deleted file mode 100644 index a38a39de3e7c8..0000000000000 --- a/Doc/includes/setup.py +++ /dev/null @@ -1,9 +0,0 @@ -from distutils.core import setup, Extension -setup(name="noddy", version="1.0", - ext_modules=[ - Extension("noddy", ["noddy.c"]), - Extension("noddy2", ["noddy2.c"]), - Extension("noddy3", ["noddy3.c"]), - Extension("noddy4", ["noddy4.c"]), - Extension("shoddy", ["shoddy.c"]), - ]) From webhook-mailer at python.org Tue Aug 22 19:39:53 2023 From: webhook-mailer at python.org (vstinner) Date: Tue, 22 Aug 2023 23:39:53 -0000 Subject: [Python-checkins] gh-90791: test.pythoninfo logs ASAN_OPTIONS env var (#108289) Message-ID: https://github.com/python/cpython/commit/3a1ac87f8f89d3206b46a0df4908afae629d669d commit: 3a1ac87f8f89d3206b46a0df4908afae629d669d branch: main author: Victor Stinner committer: vstinner date: 2023-08-23T01:39:50+02:00 summary: gh-90791: test.pythoninfo logs ASAN_OPTIONS env var (#108289) * Cleanup libregrtest code logging ASAN_OPTIONS. * Fix a typo on "ASAN_OPTIONS" vs "MSAN_OPTIONS". files: M Lib/test/libregrtest/main.py M Lib/test/pythoninfo.py M Lib/test/support/__init__.py diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9001ca33b6166..361189199d382 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -526,26 +526,33 @@ def display_header(self): print("== CPU count:", cpu_count) print("== encodings: locale=%s, FS=%s" % (locale.getencoding(), sys.getfilesystemencoding())) + self.display_sanitizers() + + def display_sanitizers(self): + # This makes it easier to remember what to set in your local + # environment when trying to reproduce a sanitizer failure. asan = support.check_sanitizer(address=True) msan = support.check_sanitizer(memory=True) ubsan = support.check_sanitizer(ub=True) - # This makes it easier to remember what to set in your local - # environment when trying to reproduce a sanitizer failure. - if asan or msan or ubsan: - names = [n for n in (asan and "address", - msan and "memory", - ubsan and "undefined behavior") - if n] - print(f"== sanitizers: {', '.join(names)}") - a_opts = os.environ.get("ASAN_OPTIONS") - if asan and a_opts is not None: - print(f"== ASAN_OPTIONS={a_opts}") - m_opts = os.environ.get("ASAN_OPTIONS") - if msan and m_opts is not None: - print(f"== MSAN_OPTIONS={m_opts}") - ub_opts = os.environ.get("UBSAN_OPTIONS") - if ubsan and ub_opts is not None: - print(f"== UBSAN_OPTIONS={ub_opts}") + sanitizers = [] + if asan: + sanitizers.append("address") + if msan: + sanitizers.append("memory") + if ubsan: + sanitizers.append("undefined behavior") + if not sanitizers: + return + + print(f"== sanitizers: {', '.join(sanitizers)}") + for sanitizer, env_var in ( + (asan, "ASAN_OPTIONS"), + (msan, "MSAN_OPTIONS"), + (ubsan, "UBSAN_OPTIONS"), + ): + options= os.environ.get(env_var) + if sanitizer and options is not None: + print(f"== {env_var}={options!r}") def no_tests_run(self): return not any((self.good, self.bad, self.skipped, self.interrupted, diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index ad7d5291af42f..53af21db0755b 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -308,6 +308,13 @@ def format_groups(groups): "_PYTHON_PROJECT_BASE", "_PYTHON_SYSCONFIGDATA_NAME", "__PYVENV_LAUNCHER__", + + # Sanitizer options + "ASAN_OPTIONS", + "LSAN_OPTIONS", + "MSAN_OPTIONS", + "TSAN_OPTIONS", + "UBSAN_OPTIONS", )) for name, value in os.environ.items(): uname = name.upper() diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 64c66d8e25d9c..8e8bf49a5cb5c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -407,19 +407,19 @@ def check_sanitizer(*, address=False, memory=False, ub=False): raise ValueError('At least one of address, memory, or ub must be True') - _cflags = sysconfig.get_config_var('CFLAGS') or '' - _config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' + cflags = sysconfig.get_config_var('CFLAGS') or '' + config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' memory_sanitizer = ( - '-fsanitize=memory' in _cflags or - '--with-memory-sanitizer' in _config_args + '-fsanitize=memory' in cflags or + '--with-memory-sanitizer' in config_args ) address_sanitizer = ( - '-fsanitize=address' in _cflags or - '--with-address-sanitizer' in _config_args + '-fsanitize=address' in cflags or + '--with-address-sanitizer' in config_args ) ub_sanitizer = ( - '-fsanitize=undefined' in _cflags or - '--with-undefined-behavior-sanitizer' in _config_args + '-fsanitize=undefined' in cflags or + '--with-undefined-behavior-sanitizer' in config_args ) return ( (memory and memory_sanitizer) or From webhook-mailer at python.org Tue Aug 22 22:53:00 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 23 Aug 2023 02:53:00 -0000 Subject: [Python-checkins] gh-105776: Fix test_cppext when CC contains -std=c11 option (#108343) Message-ID: https://github.com/python/cpython/commit/9173b2bbe13aeccc075b571da05c653a2a91de1b commit: 9173b2bbe13aeccc075b571da05c653a2a91de1b branch: main author: Victor Stinner committer: vstinner date: 2023-08-23T02:52:56Z summary: gh-105776: Fix test_cppext when CC contains -std=c11 option (#108343) Fix test_cppext when the C compiler command has the "-std=c11" option. Remove "-std=" options from the compiler command. files: A Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst M Lib/test/test_cppext/setup.py diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index 6867094a42043..976633bc33889 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -1,7 +1,9 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. import os +import shlex import sys +import sysconfig from setuptools import setup, Extension @@ -30,6 +32,17 @@ def main(): cppflags = [*CPPFLAGS, f'-std={std}'] + # gh-105776: When "gcc -std=11" is used as the C++ compiler, -std=c11 + # option emits a C++ compiler warning. Remove "-std11" option from the + # CC command. + cmd = (sysconfig.get_config_var('CC') or '') + if cmd is not None: + cmd = shlex.split(cmd) + cmd = [arg for arg in cmd if not arg.startswith('-std=')] + cmd = shlex.join(cmd) + # CC env var overrides sysconfig CC variable in setuptools + os.environ['CC'] = cmd + cpp_ext = Extension( name, sources=[SOURCE], diff --git a/Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst b/Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst new file mode 100644 index 0000000000000..0e0a3aa9b11e6 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst @@ -0,0 +1,2 @@ +Fix test_cppext when the C compiler command ``-std=c11`` option: remove +``-std=`` options from the compiler command. Patch by Victor Stinner. From webhook-mailer at python.org Tue Aug 22 23:11:57 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 23 Aug 2023 03:11:57 -0000 Subject: [Python-checkins] [3.11] gh-108303: Add Lib/test/test_cppext/ sub-directory (#108325) (#108336) Message-ID: https://github.com/python/cpython/commit/a15396146f8b4f7196e858eba577730f66ac3fb6 commit: a15396146f8b4f7196e858eba577730f66ac3fb6 branch: 3.11 author: Victor Stinner committer: vstinner date: 2023-08-23T05:11:53+02:00 summary: [3.11] gh-108303: Add Lib/test/test_cppext/ sub-directory (#108325) (#108336) gh-108303: Add Lib/test/test_cppext/ sub-directory (#108325) * Move test_cppext to its own directory * Rename setup_testcppext.py to setup.py * Rename _testcppext.cpp to extension.cpp * The source (extension.cpp) is now also copied by the test. (cherry picked from commit 21dda09600848ac280481f7c64f8d9516dc69bb2) files: A Lib/test/test_cppext/__init__.py A Lib/test/test_cppext/extension.cpp A Lib/test/test_cppext/setup.py D Lib/test/_testcppext.cpp D Lib/test/setup_testcppext.py D Lib/test/test_cppext.py M Makefile.pre.in diff --git a/Lib/test/test_cppext.py b/Lib/test/test_cppext/__init__.py similarity index 95% rename from Lib/test/test_cppext.py rename to Lib/test/test_cppext/__init__.py index 465894d24e7df..4ce29b7ff2c23 100644 --- a/Lib/test/test_cppext.py +++ b/Lib/test/test_cppext/__init__.py @@ -10,9 +10,7 @@ MS_WINDOWS = (sys.platform == 'win32') - - -SETUP_TESTCPPEXT = support.findfile('setup_testcppext.py') +SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') @support.requires_subprocess() @@ -74,14 +72,14 @@ def run_cmd(operation, cmd): # Build the C++ extension cmd = [python, '-X', 'dev', - SETUP_TESTCPPEXT, 'build_ext', '--verbose'] + SETUP, 'build_ext', '--verbose'] if std_cpp03: cmd.append('-std=c++03') run_cmd('Build', cmd) # Install the C++ extension cmd = [python, '-X', 'dev', - SETUP_TESTCPPEXT, 'install'] + SETUP, 'install'] run_cmd('Install', cmd) # Do a reference run. Until we test that running python diff --git a/Lib/test/_testcppext.cpp b/Lib/test/test_cppext/extension.cpp similarity index 99% rename from Lib/test/_testcppext.cpp rename to Lib/test/test_cppext/extension.cpp index 0e381a78c5cee..58d18ea675933 100644 --- a/Lib/test/_testcppext.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -1,5 +1,7 @@ // gh-91321: Very basic C++ test extension to check that the Python C API is // compatible with C++ and does not emit C++ compiler warnings. +// +// The code is only built, not executed. // Always enable assertions #undef NDEBUG diff --git a/Lib/test/setup_testcppext.py b/Lib/test/test_cppext/setup.py similarity index 93% rename from Lib/test/setup_testcppext.py rename to Lib/test/test_cppext/setup.py index c6b68104d1333..dac3a96609ebc 100644 --- a/Lib/test/setup_testcppext.py +++ b/Lib/test/test_cppext/setup.py @@ -1,7 +1,7 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. +import os.path import sys -from test import support from setuptools import setup, Extension @@ -9,7 +9,7 @@ MS_WINDOWS = (sys.platform == 'win32') -SOURCE = support.findfile('_testcppext.cpp') +SOURCE = os.path.join(os.path.dirname(__file__), 'extension.cpp') if not MS_WINDOWS: # C++ compiler flags for GCC and clang CPPFLAGS = [ diff --git a/Makefile.pre.in b/Makefile.pre.in index 6405d06afe14e..884bea1841c23 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1959,6 +1959,7 @@ TESTSUBDIRS= ctypes/test \ test/support \ test/test_asyncio \ test/test_capi \ + test/test_cppext \ test/test_email \ test/test_email/data \ test/test_import \ From webhook-mailer at python.org Tue Aug 22 23:47:44 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 23 Aug 2023 03:47:44 -0000 Subject: [Python-checkins] [3.11] gh-105776: Fix test_cppext when CC contains -std=c11 option (#108343) (#108347) Message-ID: https://github.com/python/cpython/commit/1aff195903ca664b5285f7ac9e999347484d9d3b commit: 1aff195903ca664b5285f7ac9e999347484d9d3b branch: 3.11 author: Victor Stinner committer: vstinner date: 2023-08-23T03:47:41Z summary: [3.11] gh-105776: Fix test_cppext when CC contains -std=c11 option (#108343) (#108347) gh-105776: Fix test_cppext when CC contains -std=c11 option (#108343) Fix test_cppext when the C compiler command has the "-std=c11" option. Remove "-std=" options from the compiler command. (cherry picked from commit 9173b2bbe13aeccc075b571da05c653a2a91de1b) files: A Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst M Lib/test/test_cppext/setup.py diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index dac3a96609ebc..f74124775acd0 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -1,7 +1,9 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. import os.path +import shlex import sys +import sysconfig from setuptools import setup, Extension @@ -36,6 +38,17 @@ def main(): cppflags = [*CPPFLAGS, f'-std={std}'] + # gh-105776: When "gcc -std=11" is used as the C++ compiler, -std=c11 + # option emits a C++ compiler warning. Remove "-std11" option from the + # CC command. + cmd = (sysconfig.get_config_var('CC') or '') + if cmd is not None: + cmd = shlex.split(cmd) + cmd = [arg for arg in cmd if not arg.startswith('-std=')] + cmd = shlex.join(cmd) + # CC env var overrides sysconfig CC variable in setuptools + os.environ['CC'] = cmd + cpp_ext = Extension( name, sources=[SOURCE], diff --git a/Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst b/Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst new file mode 100644 index 0000000000000..0e0a3aa9b11e6 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst @@ -0,0 +1,2 @@ +Fix test_cppext when the C compiler command ``-std=c11`` option: remove +``-std=`` options from the compiler command. Patch by Victor Stinner. From webhook-mailer at python.org Wed Aug 23 01:26:04 2023 From: webhook-mailer at python.org (gpshead) Date: Wed, 23 Aug 2023 05:26:04 -0000 Subject: [Python-checkins] gh-108342: Break ref cycle in SSLSocket._create() exc (#108344) Message-ID: https://github.com/python/cpython/commit/64f99350351bc46e016b2286f36ba7cd669b79e3 commit: 64f99350351bc46e016b2286f36ba7cd669b79e3 branch: main author: Victor Stinner committer: gpshead date: 2023-08-23T05:26:01Z summary: gh-108342: Break ref cycle in SSLSocket._create() exc (#108344) Explicitly break a reference cycle when SSLSocket._create() raises an exception. Clear the variable storing the exception, since the exception traceback contains the variables and so creates a reference cycle. This test leak was introduced by the test added for the fix of #108310. files: M Lib/ssl.py diff --git a/Lib/ssl.py b/Lib/ssl.py index ff363c75e7dfd..c4c5a4ca894ee 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1021,7 +1021,11 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, self.close() except OSError: pass - raise notconn_pre_handshake_data_error + try: + raise notconn_pre_handshake_data_error + finally: + # Explicitly break the reference cycle. + notconn_pre_handshake_data_error = None else: connected = True From webhook-mailer at python.org Wed Aug 23 03:11:18 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Wed, 23 Aug 2023 07:11:18 -0000 Subject: [Python-checkins] gh-108267: Dataclasses docs: Fix object.__setattr__ typo (#108355) Message-ID: https://github.com/python/cpython/commit/79fdacc0059a3959074d2d9d054653eae1dcfe06 commit: 79fdacc0059a3959074d2d9d054653eae1dcfe06 branch: main author: FrozenBob <30644137+FrozenBob at users.noreply.github.com> committer: AlexWaygood date: 2023-08-23T08:11:15+01:00 summary: gh-108267: Dataclasses docs: Fix object.__setattr__ typo (#108355) Fixed a sentence in dataclasses.rst Changed "__setattr__" to "object.__setattr__" in a section that was specifically supposed to refer to the __setattr__ method of the object class. Also suppressed the link to the data model docs for __setattr__, since we're talking about a specific __setattr__ implementation, not __setattr__ methods in general. files: M Doc/library/dataclasses.rst diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 535a60ccca8d0..d68748767c5e3 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -609,7 +609,7 @@ methods will raise a :exc:`FrozenInstanceError` when invoked. There is a tiny performance penalty when using ``frozen=True``: :meth:`~object.__init__` cannot use simple assignment to initialize fields, and -must use :meth:`~object.__setattr__`. +must use :meth:`!object.__setattr__`. Inheritance ----------- From webhook-mailer at python.org Wed Aug 23 03:21:04 2023 From: webhook-mailer at python.org (AlexWaygood) Date: Wed, 23 Aug 2023 07:21:04 -0000 Subject: [Python-checkins] [3.11] gh-108267: Dataclasses docs: Fix object.__setattr__ typo (GH-108355) (#108357) Message-ID: https://github.com/python/cpython/commit/1ecbe787dde59b3d021990dbf92680f0301977ce commit: 1ecbe787dde59b3d021990dbf92680f0301977ce branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: AlexWaygood date: 2023-08-23T07:20:57Z summary: [3.11] gh-108267: Dataclasses docs: Fix object.__setattr__ typo (GH-108355) (#108357) gh-108267: Dataclasses docs: Fix object.__setattr__ typo (GH-108355) Fixed a sentence in dataclasses.rst Changed "__setattr__" to "object.__setattr__" in a section that was specifically supposed to refer to the __setattr__ method of the object class. Also suppressed the link to the data model docs for __setattr__, since we're talking about a specific __setattr__ implementation, not __setattr__ methods in general. (cherry picked from commit 79fdacc0059a3959074d2d9d054653eae1dcfe06) Co-authored-by: FrozenBob <30644137+FrozenBob at users.noreply.github.com> files: M Doc/library/dataclasses.rst diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index aa71e5e735f2c..09f4ce812dc83 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -604,7 +604,7 @@ methods will raise a :exc:`FrozenInstanceError` when invoked. There is a tiny performance penalty when using ``frozen=True``: :meth:`~object.__init__` cannot use simple assignment to initialize fields, and -must use :meth:`~object.__setattr__`. +must use :meth:`!object.__setattr__`. Inheritance ----------- From webhook-mailer at python.org Wed Aug 23 04:01:20 2023 From: webhook-mailer at python.org (iritkatriel) Date: Wed, 23 Aug 2023 08:01:20 -0000 Subject: [Python-checkins] gh-108113: Make it possible to optimize an AST (#108282) Message-ID: https://github.com/python/cpython/commit/2dfbd4f36dd83f88f5df64c33612dd34eff256bb commit: 2dfbd4f36dd83f88f5df64c33612dd34eff256bb branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-23T09:01:17+01:00 summary: gh-108113: Make it possible to optimize an AST (#108282) files: M Include/internal/pycore_compile.h M Lib/test/test_ast.py M Lib/test/test_builtin.py M Python/bltinmodule.c M Python/compile.c M Python/pythonrun.c diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index ad657c0f0fced..0167e590e63fb 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -19,6 +19,14 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( int optimize, struct _arena *arena); +/* AST optimizations */ +PyAPI_FUNC(int) _PyCompile_AstOptimize( + struct _mod *mod, + PyObject *filename, + PyCompilerFlags *flags, + int optimize, + struct _arena *arena); + static const _PyCompilerSrcLocation NO_LOCATION = {-1, -1, -1, -1}; extern int _PyAST_Optimize( diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index f3c7229f0b6c7..68de4d6f3f192 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -361,14 +361,16 @@ def test_optimization_levels__debug__(self): cases = [(-1, '__debug__'), (0, '__debug__'), (1, False), (2, False)] for (optval, expected) in cases: with self.subTest(optval=optval, expected=expected): - res = ast.parse("__debug__", optimize=optval) - self.assertIsInstance(res.body[0], ast.Expr) - if isinstance(expected, bool): - self.assertIsInstance(res.body[0].value, ast.Constant) - self.assertEqual(res.body[0].value.value, expected) - else: - self.assertIsInstance(res.body[0].value, ast.Name) - self.assertEqual(res.body[0].value.id, expected) + res1 = ast.parse("__debug__", optimize=optval) + res2 = ast.parse(ast.parse("__debug__"), optimize=optval) + for res in [res1, res2]: + self.assertIsInstance(res.body[0], ast.Expr) + if isinstance(expected, bool): + self.assertIsInstance(res.body[0].value, ast.Constant) + self.assertEqual(res.body[0].value.value, expected) + else: + self.assertIsInstance(res.body[0].value, ast.Name) + self.assertEqual(res.body[0].value.id, expected) def test_optimization_levels_const_folding(self): folded = ('Expr', (1, 0, 1, 5), ('Constant', (1, 0, 1, 5), 3, None)) @@ -381,9 +383,11 @@ def test_optimization_levels_const_folding(self): cases = [(-1, not_folded), (0, not_folded), (1, folded), (2, folded)] for (optval, expected) in cases: with self.subTest(optval=optval): - tree = ast.parse("1 + 2", optimize=optval) - res = to_tuple(tree.body[0]) - self.assertEqual(res, expected) + tree1 = ast.parse("1 + 2", optimize=optval) + tree2 = ast.parse(ast.parse("1 + 2"), optimize=optval) + for tree in [tree1, tree2]: + res = to_tuple(tree.body[0]) + self.assertEqual(res, expected) def test_invalid_position_information(self): invalid_linenos = [ diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index ee3ba6ab07bbd..dbc706ae7b41f 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -521,9 +521,10 @@ def test_compile_async_generator(self): def test_compile_ast(self): args = ("a*(1+2)", "f.py", "exec") raw = compile(*args, flags = ast.PyCF_ONLY_AST).body[0] - opt = compile(*args, flags = ast.PyCF_OPTIMIZED_AST).body[0] + opt1 = compile(*args, flags = ast.PyCF_OPTIMIZED_AST).body[0] + opt2 = compile(ast.parse(args[0]), *args[1:], flags = ast.PyCF_OPTIMIZED_AST).body[0] - for tree in (raw, opt): + for tree in (raw, opt1, opt2): self.assertIsInstance(tree.value, ast.BinOp) self.assertIsInstance(tree.value.op, ast.Mult) self.assertIsInstance(tree.value.left, ast.Name) @@ -536,9 +537,10 @@ def test_compile_ast(self): self.assertIsInstance(raw_right.right, ast.Constant) self.assertEqual(raw_right.right.value, 2) - opt_right = opt.value.right # expect Constant(3) - self.assertIsInstance(opt_right, ast.Constant) - self.assertEqual(opt_right.value, 3) + for opt in [opt1, opt2]: + opt_right = opt.value.right # expect Constant(3) + self.assertIsInstance(opt_right, ast.Constant) + self.assertEqual(opt_right.value, 3) def test_delattr(self): sys.spam = 1 diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d06efcf6ba368..787f53fae354d 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -804,23 +804,40 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, if (is_ast == -1) goto error; if (is_ast) { - if (flags & PyCF_ONLY_AST) { + if ((flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST) { + // return an un-optimized AST result = Py_NewRef(source); } else { - PyArena *arena; - mod_ty mod; + // Return an optimized AST or code object - arena = _PyArena_New(); - if (arena == NULL) - goto error; - mod = PyAST_obj2mod(source, arena, compile_mode); - if (mod == NULL || !_PyAST_Validate(mod)) { - _PyArena_Free(arena); + PyArena *arena = _PyArena_New(); + if (arena == NULL) { goto error; } - result = (PyObject*)_PyAST_Compile(mod, filename, - &cf, optimize, arena); + + if (flags & PyCF_ONLY_AST) { + mod_ty mod = PyAST_obj2mod(source, arena, compile_mode); + if (mod == NULL || !_PyAST_Validate(mod)) { + _PyArena_Free(arena); + goto error; + } + if (_PyCompile_AstOptimize(mod, filename, &cf, optimize, + arena) < 0) { + _PyArena_Free(arena); + goto error; + } + result = PyAST_mod2obj(mod); + } + else { + mod_ty mod = PyAST_obj2mod(source, arena, compile_mode); + if (mod == NULL || !_PyAST_Validate(mod)) { + _PyArena_Free(arena); + goto error; + } + result = (PyObject*)_PyAST_Compile(mod, filename, + &cf, optimize, arena); + } _PyArena_Free(arena); } goto finally; diff --git a/Python/compile.c b/Python/compile.c index 3260dba57eac8..4b2f70a7ef01d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -557,6 +557,24 @@ _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, return co; } +int +_PyCompile_AstOptimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, + int optimize, PyArena *arena) +{ + PyFutureFeatures future; + if (!_PyFuture_FromAST(mod, filename, &future)) { + return -1; + } + int flags = future.ff_features | cf->cf_flags; + if (optimize == -1) { + optimize = _Py_GetConfig()->optimization_level; + } + if (!_PyAST_Optimize(mod, arena, optimize, flags)) { + return -1; + } + return 0; +} + static void compiler_free(struct compiler *c) { diff --git a/Python/pythonrun.c b/Python/pythonrun.c index a3de7792cf5bc..05b7dfa61d1a5 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -22,7 +22,6 @@ #include "pycore_pyerrors.h" // _PyErr_GetRaisedException, _Py_Offer_Suggestions #include "pycore_pylifecycle.h" // _Py_UnhandledKeyboardInterrupt #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_symtable.h" // _PyFuture_FromAST() #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_traceback.h" // _PyTraceBack_Print_Indented() @@ -1792,24 +1791,6 @@ run_pyc_file(FILE *fp, PyObject *globals, PyObject *locals, return NULL; } -static int -ast_optimize(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, - int optimize, PyArena *arena) -{ - PyFutureFeatures future; - if (!_PyFuture_FromAST(mod, filename, &future)) { - return -1; - } - int flags = future.ff_features | cf->cf_flags; - if (optimize == -1) { - optimize = _Py_GetConfig()->optimization_level; - } - if (!_PyAST_Optimize(mod, arena, optimize, flags)) { - return -1; - } - return 0; -} - PyObject * Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) @@ -1827,8 +1808,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, } if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { if ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_OPTIMIZED_AST) { - if (ast_optimize(mod, filename, flags, optimize, arena) < 0) { - _PyArena_Free(arena); + if (_PyCompile_AstOptimize(mod, filename, flags, optimize, arena) < 0) { return NULL; } } From webhook-mailer at python.org Wed Aug 23 05:00:26 2023 From: webhook-mailer at python.org (encukou) Date: Wed, 23 Aug 2023 09:00:26 -0000 Subject: [Python-checkins] gh-108294: Add time.sleep audit event (GH-108298) Message-ID: https://github.com/python/cpython/commit/31b61d19abcc63aa28625a31ed75411948fc1e7e commit: 31b61d19abcc63aa28625a31ed75411948fc1e7e branch: main author: Petr Viktorin committer: encukou date: 2023-08-23T11:00:22+02:00 summary: gh-108294: Add time.sleep audit event (GH-108298) files: A Misc/NEWS.d/next/Library/2023-08-22-16-18-49.gh-issue-108294.KEeUcM.rst M Doc/library/time.rst M Lib/test/audit-tests.py M Lib/test/test_audit.py M Modules/timemodule.c diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 9f23a6fc7d534..6ffe4ac428414 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -379,6 +379,8 @@ Functions * Or use ``nanosleep()`` if available (resolution: 1 nanosecond); * Or use ``select()`` (resolution: 1 microsecond). + .. audit-event:: time.sleep secs + .. versionchanged:: 3.11 On Unix, the ``clock_nanosleep()`` and ``nanosleep()`` functions are now used if available. On Windows, a waitable timer is now used. @@ -389,6 +391,9 @@ Functions :pep:`475` for the rationale). + .. versionchanged:: 3.13 + Raises an auditing event. + .. index:: single: % (percent); datetime format diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index 9504829e96f00..cc614eab90850 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -514,6 +514,21 @@ def test_not_in_gc(): assert hook not in o +def test_time(): + import time + + def hook(event, args): + if event.startswith("time."): + print(event, *args) + sys.addaudithook(hook) + + time.sleep(0) + time.sleep(0.0625) # 1/16, a small exact float + try: + time.sleep(-1) + except ValueError: + pass + def test_sys_monitoring_register_callback(): import sys diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index b12ffa5d872e8..3a15835917cc3 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -256,6 +256,21 @@ def test_not_in_gc(self): if returncode: self.fail(stderr) + def test_time(self): + returncode, events, stderr = self.run_python("test_time") + if returncode: + self.fail(stderr) + + if support.verbose: + print(*events, sep='\n') + + actual = [(ev[0], ev[2]) for ev in events] + expected = [("time.sleep", "0"), + ("time.sleep", "0.0625"), + ("time.sleep", "-1")] + + self.assertEqual(actual, expected) + def test_sys_monitoring_register_callback(self): returncode, events, stderr = self.run_python("test_sys_monitoring_register_callback") diff --git a/Misc/NEWS.d/next/Library/2023-08-22-16-18-49.gh-issue-108294.KEeUcM.rst b/Misc/NEWS.d/next/Library/2023-08-22-16-18-49.gh-issue-108294.KEeUcM.rst new file mode 100644 index 0000000000000..de2a3a8a8ad89 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-22-16-18-49.gh-issue-108294.KEeUcM.rst @@ -0,0 +1 @@ +:func:`time.sleep` now raises an auditing event. diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 912710219bd01..68948b6be1a61 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -414,6 +414,8 @@ Return the clk_id of a thread's CPU time clock."); static PyObject * time_sleep(PyObject *self, PyObject *timeout_obj) { + PySys_Audit("time.sleep", "O", timeout_obj); + _PyTime_t timeout; if (_PyTime_FromSecondsObject(&timeout, timeout_obj, _PyTime_ROUND_TIMEOUT)) return NULL; From webhook-mailer at python.org Wed Aug 23 05:01:23 2023 From: webhook-mailer at python.org (ambv) Date: Wed, 23 Aug 2023 09:01:23 -0000 Subject: [Python-checkins] [3.12] gh-107136: Remove Plausible for docs metrics (GH-107856) (#108334) Message-ID: https://github.com/python/cpython/commit/314c98d293765fb6b930c840e6f6ba80f201b3f7 commit: 314c98d293765fb6b930c840e6f6ba80f201b3f7 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2023-08-23T11:01:19+02:00 summary: [3.12] gh-107136: Remove Plausible for docs metrics (GH-107856) (#108334) (cherry picked from commit fc23f34cc9701949e6832eb32f26ea89f6622b82) Co-authored-by: Hugo van Kemenade files: M Doc/tools/templates/layout.html diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 80103158ea01e..9498b2ccc5af9 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,9 +26,6 @@ {% endblock %} {% block extrahead %} - {% if builder == "html" %} - - {% endif %} {% if builder != "htmlhelp" %} {% if pagename == 'whatsnew/changelog' and not embedded %} From webhook-mailer at python.org Wed Aug 23 05:01:33 2023 From: webhook-mailer at python.org (ambv) Date: Wed, 23 Aug 2023 09:01:33 -0000 Subject: [Python-checkins] [3.11] gh-107136: Remove Plausible for docs metrics (GH-107856) (#108335) Message-ID: https://github.com/python/cpython/commit/07c727e92a72ea8a9e7d9705b4afc261c32fa1ce commit: 07c727e92a72ea8a9e7d9705b4afc261c32fa1ce branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2023-08-23T11:01:30+02:00 summary: [3.11] gh-107136: Remove Plausible for docs metrics (GH-107856) (#108335) (cherry picked from commit fc23f34cc9701949e6832eb32f26ea89f6622b82) Co-authored-by: Hugo van Kemenade files: M Doc/tools/templates/layout.html diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 80103158ea01e..9498b2ccc5af9 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,9 +26,6 @@ {% endblock %} {% block extrahead %} - {% if builder == "html" %} - - {% endif %} {% if builder != "htmlhelp" %} {% if pagename == 'whatsnew/changelog' and not embedded %} From webhook-mailer at python.org Wed Aug 23 06:09:58 2023 From: webhook-mailer at python.org (ambv) Date: Wed, 23 Aug 2023 10:09:58 -0000 Subject: [Python-checkins] [3.12] gh-108342: Break ref cycle in SSLSocket._create() exc (GH-108344) (#108348) Message-ID: https://github.com/python/cpython/commit/61dbf46ae3df9d772bfd91b89ab61baeff97a12d commit: 61dbf46ae3df9d772bfd91b89ab61baeff97a12d branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2023-08-23T12:09:55+02:00 summary: [3.12] gh-108342: Break ref cycle in SSLSocket._create() exc (GH-108344) (#108348) Explicitly break a reference cycle when SSLSocket._create() raises an exception. Clear the variable storing the exception, since the exception traceback contains the variables and so creates a reference cycle. This test leak was introduced by the test added for the fix of GH-108310. (cherry picked from commit 64f99350351bc46e016b2286f36ba7cd669b79e3) Co-authored-by: Victor Stinner files: M Lib/ssl.py diff --git a/Lib/ssl.py b/Lib/ssl.py index ff363c75e7dfd..c4c5a4ca894ee 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1021,7 +1021,11 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, self.close() except OSError: pass - raise notconn_pre_handshake_data_error + try: + raise notconn_pre_handshake_data_error + finally: + # Explicitly break the reference cycle. + notconn_pre_handshake_data_error = None else: connected = True From webhook-mailer at python.org Wed Aug 23 06:10:08 2023 From: webhook-mailer at python.org (ambv) Date: Wed, 23 Aug 2023 10:10:08 -0000 Subject: [Python-checkins] [3.11] gh-108342: Break ref cycle in SSLSocket._create() exc (GH-108344) (#108349) Message-ID: https://github.com/python/cpython/commit/93714b7db795b14b26adffde30753cfda0ca4867 commit: 93714b7db795b14b26adffde30753cfda0ca4867 branch: 3.11 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2023-08-23T12:10:04+02:00 summary: [3.11] gh-108342: Break ref cycle in SSLSocket._create() exc (GH-108344) (#108349) Explicitly break a reference cycle when SSLSocket._create() raises an exception. Clear the variable storing the exception, since the exception traceback contains the variables and so creates a reference cycle. This test leak was introduced by the test added for the fix of GH-108310. (cherry picked from commit 64f99350351bc46e016b2286f36ba7cd669b79e3) Co-authored-by: Victor Stinner files: M Lib/ssl.py diff --git a/Lib/ssl.py b/Lib/ssl.py index ced87d406995c..48d229f810af2 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1083,7 +1083,11 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, self.close() except OSError: pass - raise notconn_pre_handshake_data_error + try: + raise notconn_pre_handshake_data_error + finally: + # Explicitly break the reference cycle. + notconn_pre_handshake_data_error = None else: connected = True From webhook-mailer at python.org Wed Aug 23 06:10:11 2023 From: webhook-mailer at python.org (erlend-aasland) Date: Wed, 23 Aug 2023 10:10:11 -0000 Subject: [Python-checkins] gh-105539: Fix ResourceWarning from unclosed SQLite connections in test_sqlite3 (#108360) Message-ID: https://github.com/python/cpython/commit/29bc6165ab8aa434145a34676b8b7e48e7c6e308 commit: 29bc6165ab8aa434145a34676b8b7e48e7c6e308 branch: main author: Mariusz Felisiak committer: erlend-aasland date: 2023-08-23T10:10:08Z summary: gh-105539: Fix ResourceWarning from unclosed SQLite connections in test_sqlite3 (#108360) Follow up to 1a1bfc28912a39b500c578e9f10a8a222638d411. Explicitly manage connections in: - test_audit.test_sqlite3 - test_sqlite3.test_audit - test_sqlite3.test_backup Co-authored-by: Erlend E. Aasland files: M Lib/test/audit-tests.py M Lib/test/test_sqlite3/test_backup.py M Lib/test/test_sqlite3/test_dbapi.py diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index cc614eab90850..ad8f72f556331 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -398,15 +398,18 @@ def hook(event, *args): cx2 = sqlite3.Connection(":memory:") # Configured without --enable-loadable-sqlite-extensions - if hasattr(sqlite3.Connection, "enable_load_extension"): - cx1.enable_load_extension(False) - try: - cx1.load_extension("test") - except sqlite3.OperationalError: - pass - else: - raise RuntimeError("Expected sqlite3.load_extension to fail") - + try: + if hasattr(sqlite3.Connection, "enable_load_extension"): + cx1.enable_load_extension(False) + try: + cx1.load_extension("test") + except sqlite3.OperationalError: + pass + else: + raise RuntimeError("Expected sqlite3.load_extension to fail") + finally: + cx1.close() + cx2.close() def test_sys_getframe(): import sys diff --git a/Lib/test/test_sqlite3/test_backup.py b/Lib/test/test_sqlite3/test_backup.py index 4584d976bce0c..c7400d8b2165e 100644 --- a/Lib/test/test_sqlite3/test_backup.py +++ b/Lib/test/test_sqlite3/test_backup.py @@ -137,7 +137,7 @@ def progress(status, remaining, total): raise SystemError('nearly out of space') with self.assertRaises(SystemError) as err: - with sqlite.connect(':memory:') as bck: + with memory_database() as bck: self.cx.backup(bck, progress=progress) self.assertEqual(str(err.exception), 'nearly out of space') diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index d80ad7af3200f..9dedbdbc4bb6d 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -40,6 +40,7 @@ from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE, unlink, temp_dir, FakePath from .util import memory_database, cx_limit +from .util import MemoryDatabaseMixin class ModuleTests(unittest.TestCase): @@ -1740,10 +1741,9 @@ def test_closed_call(self): self.check(self.con) -class ClosedCurTests(unittest.TestCase): +class ClosedCurTests(MemoryDatabaseMixin, unittest.TestCase): def test_closed(self): - con = sqlite.connect(":memory:") - cur = con.cursor() + cur = self.cx.cursor() cur.close() for method_name in ("execute", "executemany", "executescript", "fetchall", "fetchmany", "fetchone"): From webhook-mailer at python.org Wed Aug 23 06:10:17 2023 From: webhook-mailer at python.org (ambv) Date: Wed, 23 Aug 2023 10:10:17 -0000 Subject: [Python-checkins] [3.10] gh-108342: Break ref cycle in SSLSocket._create() exc (GH-108344) (#108350) Message-ID: https://github.com/python/cpython/commit/893c3b7f5c52b5c0800b72a78fb799632efcd628 commit: 893c3b7f5c52b5c0800b72a78fb799632efcd628 branch: 3.10 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2023-08-23T12:10:14+02:00 summary: [3.10] gh-108342: Break ref cycle in SSLSocket._create() exc (GH-108344) (#108350) Explicitly break a reference cycle when SSLSocket._create() raises an exception. Clear the variable storing the exception, since the exception traceback contains the variables and so creates a reference cycle. This test leak was introduced by the test added for the fix of GH-108310. (cherry picked from commit 64f99350351bc46e016b2286f36ba7cd669b79e3) Co-authored-by: Victor Stinner files: M Lib/ssl.py diff --git a/Lib/ssl.py b/Lib/ssl.py index fadbee217afd5..f386fa7831528 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1079,7 +1079,11 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, self.close() except OSError: pass - raise notconn_pre_handshake_data_error + try: + raise notconn_pre_handshake_data_error + finally: + # Explicitly break the reference cycle. + notconn_pre_handshake_data_error = None else: connected = True From webhook-mailer at python.org Wed Aug 23 06:10:53 2023 From: webhook-mailer at python.org (ambv) Date: Wed, 23 Aug 2023 10:10:53 -0000 Subject: [Python-checkins] [3.9] gh-108342: Break ref cycle in SSLSocket._create() exc (GH-108344) (#108351) Message-ID: https://github.com/python/cpython/commit/b8058b3da542101f4a227ef2d6a263a5d73d7973 commit: b8058b3da542101f4a227ef2d6a263a5d73d7973 branch: 3.9 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2023-08-23T12:10:49+02:00 summary: [3.9] gh-108342: Break ref cycle in SSLSocket._create() exc (GH-108344) (#108351) Explicitly break a reference cycle when SSLSocket._create() raises an exception. Clear the variable storing the exception, since the exception traceback contains the variables and so creates a reference cycle. This test leak was introduced by the test added for the fix of GH-108310. (cherry picked from commit 64f99350351bc46e016b2286f36ba7cd669b79e3) Co-authored-by: Victor Stinner files: M Lib/ssl.py diff --git a/Lib/ssl.py b/Lib/ssl.py index ef92e76a7c540..cb5ec51681e1c 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1049,7 +1049,11 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, self.close() except OSError: pass - raise notconn_pre_handshake_data_error + try: + raise notconn_pre_handshake_data_error + finally: + # Explicitly break the reference cycle. + notconn_pre_handshake_data_error = None else: connected = True From webhook-mailer at python.org Wed Aug 23 06:11:00 2023 From: webhook-mailer at python.org (ambv) Date: Wed, 23 Aug 2023 10:11:00 -0000 Subject: [Python-checkins] [3.8] gh-108342: Break ref cycle in SSLSocket._create() exc (GH-108344) (#108352) Message-ID: https://github.com/python/cpython/commit/6f2b64f05195520cb4bd74cebeb37498294f0687 commit: 6f2b64f05195520cb4bd74cebeb37498294f0687 branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: ambv date: 2023-08-23T12:10:56+02:00 summary: [3.8] gh-108342: Break ref cycle in SSLSocket._create() exc (GH-108344) (#108352) Explicitly break a reference cycle when SSLSocket._create() raises an exception. Clear the variable storing the exception, since the exception traceback contains the variables and so creates a reference cycle. This test leak was introduced by the test added for the fix of GH-108310. (cherry picked from commit 64f99350351bc46e016b2286f36ba7cd669b79e3) Co-authored-by: Victor Stinner files: M Lib/ssl.py diff --git a/Lib/ssl.py b/Lib/ssl.py index bd9ba346dae8c..4ccccdeba2a55 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1048,7 +1048,11 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, self.close() except OSError: pass - raise notconn_pre_handshake_data_error + try: + raise notconn_pre_handshake_data_error + finally: + # Explicitly break the reference cycle. + notconn_pre_handshake_data_error = None else: connected = True From webhook-mailer at python.org Wed Aug 23 07:45:40 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 23 Aug 2023 11:45:40 -0000 Subject: [Python-checkins] [3.12] gh-105776: Fix test_cppext when CC contains -std=c11 option (GH-108343) (#108345) Message-ID: https://github.com/python/cpython/commit/0d6e657689c58cbd790dec2f71f0879e1ed3d4a3 commit: 0d6e657689c58cbd790dec2f71f0879e1ed3d4a3 branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-23T13:45:37+02:00 summary: [3.12] gh-105776: Fix test_cppext when CC contains -std=c11 option (GH-108343) (#108345) gh-105776: Fix test_cppext when CC contains -std=c11 option (GH-108343) Fix test_cppext when the C compiler command has the "-std=c11" option. Remove "-std=" options from the compiler command. (cherry picked from commit 9173b2bbe13aeccc075b571da05c653a2a91de1b) Co-authored-by: Victor Stinner files: A Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst M Lib/test/test_cppext/setup.py diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index 6867094a42043..976633bc33889 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -1,7 +1,9 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. import os +import shlex import sys +import sysconfig from setuptools import setup, Extension @@ -30,6 +32,17 @@ def main(): cppflags = [*CPPFLAGS, f'-std={std}'] + # gh-105776: When "gcc -std=11" is used as the C++ compiler, -std=c11 + # option emits a C++ compiler warning. Remove "-std11" option from the + # CC command. + cmd = (sysconfig.get_config_var('CC') or '') + if cmd is not None: + cmd = shlex.split(cmd) + cmd = [arg for arg in cmd if not arg.startswith('-std=')] + cmd = shlex.join(cmd) + # CC env var overrides sysconfig CC variable in setuptools + os.environ['CC'] = cmd + cpp_ext = Extension( name, sources=[SOURCE], diff --git a/Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst b/Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst new file mode 100644 index 0000000000000..0e0a3aa9b11e6 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-08-23-04-08-18.gh-issue-105776.oE6wp_.rst @@ -0,0 +1,2 @@ +Fix test_cppext when the C compiler command ``-std=c11`` option: remove +``-std=`` options from the compiler command. Patch by Victor Stinner. From webhook-mailer at python.org Wed Aug 23 07:46:07 2023 From: webhook-mailer at python.org (Yhg1s) Date: Wed, 23 Aug 2023 11:46:07 -0000 Subject: [Python-checkins] [3.12] gh-108267: Dataclasses docs: Fix object.__setattr__ typo (GH-108355) (#108358) Message-ID: https://github.com/python/cpython/commit/3aa7df6544950c21be57bd4bee5edbef4d196fce commit: 3aa7df6544950c21be57bd4bee5edbef4d196fce branch: 3.12 author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com> committer: Yhg1s date: 2023-08-23T13:46:03+02:00 summary: [3.12] gh-108267: Dataclasses docs: Fix object.__setattr__ typo (GH-108355) (#108358) gh-108267: Dataclasses docs: Fix object.__setattr__ typo (GH-108355) Fixed a sentence in dataclasses.rst Changed "__setattr__" to "object.__setattr__" in a section that was specifically supposed to refer to the __setattr__ method of the object class. Also suppressed the link to the data model docs for __setattr__, since we're talking about a specific __setattr__ implementation, not __setattr__ methods in general. (cherry picked from commit 79fdacc0059a3959074d2d9d054653eae1dcfe06) Co-authored-by: FrozenBob <30644137+FrozenBob at users.noreply.github.com> files: M Doc/library/dataclasses.rst diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 535a60ccca8d0..d68748767c5e3 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -609,7 +609,7 @@ methods will raise a :exc:`FrozenInstanceError` when invoked. There is a tiny performance penalty when using ``frozen=True``: :meth:`~object.__init__` cannot use simple assignment to initialize fields, and -must use :meth:`~object.__setattr__`. +must use :meth:`!object.__setattr__`. Inheritance ----------- From webhook-mailer at python.org Wed Aug 23 08:23:45 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 23 Aug 2023 12:23:45 -0000 Subject: [Python-checkins] gh-50002: xml.dom.minidom now preserves whitespaces in attributes (GH-107947) Message-ID: https://github.com/python/cpython/commit/154477be722ae5c4e18d22d0860e284006b09c4f commit: 154477be722ae5c4e18d22d0860e284006b09c4f branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-23T15:23:41+03:00 summary: gh-50002: xml.dom.minidom now preserves whitespaces in attributes (GH-107947) Also double quotes (") are now only quoted in attributes. files: A Misc/NEWS.d/next/Library/2023-08-14-20-01-14.gh-issue-50002.E-bpj8.rst A Misc/NEWS.d/next/Library/2023-08-14-20-18-59.gh-issue-81555.cWdP4a.rst M Lib/test/test_minidom.py M Lib/xml/dom/minidom.py diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 699265ccadc7f..3ecd1af31eea7 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -505,6 +505,46 @@ def testWriteXML(self): dom.unlink() self.confirm(str == domstr) + def test_toxml_quote_text(self): + dom = Document() + elem = dom.appendChild(dom.createElement('elem')) + elem.appendChild(dom.createTextNode('&<>"')) + cr = elem.appendChild(dom.createElement('cr')) + cr.appendChild(dom.createTextNode('\r')) + crlf = elem.appendChild(dom.createElement('crlf')) + crlf.appendChild(dom.createTextNode('\r\n')) + lflf = elem.appendChild(dom.createElement('lflf')) + lflf.appendChild(dom.createTextNode('\n\n')) + ws = elem.appendChild(dom.createElement('ws')) + ws.appendChild(dom.createTextNode('\t\n\r ')) + domstr = dom.toxml() + dom.unlink() + self.assertEqual(domstr, '' + '&<>"' + '\r' + '\r\n' + '\n\n' + '\t\n\r ') + + def test_toxml_quote_attrib(self): + dom = Document() + elem = dom.appendChild(dom.createElement('elem')) + elem.setAttribute("a", '&<>"') + elem.setAttribute("cr", "\r") + elem.setAttribute("lf", "\n") + elem.setAttribute("crlf", "\r\n") + elem.setAttribute("lflf", "\n\n") + elem.setAttribute("ws", "\t\n\r ") + domstr = dom.toxml() + dom.unlink() + self.assertEqual(domstr, '' + '') + def testAltNewline(self): str = '\n\n' dom = parseString(str) diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index ef8a159833bbc..db51f350ea015 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -300,12 +300,28 @@ def _in_document(node): node = node.parentNode return False -def _write_data(writer, data): +def _write_data(writer, text, attr): "Writes datachars to writer." - if data: - data = data.replace("&", "&").replace("<", "<"). \ - replace("\"", """).replace(">", ">") - writer.write(data) + if not text: + return + # See the comments in ElementTree.py for behavior and + # implementation details. + if "&" in text: + text = text.replace("&", "&") + if "<" in text: + text = text.replace("<", "<") + if ">" in text: + text = text.replace(">", ">") + if attr: + if '"' in text: + text = text.replace('"', """) + if "\r" in text: + text = text.replace("\r", " ") + if "\n" in text: + text = text.replace("\n", " ") + if "\t" in text: + text = text.replace("\t", " ") + writer.write(text) def _get_elements_by_tagName_helper(parent, name, rc): for node in parent.childNodes: @@ -883,7 +899,7 @@ def writexml(self, writer, indent="", addindent="", newl=""): for a_name in attrs.keys(): writer.write(" %s=\"" % a_name) - _write_data(writer, attrs[a_name].value) + _write_data(writer, attrs[a_name].value, True) writer.write("\"") if self.childNodes: writer.write(">") @@ -1112,7 +1128,7 @@ def splitText(self, offset): return newText def writexml(self, writer, indent="", addindent="", newl=""): - _write_data(writer, "%s%s%s" % (indent, self.data, newl)) + _write_data(writer, "%s%s%s" % (indent, self.data, newl), False) # DOM Level 3 (WD 9 April 2002) diff --git a/Misc/NEWS.d/next/Library/2023-08-14-20-01-14.gh-issue-50002.E-bpj8.rst b/Misc/NEWS.d/next/Library/2023-08-14-20-01-14.gh-issue-50002.E-bpj8.rst new file mode 100644 index 0000000000000..ca5c0740802ea --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-14-20-01-14.gh-issue-50002.E-bpj8.rst @@ -0,0 +1 @@ +:mod:`xml.dom.minidom` now preserves whitespaces in attributes. diff --git a/Misc/NEWS.d/next/Library/2023-08-14-20-18-59.gh-issue-81555.cWdP4a.rst b/Misc/NEWS.d/next/Library/2023-08-14-20-18-59.gh-issue-81555.cWdP4a.rst new file mode 100644 index 0000000000000..241a50f8b41c2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-14-20-18-59.gh-issue-81555.cWdP4a.rst @@ -0,0 +1 @@ +:mod:`xml.dom.minidom` now only quotes ``"`` in attributes. From webhook-mailer at python.org Wed Aug 23 11:40:54 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 23 Aug 2023 15:40:54 -0000 Subject: [Python-checkins] gh-108308: Replace PyDict_GetItem() with PyDict_GetItemRef() (#108309) Message-ID: https://github.com/python/cpython/commit/f5559f38d9831e7e55a518e516bcd620ec13af14 commit: f5559f38d9831e7e55a518e516bcd620ec13af14 branch: main author: Victor Stinner committer: vstinner date: 2023-08-23T17:40:26+02:00 summary: gh-108308: Replace PyDict_GetItem() with PyDict_GetItemRef() (#108309) Replace PyDict_GetItem() calls with PyDict_GetItemRef() or PyDict_GetItemWithError() to handle errors. * Replace PyLong_AS_LONG() with _PyLong_AsInt() and check for errors. * Check for PyDict_Contains() error. * pycore_init_builtins() checks for _PyType_Lookup() failure. files: M Python/assemble.c M Python/compile.c M Python/flowgraph.c M Python/pylifecycle.c diff --git a/Python/assemble.c b/Python/assemble.c index b7012534d6cc4..4f66cf294e38c 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -18,7 +18,7 @@ #define ERROR -1 #define RETURN_IF_ERROR(X) \ - if ((X) == -1) { \ + if ((X) < 0) { \ return ERROR; \ } @@ -448,13 +448,17 @@ static PyObject * dict_keys_inorder(PyObject *dict, Py_ssize_t offset) { PyObject *tuple, *k, *v; - Py_ssize_t i, pos = 0, size = PyDict_GET_SIZE(dict); + Py_ssize_t pos = 0, size = PyDict_GET_SIZE(dict); tuple = PyTuple_New(size); if (tuple == NULL) return NULL; while (PyDict_Next(dict, &pos, &k, &v)) { - i = PyLong_AS_LONG(v); + Py_ssize_t i = PyLong_AsSsize_t(v); + if (i == -1 && PyErr_Occurred()) { + Py_DECREF(tuple); + return NULL; + } assert((i - offset) < size); assert((i - offset) >= 0); PyTuple_SET_ITEM(tuple, i - offset, Py_NewRef(k)); @@ -466,24 +470,34 @@ dict_keys_inorder(PyObject *dict, Py_ssize_t offset) extern void _Py_set_localsplus_info(int, PyObject *, unsigned char, PyObject *, PyObject *); -static void +static int compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, PyObject *names, PyObject *kinds) { PyObject *k, *v; Py_ssize_t pos = 0; while (PyDict_Next(umd->u_varnames, &pos, &k, &v)) { - int offset = (int)PyLong_AS_LONG(v); + int offset = _PyLong_AsInt(v); + if (offset == -1 && PyErr_Occurred()) { + return ERROR; + } assert(offset >= 0); assert(offset < nlocalsplus); + // For now we do not distinguish arg kinds. _PyLocals_Kind kind = CO_FAST_LOCAL; - if (PyDict_Contains(umd->u_fasthidden, k)) { + int has_key = PyDict_Contains(umd->u_fasthidden, k); + RETURN_IF_ERROR(has_key); + if (has_key) { kind |= CO_FAST_HIDDEN; } - if (PyDict_GetItem(umd->u_cellvars, k) != NULL) { + + has_key = PyDict_Contains(umd->u_cellvars, k); + RETURN_IF_ERROR(has_key); + if (has_key) { kind |= CO_FAST_CELL; } + _Py_set_localsplus_info(offset, k, kind, names, kinds); } int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); @@ -492,12 +506,18 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, int numdropped = 0; pos = 0; while (PyDict_Next(umd->u_cellvars, &pos, &k, &v)) { - if (PyDict_GetItem(umd->u_varnames, k) != NULL) { + int has_name = PyDict_Contains(umd->u_varnames, k); + RETURN_IF_ERROR(has_name); + if (has_name) { // Skip cells that are already covered by locals. numdropped += 1; continue; } - int offset = (int)PyLong_AS_LONG(v); + + int offset = _PyLong_AsInt(v); + if (offset == -1 && PyErr_Occurred()) { + return ERROR; + } assert(offset >= 0); offset += nlocals - numdropped; assert(offset < nlocalsplus); @@ -506,12 +526,16 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, pos = 0; while (PyDict_Next(umd->u_freevars, &pos, &k, &v)) { - int offset = (int)PyLong_AS_LONG(v); + int offset = _PyLong_AsInt(v); + if (offset == -1 && PyErr_Occurred()) { + return ERROR; + } assert(offset >= 0); offset += nlocals - numdropped; assert(offset < nlocalsplus); _Py_set_localsplus_info(offset, k, CO_FAST_FREE, names, kinds); } + return SUCCESS; } static PyCodeObject * @@ -556,7 +580,10 @@ makecode(_PyCompile_CodeUnitMetadata *umd, struct assembler *a, PyObject *const_ if (localspluskinds == NULL) { goto error; } - compute_localsplus_info(umd, nlocalsplus, localsplusnames, localspluskinds); + if (compute_localsplus_info(umd, nlocalsplus, + localsplusnames, localspluskinds) == ERROR) { + goto error; + } struct _PyCodeConstructor con = { .filename = filename, diff --git a/Python/compile.c b/Python/compile.c index 4b2f70a7ef01d..b67a1885ddc98 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4212,9 +4212,20 @@ compiler_nameop(struct compiler *c, location loc, optype = OP_DEREF; break; case LOCAL: - if (_PyST_IsFunctionLike(c->u->u_ste) || - (PyDict_GetItem(c->u->u_metadata.u_fasthidden, mangled) == Py_True)) + if (_PyST_IsFunctionLike(c->u->u_ste)) { optype = OP_FAST; + } + else { + PyObject *item; + if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, mangled, + &item) < 0) { + goto error; + } + if (item == Py_True) { + optype = OP_FAST; + } + Py_XDECREF(item); + } break; case GLOBAL_IMPLICIT: if (_PyST_IsFunctionLike(c->u->u_ste)) @@ -4239,7 +4250,7 @@ compiler_nameop(struct compiler *c, location loc, op = LOAD_FROM_DICT_OR_DEREF; // First load the locals if (codegen_addop_noarg(INSTR_SEQUENCE(c), LOAD_LOCALS, loc) < 0) { - return ERROR; + goto error; } } else if (c->u->u_ste->ste_can_see_class_scope) { @@ -4247,7 +4258,7 @@ compiler_nameop(struct compiler *c, location loc, // First load the classdict if (compiler_addop_o(c->u, loc, LOAD_DEREF, c->u->u_metadata.u_freevars, &_Py_ID(__classdict__)) < 0) { - return ERROR; + goto error; } } else { @@ -4274,7 +4285,7 @@ compiler_nameop(struct compiler *c, location loc, // First load the classdict if (compiler_addop_o(c->u, loc, LOAD_DEREF, c->u->u_metadata.u_freevars, &_Py_ID(__classdict__)) < 0) { - return ERROR; + goto error; } } else { op = LOAD_GLOBAL; @@ -4308,6 +4319,10 @@ compiler_nameop(struct compiler *c, location loc, arg <<= 1; } return codegen_addop_i(INSTR_SEQUENCE(c), op, arg, loc); + +error: + Py_DECREF(mangled); + return ERROR; } static int @@ -5536,8 +5551,13 @@ push_inlined_comprehension_state(struct compiler *c, location loc, if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { if (!_PyST_IsFunctionLike(c->u->u_ste)) { // non-function scope: override this name to use fast locals - PyObject *orig = PyDict_GetItem(c->u->u_metadata.u_fasthidden, k); - if (orig != Py_True) { + PyObject *orig; + if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, k, &orig) < 0) { + return ERROR; + } + int orig_is_true = (orig == Py_True); + Py_XDECREF(orig); + if (!orig_is_true) { if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) { return ERROR; } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 719ed92105074..e620e7cf1b9e9 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -2404,17 +2404,31 @@ build_cellfixedoffsets(_PyCompile_CodeUnitMetadata *umd) PyObject *varname, *cellindex; Py_ssize_t pos = 0; while (PyDict_Next(umd->u_cellvars, &pos, &varname, &cellindex)) { - PyObject *varindex = PyDict_GetItem(umd->u_varnames, varname); - if (varindex != NULL) { - assert(PyLong_AS_LONG(cellindex) < INT_MAX); - assert(PyLong_AS_LONG(varindex) < INT_MAX); - int oldindex = (int)PyLong_AS_LONG(cellindex); - int argoffset = (int)PyLong_AS_LONG(varindex); - fixed[oldindex] = argoffset; + PyObject *varindex; + if (PyDict_GetItemRef(umd->u_varnames, varname, &varindex) < 0) { + goto error; + } + if (varindex == NULL) { + continue; + } + + int argoffset = _PyLong_AsInt(varindex); + Py_DECREF(varindex); + if (argoffset == -1 && PyErr_Occurred()) { + goto error; } - } + int oldindex = _PyLong_AsInt(cellindex); + if (oldindex == -1 && PyErr_Occurred()) { + goto error; + } + fixed[oldindex] = argoffset; + } return fixed; + +error: + PyMem_Free(fixed); + return NULL; } #define IS_GENERATOR(CF) \ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 9837e0f0d52e6..8c321fbcfe410 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -762,18 +762,30 @@ pycore_init_builtins(PyThreadState *tstate) } interp->builtins = Py_NewRef(builtins_dict); - PyObject *isinstance = PyDict_GetItem(builtins_dict, &_Py_ID(isinstance)); - assert(isinstance); + PyObject *isinstance = PyDict_GetItemWithError(builtins_dict, &_Py_ID(isinstance)); + if (!isinstance) { + goto error; + } interp->callable_cache.isinstance = isinstance; - PyObject *len = PyDict_GetItem(builtins_dict, &_Py_ID(len)); - assert(len); + + PyObject *len = PyDict_GetItemWithError(builtins_dict, &_Py_ID(len)); + if (!len) { + goto error; + } interp->callable_cache.len = len; + PyObject *list_append = _PyType_Lookup(&PyList_Type, &_Py_ID(append)); - assert(list_append); + if (list_append == NULL) { + goto error; + } interp->callable_cache.list_append = list_append; + PyObject *object__getattribute__ = _PyType_Lookup(&PyBaseObject_Type, &_Py_ID(__getattribute__)); - assert(object__getattribute__); + if (object__getattribute__ == NULL) { + goto error; + } interp->callable_cache.object__getattribute__ = object__getattribute__; + if (_PyBuiltins_AddExceptions(bimod) < 0) { return _PyStatus_ERR("failed to add exceptions to builtins"); } From webhook-mailer at python.org Wed Aug 23 11:44:51 2023 From: webhook-mailer at python.org (vstinner) Date: Wed, 23 Aug 2023 15:44:51 -0000 Subject: [Python-checkins] gh-108303: Move `ann_module*.py` files to `typinganndata/` folder (#108354) Message-ID: https://github.com/python/cpython/commit/3f61cf646d0506baa0c0c2118f05110446519c62 commit: 3f61cf646d0506baa0c0c2118f05110446519c62 branch: main author: Nikita Sobolev committer: vstinner date: 2023-08-23T17:42:08+02:00 summary: gh-108303: Move `ann_module*.py` files to `typinganndata/` folder (#108354) files: A Lib/test/typinganndata/ann_module.py A Lib/test/typinganndata/ann_module2.py A Lib/test/typinganndata/ann_module3.py A Lib/test/typinganndata/ann_module4.py A Lib/test/typinganndata/ann_module5.py A Lib/test/typinganndata/ann_module6.py A Lib/test/typinganndata/ann_module7.py A Lib/test/typinganndata/ann_module8.py D Lib/test/ann_module.py D Lib/test/ann_module2.py D Lib/test/ann_module3.py D Lib/test/ann_module4.py D Lib/test/ann_module5.py D Lib/test/ann_module6.py D Lib/test/ann_module7.py D Lib/test/ann_module8.py M Lib/test/test_grammar.py M Lib/test/test_inspect.py M Lib/test/test_module/__init__.py M Lib/test/test_opcodes.py M Lib/test/test_typing.py diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index ee105a3de17f8..8507a07e49853 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -12,9 +12,9 @@ # different import patterns to check that __annotations__ does not interfere # with import machinery -import test.ann_module as ann_module +import test.typinganndata.ann_module as ann_module import typing -from test import ann_module2 +from test.typinganndata import ann_module2 import test # These are shared with test_tokenize and other test modules. @@ -452,7 +452,7 @@ def test_var_annot_module_semantics(self): def test_var_annot_in_module(self): # check that functions fail the same way when executed # outside of module where they were defined - ann_module3 = import_helper.import_fresh_module("test.ann_module3") + ann_module3 = import_helper.import_fresh_module("test.typinganndata.ann_module3") with self.assertRaises(NameError): ann_module3.f_bad_ann() with self.assertRaises(NameError): diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 07c48eac5b48b..9cb92c02d3e7d 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -4726,7 +4726,7 @@ def func(*args, **kwargs): def test_base_class_have_text_signature(self): # see issue 43118 - from test.ann_module7 import BufferedReader + from test.typinganndata.ann_module7 import BufferedReader class MyBufferedReader(BufferedReader): """buffer reader class.""" diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index cfc4d9ccf1cc8..2524e6c87cb45 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -324,7 +324,9 @@ def test_annotations_getset_raises(self): del foo.__annotations__ def test_annotations_are_created_correctly(self): - ann_module4 = import_helper.import_fresh_module('test.ann_module4') + ann_module4 = import_helper.import_fresh_module( + 'test.typinganndata.ann_module4', + ) self.assertTrue("__annotations__" in ann_module4.__dict__) del ann_module4.__annotations__ self.assertFalse("__annotations__" in ann_module4.__dict__) diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py index e880c3f1ac875..72488b2bb6b4f 100644 --- a/Lib/test/test_opcodes.py +++ b/Lib/test/test_opcodes.py @@ -1,7 +1,8 @@ # Python test set -- part 2, opcodes import unittest -from test import ann_module, support +from test import support +from test.typinganndata import ann_module class OpcodeTest(unittest.TestCase): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index fa39c79619795..38baf9546f8b0 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -5377,7 +5377,7 @@ def test_errors(self): # We need this to make sure that `@no_type_check` respects `__module__` attr: -from test import ann_module8 +from test.typinganndata import ann_module8 @no_type_check class NoTypeCheck_Outer: @@ -5968,7 +5968,9 @@ def test_overload_registry_repeated(self): # Definitions needed for features introduced in Python 3.6 -from test import ann_module, ann_module2, ann_module3, ann_module5, ann_module6 +from test.typinganndata import ( + ann_module, ann_module2, ann_module3, ann_module5, ann_module6, +) T_a = TypeVar('T_a') diff --git a/Lib/test/ann_module.py b/Lib/test/typinganndata/ann_module.py similarity index 100% rename from Lib/test/ann_module.py rename to Lib/test/typinganndata/ann_module.py diff --git a/Lib/test/ann_module2.py b/Lib/test/typinganndata/ann_module2.py similarity index 100% rename from Lib/test/ann_module2.py rename to Lib/test/typinganndata/ann_module2.py diff --git a/Lib/test/ann_module3.py b/Lib/test/typinganndata/ann_module3.py similarity index 100% rename from Lib/test/ann_module3.py rename to Lib/test/typinganndata/ann_module3.py diff --git a/Lib/test/ann_module4.py b/Lib/test/typinganndata/ann_module4.py similarity index 100% rename from Lib/test/ann_module4.py rename to Lib/test/typinganndata/ann_module4.py diff --git a/Lib/test/ann_module5.py b/Lib/test/typinganndata/ann_module5.py similarity index 100% rename from Lib/test/ann_module5.py rename to Lib/test/typinganndata/ann_module5.py diff --git a/Lib/test/ann_module6.py b/Lib/test/typinganndata/ann_module6.py similarity index 100% rename from Lib/test/ann_module6.py rename to Lib/test/typinganndata/ann_module6.py diff --git a/Lib/test/ann_module7.py b/Lib/test/typinganndata/ann_module7.py similarity index 100% rename from Lib/test/ann_module7.py rename to Lib/test/typinganndata/ann_module7.py diff --git a/Lib/test/ann_module8.py b/Lib/test/typinganndata/ann_module8.py similarity index 100% rename from Lib/test/ann_module8.py rename to Lib/test/typinganndata/ann_module8.py From webhook-mailer at python.org Wed Aug 23 11:45:29 2023 From: webhook-mailer at python.org (gvanrossum) Date: Wed, 23 Aug 2023 15:45:29 -0000 Subject: [Python-checkins] gh-107265: Ensure de_instrument does not handle ENTER_EXECUTOR (#108366) Message-ID: https://github.com/python/cpython/commit/2135bcd3ca9538c6782129f9a5837d62c2036102 commit: 2135bcd3ca9538c6782129f9a5837d62c2036102 branch: main author: Dong-hee Na committer: gvanrossum date: 2023-08-23T08:45:20-07:00 summary: gh-107265: Ensure de_instrument does not handle ENTER_EXECUTOR (#108366) files: M Python/instrumentation.c diff --git a/Python/instrumentation.c b/Python/instrumentation.c index a7a5b4a5dc5f6..f77c2e696f453 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -564,6 +564,7 @@ de_instrument(PyCodeObject *code, int i, int event) _Py_CODEUNIT *instr = &_PyCode_CODE(code)[i]; uint8_t *opcode_ptr = &instr->op.code; int opcode = *opcode_ptr; + assert(opcode != ENTER_EXECUTOR); if (opcode == INSTRUMENTED_LINE) { opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; opcode = *opcode_ptr; From webhook-mailer at python.org Wed Aug 23 13:12:46 2023 From: webhook-mailer at python.org (serhiy-storchaka) Date: Wed, 23 Aug 2023 17:12:46 -0000 Subject: [Python-checkins] gh-107298: Fix a few more refs in the C API docs (GH-108361) Message-ID: https://github.com/python/cpython/commit/422f81b5d2359063826b8561f698d57e94f6a5d8 commit: 422f81b5d2359063826b8561f698d57e94f6a5d8 branch: main author: Serhiy Storchaka committer: serhiy-storchaka date: 2023-08-23T20:12:42+03:00 summary: gh-107298: Fix a few more refs in the C API docs (GH-108361) gh-107298: Fix few more refs in the C API docs files: M Doc/c-api/typeobj.rst M Doc/whatsnew/2.6.rst M Doc/whatsnew/2.7.rst M Doc/whatsnew/3.4.rst M Doc/whatsnew/3.7.rst M Misc/NEWS.d/3.11.0b1.rst M Misc/NEWS.d/3.7.0a4.rst M Misc/NEWS.d/3.8.0a4.rst diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index cd037b4de882e..acaf0eced35b3 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1403,7 +1403,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) are allowed to be removed even if the instance is still alive). Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to - :c:func:`local_traverse` to have these specific names; don't name them just + :c:func:`!local_traverse` to have these specific names; don't name them just anything. Instances of :ref:`heap-allocated types ` hold a reference to diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 8b3d3a324f68f..beba4428e67c3 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -121,7 +121,7 @@ about features that will be removed in Python 3.0. You can run code with this switch to see how much work will be necessary to port code to 3.0. The value of this switch is available to Python code as the boolean variable :data:`sys.py3kwarning`, -and to C extension code as :c:data:`Py_Py3kWarningFlag`. +and to C extension code as :c:data:`!Py_Py3kWarningFlag`. .. seealso:: diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 37e2d04d516ab..e82e8e4db1abc 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2155,13 +2155,13 @@ Changes to Python's build process and to the C API include: :c:func:`!PyOS_ascii_strtod` and :c:func:`!PyOS_ascii_atof` functions are now deprecated. -* New function: :c:func:`PySys_SetArgvEx` sets the value of +* New function: :c:func:`!PySys_SetArgvEx` sets the value of ``sys.argv`` and can optionally update ``sys.path`` to include the directory containing the script named by ``sys.argv[0]`` depending on the value of an *updatepath* parameter. This function was added to close a security hole for applications - that embed Python. The old function, :c:func:`PySys_SetArgv`, would + that embed Python. The old function, :c:func:`!PySys_SetArgv`, would always update ``sys.path``, and sometimes it would add the current directory. This meant that, if you ran an application embedding Python in a directory controlled by someone else, attackers could @@ -2169,8 +2169,8 @@ Changes to Python's build process and to the C API include: :file:`os.py`) that your application would then import and run. If you maintain a C/C++ application that embeds Python, check - whether you're calling :c:func:`PySys_SetArgv` and carefully consider - whether the application should be using :c:func:`PySys_SetArgvEx` + whether you're calling :c:func:`!PySys_SetArgv` and carefully consider + whether the application should be using :c:func:`!PySys_SetArgvEx` with *updatepath* set to false. Security issue reported as `CVE-2008-5983 @@ -2545,11 +2545,11 @@ For C extensions: For applications that embed Python: -* The :c:func:`PySys_SetArgvEx` function was added, letting +* The :c:func:`!PySys_SetArgvEx` function was added, letting applications close a security hole when the existing - :c:func:`PySys_SetArgv` function was used. Check whether you're - calling :c:func:`PySys_SetArgv` and carefully consider whether the - application should be using :c:func:`PySys_SetArgvEx` with + :c:func:`!PySys_SetArgv` function was used. Check whether you're + calling :c:func:`!PySys_SetArgv` and carefully consider whether the + application should be using :c:func:`!PySys_SetArgvEx` with *updatepath* set to false. .. ====================================================================== diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 794271f3c32b8..a36e9fa852723 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -1891,7 +1891,7 @@ Other Build and C API Changes allowing retrieval of function pointers from named type slots when using the limited API. (Contributed by Martin von L?wis in :issue:`17162`.) -* The new :c:func:`Py_SetStandardStreamEncoding` pre-initialization API +* The new :c:func:`!Py_SetStandardStreamEncoding` pre-initialization API allows applications embedding the CPython interpreter to reliably force a particular encoding and error handler for the standard streams. (Contributed by Bastien Montagne and Nick Coghlan in :issue:`16129`.) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 218a37cd264c7..eb083737b0b7e 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -2495,12 +2495,12 @@ either in embedding applications, or in CPython itself. :issue:`22257`, and further updated by Nick, Eric, and Victor Stinner in a number of other issues). Some known details affected: -* :c:func:`PySys_AddWarnOptionUnicode` is not currently usable by embedding +* :c:func:`!PySys_AddWarnOptionUnicode` is not currently usable by embedding applications due to the requirement to create a Unicode object prior to - calling ``Py_Initialize``. Use :c:func:`PySys_AddWarnOption` instead. + calling ``Py_Initialize``. Use :c:func:`!PySys_AddWarnOption` instead. * warnings filters added by an embedding application with - :c:func:`PySys_AddWarnOption` should now more consistently take precedence + :c:func:`!PySys_AddWarnOption` should now more consistently take precedence over the default filters set by the interpreter Due to changes in the way the default warnings filters are configured, diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index 766ada4f8e117..2bcccc7dae373 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -2028,8 +2028,8 @@ https://gitlab.com/warsaw/pynche .. nonce: 3mQ54t .. section: C API -Deprecate the C functions: :c:func:`PySys_SetArgv`, -:c:func:`PySys_SetArgvEx`, :c:func:`PySys_SetPath`. Patch by Victor Stinner. +Deprecate the C functions: :c:func:`!PySys_SetArgv`, +:c:func:`!PySys_SetArgvEx`, :c:func:`!PySys_SetPath`. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.7.0a4.rst b/Misc/NEWS.d/3.7.0a4.rst index f19d1a1823584..ebae046a7a6ba 100644 --- a/Misc/NEWS.d/3.7.0a4.rst +++ b/Misc/NEWS.d/3.7.0a4.rst @@ -842,5 +842,5 @@ Moved the pygetopt.h header into internal/, since it has no public APIs. .. nonce: LbyQt6 .. section: C API -:c:func:`Py_SetProgramName` and :c:func:`Py_SetPythonHome` now take the +:c:func:`!Py_SetProgramName` and :c:func:`!Py_SetPythonHome` now take the ``const wchar *`` arguments instead of ``wchar *``. diff --git a/Misc/NEWS.d/3.8.0a4.rst b/Misc/NEWS.d/3.8.0a4.rst index 2ce60f39539e8..da03d93eae396 100644 --- a/Misc/NEWS.d/3.8.0a4.rst +++ b/Misc/NEWS.d/3.8.0a4.rst @@ -1344,7 +1344,7 @@ Fix the argument handling in Tools/scripts/lll.py. .. nonce: vghb86 .. section: C API -Fix memory leak in :c:func:`Py_SetStandardStreamEncoding`: release memory if +Fix memory leak in :c:func:`!Py_SetStandardStreamEncoding`: release memory if the function is called twice. .. From webhook-mailer at python.org Wed Aug 23 13:39:03 2023 From: webhook-mailer at python.org (iritkatriel) Date: Wed, 23 Aug 2023 17:39:03 -0000 Subject: [Python-checkins] gh-105481: remove regen-opcode. Generated _PyOpcode_Caches in regen-cases. (#108367) Message-ID: https://github.com/python/cpython/commit/72119d16a5f658939809febef29dadeca02cf34d commit: 72119d16a5f658939809febef29dadeca02cf34d branch: main author: Irit Katriel <1055913+iritkatriel at users.noreply.github.com> committer: iritkatriel <1055913+iritkatriel at users.noreply.github.com> date: 2023-08-23T18:39:00+01:00 summary: gh-105481: remove regen-opcode. Generated _PyOpcode_Caches in regen-cases. (#108367) files: A Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst D Include/internal/pycore_opcode.h D Tools/build/generate_opcode_h.py M Include/internal/pycore_opcode_metadata.h M Include/internal/pycore_opcode_utils.h M Lib/opcode.py M Makefile.pre.in M Objects/codeobject.c M Objects/frameobject.c M PCbuild/regen.targets M Python/assemble.c M Python/ceval.c M Python/compile.c M Python/executor.c M Python/instrumentation.c M Python/optimizer.c M Python/optimizer_analysis.c M Python/specialize.c M Tools/cases_generator/generate_cases.py diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h deleted file mode 100644 index b47e796485236..0000000000000 --- a/Include/internal/pycore_opcode.h +++ /dev/null @@ -1,40 +0,0 @@ -// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py - -#ifndef Py_INTERNAL_OPCODE_H -#define Py_INTERNAL_OPCODE_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#include "opcode.h" - -extern const uint8_t _PyOpcode_Caches[256]; - -#ifdef NEED_OPCODE_TABLES - -const uint8_t _PyOpcode_Caches[256] = { - [LOAD_GLOBAL] = 4, - [BINARY_OP] = 1, - [UNPACK_SEQUENCE] = 1, - [COMPARE_OP] = 1, - [BINARY_SUBSCR] = 1, - [FOR_ITER] = 1, - [LOAD_SUPER_ATTR] = 1, - [LOAD_ATTR] = 9, - [STORE_ATTR] = 4, - [CALL] = 3, - [STORE_SUBSCR] = 1, - [SEND] = 1, - [JUMP_BACKWARD] = 1, - [TO_BOOL] = 3, -}; -#endif // NEED_OPCODE_TABLES - -#ifdef __cplusplus -} -#endif -#endif // !Py_INTERNAL_OPCODE_H diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index e35db0c4c5a59..cc8894ad53933 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1814,6 +1814,26 @@ const char *const _PyOpcode_OpName[268] = { }; #endif // NEED_OPCODE_METADATA +extern const uint8_t _PyOpcode_Caches[256]; +#ifdef NEED_OPCODE_METADATA +const uint8_t _PyOpcode_Caches[256] = { + [TO_BOOL] = 3, + [BINARY_OP] = 1, + [BINARY_SUBSCR] = 1, + [STORE_SUBSCR] = 1, + [SEND] = 1, + [UNPACK_SEQUENCE] = 1, + [STORE_ATTR] = 4, + [LOAD_GLOBAL] = 4, + [LOAD_SUPER_ATTR] = 1, + [LOAD_ATTR] = 9, + [COMPARE_OP] = 1, + [FOR_ITER] = 1, + [CALL] = 3, + [JUMP_BACKWARD] = 1, +}; +#endif // NEED_OPCODE_METADATA + extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index f17612908cebd..c4acb00a4b291 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -8,8 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_opcode.h" // JUMP_FORWARD - +#include "opcode_ids.h" #define MAX_REAL_OPCODE 254 diff --git a/Lib/opcode.py b/Lib/opcode.py index f8487522bfdc6..386a2fba396a6 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -5,48 +5,40 @@ """ -# Note that __all__ is further extended below -__all__ = ["cmp_op", "stack_effect", "hascompare"] +__all__ = ["cmp_op", "stack_effect", "hascompare", "opname", "opmap", + "HAVE_ARGUMENT", "EXTENDED_ARG", "hasarg", "hasconst", "hasname", + "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc"] import _opcode from _opcode import stack_effect -import sys -# The build uses older versions of Python which do not have _opcode_metadata -if sys.version_info[:2] >= (3, 13): - from _opcode_metadata import _specializations, _specialized_opmap - from _opcode_metadata import opmap, HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE - EXTENDED_ARG = opmap['EXTENDED_ARG'] +from _opcode_metadata import (_specializations, _specialized_opmap, opmap, + HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) +EXTENDED_ARG = opmap['EXTENDED_ARG'] - opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] - for op, i in opmap.items(): - opname[i] = op - - __all__.extend(["opname", "opmap", "HAVE_ARGUMENT", "EXTENDED_ARG"]) +opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] +for op, i in opmap.items(): + opname[i] = op cmp_op = ('<', '<=', '==', '!=', '>', '>=') -# The build uses older versions of Python which do not have _opcode.has_* functions -if sys.version_info[:2] >= (3, 13): - # These lists are documented as part of the dis module's API - hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] - hasconst = [op for op in opmap.values() if _opcode.has_const(op)] - hasname = [op for op in opmap.values() if _opcode.has_name(op)] - hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] - hasjrel = hasjump # for backward compatibility - hasjabs = [] - hasfree = [op for op in opmap.values() if _opcode.has_free(op)] - haslocal = [op for op in opmap.values() if _opcode.has_local(op)] - hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] +# These lists are documented as part of the dis module's API +hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] +hasconst = [op for op in opmap.values() if _opcode.has_const(op)] +hasname = [op for op in opmap.values() if _opcode.has_name(op)] +hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] +hasjrel = hasjump # for backward compatibility +hasjabs = [] +hasfree = [op for op in opmap.values() if _opcode.has_free(op)] +haslocal = [op for op in opmap.values() if _opcode.has_local(op)] +hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] - __all__.extend(["hasarg", "hasconst", "hasname", "hasjump", "hasjrel", - "hasjabs", "hasfree", "haslocal", "hasexc"]) - _intrinsic_1_descs = _opcode.get_intrinsic1_descs() - _intrinsic_2_descs = _opcode.get_intrinsic2_descs() - _nb_ops = _opcode.get_nb_ops() +_intrinsic_1_descs = _opcode.get_intrinsic1_descs() +_intrinsic_2_descs = _opcode.get_intrinsic2_descs() +_nb_ops = _opcode.get_nb_ops() - hascompare = [opmap["COMPARE_OP"]] +hascompare = [opmap["COMPARE_OP"]] _cache_format = { "LOAD_GLOBAL": { diff --git a/Makefile.pre.in b/Makefile.pre.in index d66764e616540..54b64e123cd7b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1321,7 +1321,7 @@ regen-limited-abi: all # Regenerate all generated files .PHONY: regen-all -regen-all: regen-cases regen-opcode regen-typeslots \ +regen-all: regen-cases regen-typeslots \ regen-token regen-ast regen-keyword regen-sre regen-frozen clinic \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ regen-test-levenshtein regen-global-objects @@ -1425,15 +1425,6 @@ regen-ast: $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_ast_state.h $(srcdir)/Include/internal/pycore_ast_state.h.new $(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new -.PHONY: regen-opcode -regen-opcode: - # Regenerate Include/internal/pycore_opcode.h from Lib/opcode.py - # using Tools/build/generate_opcode_h.py - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \ - $(srcdir)/Lib/opcode.py \ - $(srcdir)/Include/internal/pycore_opcode.h.new - $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new - .PHONY: regen-token regen-token: # Regenerate Doc/library/token-list.inc from Grammar/Tokens @@ -1651,6 +1642,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/object.h \ $(srcdir)/Include/objimpl.h \ $(srcdir)/Include/opcode.h \ + $(srcdir)/Include/opcode_ids.h \ $(srcdir)/Include/osdefs.h \ $(srcdir)/Include/osmodule.h \ $(srcdir)/Include/patchlevel.h \ @@ -1790,7 +1782,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_object_state.h \ $(srcdir)/Include/internal/pycore_obmalloc.h \ $(srcdir)/Include/internal/pycore_obmalloc_init.h \ - $(srcdir)/Include/internal/pycore_opcode.h \ + $(srcdir)/Include/internal/pycore_opcode_metadata.h \ $(srcdir)/Include/internal/pycore_opcode_utils.h \ $(srcdir)/Include/internal/pycore_optimizer.h \ $(srcdir)/Include/internal/pycore_pathconfig.h \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst b/Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst new file mode 100644 index 0000000000000..19746ebb701d0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-08-23-14-54-15.gh-issue-105481.40q-c4.rst @@ -0,0 +1,2 @@ +The regen-opcode build stage was removed and its work is now done in +regen-cases. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index dca5804a91d2c..70a0c2ebd66b2 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -6,8 +6,7 @@ #include "pycore_code.h" // _PyCodeConstructor #include "pycore_frame.h" // FRAME_SPECIALS_SIZE #include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs -#include "pycore_opcode.h" // _PyOpcode_Caches -#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt +#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_tuple.h" // _PyTuple_ITEMS() diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 80e118e8a8aa9..28f5a5a122280 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -6,8 +6,7 @@ #include "pycore_function.h" // _PyFunction_FromConstructor() #include "pycore_moduleobject.h" // _PyModule_GetDict() #include "pycore_object.h" // _PyObject_GC_UNTRACK() -#include "pycore_opcode.h" // _PyOpcode_Caches -#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt +#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches #include "frameobject.h" // PyFrameObject diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 2ff18a8966f55..cc9469c7ddd72 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -13,8 +13,6 @@ <_ASTOutputs Include="$(PySourcePath)Python\Python-ast.c"> -C - <_OpcodeSources Include="$(PySourcePath)Tools\build\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" /> - <_OpcodeOutputs Include="$(PySourcePath)Include\internal\pycore_opcode.h" /> <_TokenSources Include="$(PySourcePath)Grammar\Tokens" /> <_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc"> rst @@ -34,7 +32,7 @@ - @@ -55,14 +53,6 @@ WorkingDirectory="$(PySourcePath)" /> - - - - - @@ -89,7 +79,7 @@ + DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenTokens;_RegenKeywords;_RegenGlobalObjects"> diff --git a/Python/assemble.c b/Python/assemble.c index 4f66cf294e38c..c770fd108d41d 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -3,9 +3,8 @@ #include "Python.h" #include "pycore_code.h" // write_location_entry_start() #include "pycore_compile.h" -#include "pycore_opcode.h" // _PyOpcode_Caches[] and opcode category macros #include "pycore_opcode_utils.h" // IS_BACKWARDS_JUMP_OPCODE -#include "pycore_opcode_metadata.h" // IS_PSEUDO_INSTR +#include "pycore_opcode_metadata.h" // IS_PSEUDO_INSTR, _PyOpcode_Caches #define DEFAULT_CODE_SIZE 128 diff --git a/Python/ceval.c b/Python/ceval.c index f7dfaebcdd8f8..55dfe6b1efc47 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -14,8 +14,7 @@ #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_opcode.h" // EXTRA_CASES -#include "pycore_opcode_metadata.h" +#include "pycore_opcode_metadata.h" // EXTRA_CASES #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Python/compile.c b/Python/compile.c index b67a1885ddc98..6b816b4c6eda6 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -24,6 +24,7 @@ #include #include "Python.h" +#include "opcode.h" #include "pycore_ast.h" // _PyAST_GetDocString() #define NEED_OPCODE_TABLES #include "pycore_opcode_utils.h" diff --git a/Python/executor.c b/Python/executor.c index 88c039da8539e..0ff5106fe446f 100644 --- a/Python/executor.c +++ b/Python/executor.c @@ -1,5 +1,7 @@ #include "Python.h" +#include "opcode.h" + #include "pycore_call.h" #include "pycore_ceval.h" #include "pycore_dict.h" diff --git a/Python/instrumentation.c b/Python/instrumentation.c index f77c2e696f453..8c7a3a06c9b93 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1,4 +1,7 @@ #include "Python.h" + +#include "opcode_ids.h" + #include "pycore_call.h" #include "pycore_frame.h" #include "pycore_interp.h" @@ -6,8 +9,7 @@ #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_namespace.h" #include "pycore_object.h" -#include "pycore_opcode.h" -#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE +#include "pycore_opcode_metadata.h" // IS_VALID_OPCODE, _PyOpcode_Caches #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Python/optimizer.c b/Python/optimizer.c index 57518404c3f19..bbc125954c701 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1,7 +1,6 @@ #include "Python.h" #include "opcode.h" #include "pycore_interp.h" -#include "pycore_opcode.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index e48e018052c71..2d177f14ff268 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -1,7 +1,6 @@ #include "Python.h" #include "opcode.h" #include "pycore_interp.h" -#include "pycore_opcode.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() diff --git a/Python/specialize.c b/Python/specialize.c index 2d514c0dc476d..a467f163f2ca9 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1,4 +1,7 @@ #include "Python.h" + +#include "opcode.h" + #include "pycore_code.h" #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" @@ -7,7 +10,7 @@ #include "pycore_long.h" #include "pycore_moduleobject.h" #include "pycore_object.h" -#include "pycore_opcode.h" // _PyOpcode_Caches +#include "pycore_opcode_metadata.h" // _PyOpcode_Caches #include "pycore_pylifecycle.h" // _PyOS_URandomNonblock() diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py deleted file mode 100644 index 643918c1bc2ce..0000000000000 --- a/Tools/build/generate_opcode_h.py +++ /dev/null @@ -1,67 +0,0 @@ -# This script generates the pycore_opcode.h header file. - -import sys -import tokenize - -SCRIPT_NAME = "Tools/build/generate_opcode_h.py" -PYTHON_OPCODE = "Lib/opcode.py" - -internal_header = f""" -// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} - -#ifndef Py_INTERNAL_OPCODE_H -#define Py_INTERNAL_OPCODE_H -#ifdef __cplusplus -extern "C" {{ -#endif - -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - -#include "opcode.h" -""".lstrip() - -internal_footer = """ -#ifdef __cplusplus -} -#endif -#endif // !Py_INTERNAL_OPCODE_H -""" - -DEFINE = "#define {:<38} {:>3}\n" - -UINT32_MASK = (1<<32)-1 - -def get_python_module_dict(filename): - mod = {} - with tokenize.open(filename) as fp: - code = fp.read() - exec(code, mod) - return mod - -def main(opcode_py, - internal_opcode_h='Include/internal/pycore_opcode.h'): - - opcode = get_python_module_dict(opcode_py) - - with open(internal_opcode_h, 'w') as iobj: - iobj.write(internal_header) - - iobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n") - iobj.write("\n#ifdef NEED_OPCODE_TABLES\n") - - iobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n") - for name, entries in opcode["_inline_cache_entries"].items(): - iobj.write(f" [{name}] = {entries},\n") - iobj.write("};\n") - - iobj.write("#endif // NEED_OPCODE_TABLES\n") - - iobj.write(internal_footer) - - print(f"{internal_opcode_h} regenerated from {opcode_py}") - - -if __name__ == '__main__': - main(sys.argv[1], sys.argv[2]) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index f605dcc5e4632..8f9a6502e529c 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -540,6 +540,18 @@ def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> No for name in self.opmap: self.out.emit(f'[{name}] = "{name}",') + with self.metadata_item( + f"const uint8_t _PyOpcode_Caches[256]", + "=", + ";", + ): + for name, _ in self.families.items(): + instr = self.instrs[name] + if instr.cache_offset > 0: + self.out.emit(f'[{name}] = {instr.cache_offset},') + # Irregular case: + self.out.emit('[JUMP_BACKWARD] = 1,') + deoptcodes = {} for name, op in self.opmap.items(): if op < 256: From webhook-mailer at python.org Wed Aug 23 14:00:13 2023 From: webhook-mailer at python.org (encukou) Date: Wed, 23 Aug 2023 18:00:13 -0000 Subject: [Python-checkins] gh-107811: tarfile: treat overflow in UID/GID as failure to set it (#108369) Message-ID: https://github.com/python/cpython/commit/5d1871576500adc4ebaa7f59b8559605a57ad36b commit: 5d1871576500adc4ebaa7f59b8559605a57ad36b branch: main author: Petr Viktorin committer: encukou date: 2023-08-23T20:00:07+02:00 summary: gh-107811: tarfile: treat overflow in UID/GID as failure to set it (#108369) files: A Misc/NEWS.d/next/Library/2023-08-23-17-34-39.gh-issue-107811.3Fng72.rst M Lib/tarfile.py diff --git a/Lib/tarfile.py b/Lib/tarfile.py index a835d00c90c92..726f9f50ba2e7 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2557,7 +2557,8 @@ def chown(self, tarinfo, targetpath, numeric_owner): os.lchown(targetpath, u, g) else: os.chown(targetpath, u, g) - except OSError as e: + except (OSError, OverflowError) as e: + # OverflowError can be raised if an ID doesn't fit in `id_t` raise ExtractError("could not change owner") from e def chmod(self, tarinfo, targetpath): diff --git a/Misc/NEWS.d/next/Library/2023-08-23-17-34-39.gh-issue-107811.3Fng72.rst b/Misc/NEWS.d/next/Library/2023-08-23-17-34-39.gh-issue-107811.3Fng72.rst new file mode 100644 index 0000000000000..ffca4131db228 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-08-23-17-34-39.gh-issue-107811.3Fng72.rst @@ -0,0 +1,3 @@ +:mod:`tarfile`: extraction of members with overly large UID or GID (e.g. on +an OS with 32-bit :c:type:`!id_t`) now fails in the same way as failing to +set the ID. From webhook-mailer at python.org Wed Aug 23 16:27:39 2023 From: webhook-mailer at python.org (pitrou) Date: Wed, 23 Aug 2023 20:27:39 -0000 Subject: [Python-checkins] gh-77377: Ensure multiprocessing SemLock is valid for spawn-based Process before serializing it (#107275) Message-ID: https://github.com/python/cpython/commit/1700d34d314f5304a7a75363bda295a8c15c371f commit: 1700d34d314f5304a7a75363bda295a8c15c371f branch: main author: albanD committer: pitrou date: 2023-08-23T20:27:35Z summary: gh-77377: Ensure multiprocessing SemLock is valid for spawn-based Process before serializing it (#107275) Ensure multiprocessing SemLock is valid for spawn Process before serializing it. Creating a multiprocessing SemLock with a fork context, and then trying to pass it to a spawn-created Process, would segfault if not detected early. --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Antoine Pitrou files: A Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst M Lib/multiprocessing/synchronize.py M Lib/test/_test_multiprocessing.py diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 42624b543601a..2328d33212308 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -50,8 +50,8 @@ class SemLock(object): def __init__(self, kind, value, maxvalue, *, ctx): if ctx is None: ctx = context._default_context.get_context() - name = ctx.get_start_method() - unlink_now = sys.platform == 'win32' or name == 'fork' + self.is_fork_ctx = ctx.get_start_method() == 'fork' + unlink_now = sys.platform == 'win32' or self.is_fork_ctx for i in range(100): try: sl = self._semlock = _multiprocessing.SemLock( @@ -103,6 +103,11 @@ def __getstate__(self): if sys.platform == 'win32': h = context.get_spawning_popen().duplicate_for_child(sl.handle) else: + if self.is_fork_ctx: + raise RuntimeError('A SemLock created in a fork context is being ' + 'shared with a process in a spawn context. This is ' + 'not supported. Please use the same context to create ' + 'multiprocessing objects and Process.') h = sl.handle return (h, sl.kind, sl.maxvalue, sl.name) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 10754964e73bc..a826e31f5ee07 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5421,6 +5421,28 @@ def test_preload_resources(self): print(err) self.fail("failed spawning forkserver or grandchild") + @unittest.skipIf(sys.platform == "win32", + "Only Spawn on windows so no risk of mixing") + @only_run_in_spawn_testsuite("avoids redundant testing.") + def test_mixed_startmethod(self): + # Fork-based locks cannot be used with spawned process + for process_method in ["spawn", "forkserver"]: + queue = multiprocessing.get_context("fork").Queue() + process_ctx = multiprocessing.get_context(process_method) + p = process_ctx.Process(target=close_queue, args=(queue,)) + err_msg = "A SemLock created in a fork" + with self.assertRaisesRegex(RuntimeError, err_msg): + p.start() + + # non-fork-based locks can be used with all other start methods + for queue_method in ["spawn", "forkserver"]: + for process_method in multiprocessing.get_all_start_methods(): + queue = multiprocessing.get_context(queue_method).Queue() + process_ctx = multiprocessing.get_context(process_method) + p = process_ctx.Process(target=close_queue, args=(queue,)) + p.start() + p.join() + @unittest.skipIf(sys.platform == "win32", "test semantics don't make sense on Windows") diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst new file mode 100644 index 0000000000000..194851dea1335 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst @@ -0,0 +1 @@ +Ensure that multiprocessing synchronization objects created in a fork context are not sent to a different process created in a spawn context. This changes a segfault into an actionable RuntimeError in the parent process.